yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4688 from umbraco/temp8-4674-media-paths

Browse Source 
Fix media paths length issues
This commit is contained in:
Bjarke Berg
2019-02-21 13:21:39 +01:00
committed by GitHub
parent c21039f1af 9ecd63eaed
commit d29d629eae
11 changed files with 188 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Umbraco.Core/Composing/CompositionExtensions/FileSystems.cs
View File

@@ -79,7 +79,7 @@ namespace Umbraco.Core.Composing.CompositionExtensions
composition.RegisterUnique<IFileSystems>(factory => factory.GetInstance<IO.FileSystems>());
// register the scheme for media paths
composition.RegisterUnique<IMediaPathScheme, TwoGuidsMediaPathScheme>();
composition.RegisterUnique<IMediaPathScheme, UniqueMediaPathScheme>();
// register the IMediaFileSystem implementation
composition.RegisterFileSystem<IMediaFileSystem, MediaFileSystem>();

67
src/Umbraco.Core/GuidUtils.cs
View File

@@ -40,5 +40,72 @@ namespace Umbraco.Core
public DecomposedGuid(Guid value) : this() => this.Value = value;
}
private static readonly char[] Base32Table =
{
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p',
'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5'
};
/// <summary>
/// Converts a Guid into a base-32 string.
/// </summary>
/// <param name="guid">A Guid.</param>
/// <param name="length">The string length.</param>
/// <returns>A base-32 encoded string.</returns>
/// <remarks>
/// <para>A base-32 string representation of a Guid is the shortest, efficient, representation
/// that is case insensitive (base-64 is case sensitive).</para>
/// <para>Length must be 1-26, anything else becomes 26.</para>
/// </remarks>
public static string ToBase32String(Guid guid, int length = 26)
{
if (length <= 0 || length > 26)
length = 26;
var bytes = guid.ToByteArray(); // a Guid is 128 bits ie 16 bytes
// this could be optimized by making it unsafe,
// and fixing the table + bytes + chars (see Convert.ToBase64CharArray)
// each block of 5 bytes = 5*8 = 40 bits
// becomes 40 bits = 8*5 = 8 byte-32 chars
// a Guid is 3 blocks + 8 bits
// so it turns into a 3*8+2 = 26 chars string
var chars = new char[length];
var i = 0;
var j = 0;
while (i < 15)
{
if (j == length) break;
chars[j++] = Base32Table[(bytes[i] & 0b1111_1000) >> 3];
if (j == length) break;
chars[j++] = Base32Table[((bytes[i] & 0b0000_0111) << 2) | ((bytes[i + 1] & 0b1100_0000) >> 6)];
if (j == length) break;
chars[j++] = Base32Table[(bytes[i + 1] & 0b0011_1110) >> 1];
if (j == length) break;
chars[j++] = Base32Table[(bytes[i + 1] & 0b0000_0001) | ((bytes[i + 2] & 0b1111_0000) >> 4)];
if (j == length) break;
chars[j++] = Base32Table[((bytes[i + 2] & 0b0000_1111) << 1) | ((bytes[i + 3] & 0b1000_0000) >> 7)];
if (j == length) break;
chars[j++] = Base32Table[(bytes[i + 3] & 0b0111_1100) >> 2];
if (j == length) break;
chars[j++] = Base32Table[((bytes[i + 3] & 0b0000_0011) << 3) | ((bytes[i + 4] & 0b1110_0000) >> 5)];
if (j == length) break;
chars[j++] = Base32Table[bytes[i + 4] & 0b0001_1111];
i += 5;
}
if (j < length)
chars[j++] = Base32Table[(bytes[i] & 0b1111_1000) >> 3];
if (j < length)
chars[j] = Base32Table[(bytes[i] & 0b0000_0111) << 2];
return new string(chars);
}
}
}

9
src/Umbraco.Core/IO/MediaPathSchemes/CombinedGuidsMediaPathScheme.cs
View File

@@ -8,16 +8,21 @@ namespace Umbraco.Core.IO.MediaPathSchemes
/// </summary>
/// <remarks>
/// <para>Path is "{combinedGuid}/{filename}" where combinedGuid is a combination of itemGuid and propertyGuid.</para>
/// <para>This scheme is dangerous, as it does not prevent potential collisions.</para>
/// </remarks>
public class CombinedGuidsMediaPathScheme : IMediaPathScheme
{
private const int DirectoryLength = 8;
/// <inheritdoc />
public string GetFilePath(IMediaFileSystem fileSystem, Guid itemGuid, Guid propertyGuid, string filename, string previous = null)
{
// assumes that cuid and puid keys can be trusted - and that a single property type
// for a single content cannot store two different files with the same name
var directory = HexEncoder.Encode(GuidUtils.Combine(itemGuid, propertyGuid).ToByteArray()/*'/', 2, 4*/); // could use ext to fragment path eg 12/e4/f2/...
return Path.Combine(directory, filename).Replace('\\', '/').Substring(0, 9);
var combinedGuid = GuidUtils.Combine(itemGuid, propertyGuid);
var directory = GuidUtils.ToBase32String(combinedGuid, DirectoryLength); // see also HexEncoder, we may want to fragment path eg 12/e4/f3...
return Path.Combine(directory, filename).Replace('\\', '/');
}
/// <inheritdoc />

38
src/Umbraco.Core/IO/MediaPathSchemes/UniqueMediaPathScheme.cs Normal file
View File

@@ -0,0 +1,38 @@
using System;
using System.IO;
namespace Umbraco.Core.IO.MediaPathSchemes
{
/// <summary>
/// Implements a unique directory media path scheme.
/// </summary>
/// <remarks>
/// <para>This scheme provides short paths, yet handle potential collisions.</para>
/// </remarks>
public class UniqueMediaPathScheme : IMediaPathScheme
{
private const int DirectoryLength = 8;
/// <inheritdoc />
public string GetFilePath(IMediaFileSystem fileSystem, Guid itemGuid, Guid propertyGuid, string filename, string previous = null)
{
string directory;
// no point "combining" guids if all we want is some random guid - just get a new one
// and then, because we don't want collisions, ensure that the directory does not already exist
// (should be quite rare, but eh...)
do
{
var combinedGuid = Guid.NewGuid();
directory = GuidUtils.ToBase32String(combinedGuid, DirectoryLength); // see also HexEncoder, we may want to fragment path eg 12/e4/f3...
} while (fileSystem.DirectoryExists(directory));
return Path.Combine(directory, filename).Replace('\\', '/');
}
/// <inheritdoc />
public string GetDeleteDirectory(IMediaFileSystem fileSystem, string filepath) => Path.GetDirectoryName(filepath);
}
}

4
src/Umbraco.Core/IO/ShadowWrapper.cs
View File

@@ -25,7 +25,7 @@ namespace Umbraco.Core.IO
public static string CreateShadowId()
{
const int retries = 50; // avoid infinite loop
const int idLength = 6; // 6 chars
const int idLength = 8; // 6 chars
// shorten a Guid to idLength chars, and see whether it collides
// with an existing directory or not - if it does, try again, and
@@ -34,7 +34,7 @@ namespace Umbraco.Core.IO
for (var i = 0; i < retries; i++)
{
var id = Guid.NewGuid().ToString("N").Substring(0, idLength);
var id = GuidUtils.ToBase32String(Guid.NewGuid(), idLength);
var virt = ShadowFsPath + "/" + id;
var shadowDir = IOHelper.MapPath(virt);

109
src/Umbraco.Core/StringExtensions.cs
View File

@@ -593,13 +593,14 @@ namespace Umbraco.Core
///<returns></returns>
public static string ToUrlBase64(this string input)
{
if (input == null) throw new ArgumentNullException("input");
if (input == null) throw new ArgumentNullException(nameof(input));
if (String.IsNullOrEmpty(input)) return String.Empty;
if (string.IsNullOrEmpty(input))
return string.Empty;
//return Convert.ToBase64String(bytes).Replace(".", "-").Replace("/", "_").Replace("=", ",");
var bytes = Encoding.UTF8.GetBytes(input);
return UrlTokenEncode(bytes);
//return Convert.ToBase64String(bytes).Replace(".", "-").Replace("/", "_").Replace("=", ",");
}
/// <summary>
@@ -609,14 +610,14 @@ namespace Umbraco.Core
/// <returns></returns>
public static string FromUrlBase64(this string input)
{
if (input == null) throw new ArgumentNullException("input");
if (input == null) throw new ArgumentNullException(nameof(input));
//if (input.IsInvalidBase64()) return null;
try
{
//var decodedBytes = Convert.FromBase64String(input.Replace("-", ".").Replace("_", "/").Replace(",", "="));
byte[] decodedBytes = UrlTokenDecode(input);
var decodedBytes = UrlTokenDecode(input);
return decodedBytes != null ? Encoding.UTF8.GetString(decodedBytes) : null;
}
catch (FormatException)
@@ -795,42 +796,40 @@ namespace Umbraco.Core
internal static byte[] UrlTokenDecode(string input)
{
if (input == null)
throw new ArgumentNullException(nameof(input));
if (input.Length == 0)
return Array.Empty<byte>();
// calc array size - must be groups of 4
var arrayLength = input.Length;
var remain = arrayLength % 4;
if (remain != 0) arrayLength += 4 - remain;
var inArray = new char[arrayLength];
for (var i = 0; i < input.Length; i++)
{
throw new ArgumentNullException("input");
}
int length = input.Length;
if (length < 1)
{
return new byte[0];
}
int num2 = input[length - 1] - '0';
if ((num2 < 0) || (num2 > 10))
{
return null;
}
char[] inArray = new char[(length - 1) + num2];
for (int i = 0; i < (length - 1); i++)
{
char ch = input[i];
var ch = input[i];
switch (ch)
{
case '-':
case '-': // restore '-' as '+'
inArray[i] = '+';
break;
case '_':
case '_': // restore '_' as '/'
inArray[i] = '/';
break;
default:
default: // keep char unchanged
inArray[i] = ch;
break;
}
}
for (int j = length - 1; j < inArray.Length; j++)
{
// pad with '='
for (var j = input.Length; j < inArray.Length; j++)
inArray[j] = '=';
}
return Convert.FromBase64CharArray(inArray, 0, inArray.Length);
}
@@ -842,54 +841,40 @@ namespace Umbraco.Core
internal static string UrlTokenEncode(byte[] input)
{
if (input == null)
throw new ArgumentNullException(nameof(input));
if (input.Length == 0)
return string.Empty;
// base-64 digits are A-Z, a-z, 0-9, + and /
// the = char is used for trailing padding
var str = Convert.ToBase64String(input);
var pos = str.IndexOf('=');
if (pos < 0) pos = str.Length;
// replace chars that would cause problems in urls
var chArray = new char[pos];
for (var i = 0; i < pos; i++)
{
throw new ArgumentNullException("input");
}
if (input.Length < 1)
{
return String.Empty;
}
string str = null;
int index = 0;
char[] chArray = null;
str = Convert.ToBase64String(input);
if (str == null)
{
return null;
}
index = str.Length;
while (index > 0)
{
if (str[index - 1] != '=')
{
break;
}
index--;
}
chArray = new char[index + 1];
chArray[index] = (char)((0x30 + str.Length) - index);
for (int i = 0; i < index; i++)
{
char ch = str[i];
var ch = str[i];
switch (ch)
{
case '+':
case '+': // replace '+' with '-'
chArray[i] = '-';
break;
case '/':
case '/': // replace '/' with '_'
chArray[i] = '_';
break;
case '=':
chArray[i] = ch;
break;
default:
default: // keep char unchanged
chArray[i] = ch;
break;
}
}
return new string(chArray);
}

1
src/Umbraco.Core/Umbraco.Core.csproj
View File

@@ -207,6 +207,7 @@
<Compile Include="Composing\TypeFinder.cs" />
<Compile Include="Composing\TypeHelper.cs" />
<Compile Include="Composing\TypeLoader.cs" />
<Compile Include="IO\MediaPathSchemes\UniqueMediaPathScheme.cs" />
<Compile Include="Migrations\Upgrade\V_8_0_0\MergeDateAndDateTimePropertyEditor.cs" />
<Compile Include="PropertyEditors\DateTimeConfiguration.cs" />
<Compile Include="Migrations\Upgrade\V_8_0_0\RenameLabelAndRichTextPropertyEditorAliases.cs" />

8
src/Umbraco.Tests/CoreThings/GuidUtilsTests.cs
View File

@@ -15,6 +15,14 @@ namespace Umbraco.Tests.CoreThings
Assert.AreEqual(GuidUtils.Combine(a, b).ToByteArray(), Combine(a, b));
}
[Test]
public void GuidThingTest()
{
var guid = new Guid("f918382f-2bba-453f-a3e2-1f594016ed3b");
Assert.AreEqual("f22br4n0fm5fli5c", GuidUtils.ToBase32String(guid, 16));
Assert.AreEqual("f22br4n0f", GuidUtils.ToBase32String(guid, 9));
}
// Reference implementation taken from original code.
private static byte[] Combine(Guid guid1, Guid guid2)
{

2
src/Umbraco.Tests/IO/FileSystemsTests.cs
View File

@@ -33,7 +33,7 @@ namespace Umbraco.Tests.IO
composition.Register(_ => Mock.Of<ILogger>());
composition.Register(_ => Mock.Of<IDataTypeService>());
composition.Register(_ => Mock.Of<IContentSection>());
composition.RegisterUnique<IMediaPathScheme, OriginalMediaPathScheme>();
composition.RegisterUnique<IMediaPathScheme, UniqueMediaPathScheme>();
composition.Configs.Add(SettingsForTests.GetDefaultGlobalSettings);
composition.Configs.Add(SettingsForTests.GetDefaultUmbracoSettings);

15
src/Umbraco.Tests/Strings/StringExtensionsTests.cs
View File

@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Composing;
@@ -202,6 +203,20 @@ namespace Umbraco.Tests.Strings
Assert.AreEqual(expected, result);
}
[TestCase("hello", "aGVsbG8")]
[TestCase("tad", "dGFk")]
[TestCase("AmqGr+Fd!~ééé", "QW1xR3IrRmQhfsOpw6nDqQ")]
public void UrlTokenEncoding(string value, string expected)
{
var bytes = Encoding.UTF8.GetBytes(value);
Console.WriteLine("base64: " + Convert.ToBase64String(bytes));
var encoded = StringExtensions.UrlTokenEncode(bytes);
Assert.AreEqual(expected, encoded);
var backBytes = StringExtensions.UrlTokenDecode(encoded);
var backString = Encoding.UTF8.GetString(backBytes);
Assert.AreEqual(value, backString);
}
// FORMAT STRINGS

2
src/Umbraco.Tests/Testing/UmbracoTestBase.cs
View File

@@ -243,7 +243,7 @@ namespace Umbraco.Tests.Testing
Composition.WithCollectionBuilder<PropertyValueConverterCollectionBuilder>();
Composition.RegisterUnique<IPublishedContentTypeFactory, PublishedContentTypeFactory>();
Composition.RegisterUnique<IMediaPathScheme, OriginalMediaPathScheme>();
Composition.RegisterUnique<IMediaPathScheme, UniqueMediaPathScheme>();
// register empty content apps collection
Composition.WithCollectionBuilder<ContentAppFactoryCollectionBuilder>();