From 578e1317a08b68eb0f9df0945fe671ad5de4dd19 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Wed, 1 Jul 2020 17:19:56 +1200 Subject: [PATCH 001/147] Introduce IContentNestedDataSerializer to allow injecting a custom serializer for nucache --- .../PublishedContent/NuCacheChildrenTests.cs | 5 +- .../PublishedContent/NuCacheTests.cs | 5 +- .../Scoping/ScopedNuCacheTests.cs | 6 +- .../ContentTypeServiceVariantsTests.cs | 7 +- .../NuCache/DataSource/ContentNestedData.cs | 2 +- .../NuCache/DataSource/CultureVariation.cs | 2 +- .../NuCache/DataSource/DatabaseDataSource.cs | 31 ++-- .../IContentNestedDataSerializer.cs | 14 ++ .../JsonContentNestedDataSerializer.cs | 31 ++++ ...opertiesJsonContentNestedDataSerializer.cs | 157 ++++++++++++++++++ .../NuCache/DataSource/PropertyData.cs | 2 +- .../PublishedCache/NuCache/NuCacheComposer.cs | 3 + .../NuCache/PublishedSnapshotService.cs | 6 +- src/Umbraco.Web/Umbraco.Web.csproj | 2 + 14 files changed, 245 insertions(+), 28 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index 834d211994..fef096498c 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -37,6 +37,7 @@ namespace Umbraco.Tests.PublishedContent private ContentType _contentTypeInvariant; private ContentType _contentTypeVariant; private TestDataSource _source; + private IContentNestedDataSerializer _contentNestedDataSerializer; [TearDown] public void Teardown() @@ -134,6 +135,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache _source = new TestDataSource(kits()); + _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); // at last, create the complete NuCache snapshot service! var options = new PublishedSnapshotServiceOptions { IgnoreLocalDb = true }; @@ -155,7 +157,8 @@ namespace Umbraco.Tests.PublishedContent globalSettings, Mock.Of(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + _contentNestedDataSerializer); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 0e05e6baad..792ccc8529 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -33,6 +33,7 @@ namespace Umbraco.Tests.PublishedContent { private IPublishedSnapshotService _snapshotService; private IVariationContextAccessor _variationAccesor; + private IContentNestedDataSerializer _contentNestedDataSerializer; private ContentType _contentType; private PropertyType _propertyType; @@ -114,6 +115,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache var dataSource = new TestDataSource(kit); + _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); var runtime = Mock.Of(); Mock.Get(runtime).Setup(x => x.Level).Returns(RuntimeLevel.Run); @@ -201,7 +203,8 @@ namespace Umbraco.Tests.PublishedContent globalSettings, Mock.Of(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + _contentNestedDataSerializer); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index c7c403b260..5f72947382 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -82,6 +82,7 @@ namespace Umbraco.Tests.Scoping var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); + var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); return new PublishedSnapshotService( options, null, @@ -95,11 +96,12 @@ namespace Umbraco.Tests.Scoping ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(), + new DatabaseDataSource(nestedContentDataSerializer), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + nestedContentDataSerializer); } protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable urlProviders = null) diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index 9391b7442f..938b14c3a9 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -53,6 +53,8 @@ namespace Umbraco.Tests.Services var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); + var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); + return new PublishedSnapshotService( options, null, @@ -66,11 +68,12 @@ namespace Umbraco.Tests.Services ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(), + new DatabaseDataSource(nestedContentDataSerializer), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), - new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() })); + new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + nestedContentDataSerializer); } public class LocalServerMessenger : ServerMessengerBase diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs index ec5424ad9a..5f3edc4aa9 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs @@ -7,7 +7,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// The content item 1:M data that is serialized to JSON /// - internal class ContentNestedData + public class ContentNestedData { //dont serialize empty properties [JsonProperty("pd")] diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs index 57ffbba34e..b59e8c403c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs @@ -6,7 +6,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Represents the culture variation information on a content item /// - internal class CultureVariation + public class CultureVariation { [JsonProperty("nm")] public string Name { get; set; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 694dac04df..80cfabd470 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -20,7 +20,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // provides efficient database access for NuCache internal class DatabaseDataSource : IDataSource { + + private const int PageSize = 500; + private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + + internal DatabaseDataSource(IContentNestedDataSerializer contentNestedDataSerializer) + { + _contentNestedDataSerializer = contentNestedDataSerializer; + } // we want arrays, we want them all loaded, not an enumerable @@ -198,7 +206,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource yield return CreateMediaNodeKit(row); } - private static ContentNodeKit CreateContentNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto) { ContentData d = null; ContentData p = null; @@ -213,7 +221,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = DeserializeNestedData(dto.EditData); + var nested = _contentNestedDataSerializer.Deserialize(dto.EditData); d = new ContentData { @@ -240,7 +248,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = DeserializeNestedData(dto.PubData); + var nested = _contentNestedDataSerializer.Deserialize(dto.PubData); p = new ContentData { @@ -271,12 +279,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return s; } - private static ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto) { if (dto.EditData == null) throw new Exception("No data for media " + dto.Id); - var nested = DeserializeNestedData(dto.EditData); + var nested = _contentNestedDataSerializer.Deserialize(dto.EditData); var p = new ContentData { @@ -303,17 +311,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return s; } - private static ContentNestedData DeserializeNestedData(string data) - { - // by default JsonConvert will deserialize our numeric values as Int64 - // which is bad, because they were Int32 in the database - take care - - var settings = new JsonSerializerSettings - { - Converters = new List { new ForceInt32Converter() } - }; - - return JsonConvert.DeserializeObject(data, settings); - } + } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs new file mode 100644 index 0000000000..d9e2702d08 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public interface IContentNestedDataSerializer + { + ContentNestedData Deserialize(string data); + string Serialize(ContentNestedData nestedData); + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs new file mode 100644 index 0000000000..4ef63c09fb --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -0,0 +1,31 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Serialization; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer + { + public ContentNestedData Deserialize(string data) + { + // by default JsonConvert will deserialize our numeric values as Int64 + // which is bad, because they were Int32 in the database - take care + + var settings = new JsonSerializerSettings + { + Converters = new List { new ForceInt32Converter() } + }; + + return JsonConvert.DeserializeObject(data, settings); + } + + public string Serialize(ContentNestedData nestedData) + { + return JsonConvert.SerializeObject(nestedData); + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs new file mode 100644 index 0000000000..4919404e11 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs @@ -0,0 +1,157 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Serialization; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class MappedPropertiesJsonContentNestedDataSerializer : IContentNestedDataSerializer + { + private readonly IDictionary _serializeMap; + private readonly IDictionary _deserializeMap; + + + /// + /// Constructor + /// + /// Map for PropertData properties + /// + public MappedPropertiesJsonContentNestedDataSerializer(IDictionary serializeMap, IDictionary deserializeMap) + { + _serializeMap = serializeMap; + _deserializeMap = deserializeMap; + } + + public (string mappedName, bool isCompressed) ToSerializedProperty(string name) + { + if(_serializeMap.TryGetValue(name,out PropertyMap map)) + { + return (map.To,map.IsCompressed); + } + return (name,false); + } + public (string mappedName, bool isCompressed) ToDeserializedProperty(string name) + { + if (_deserializeMap.TryGetValue(name, out PropertyMap map)) + { + return (map.To, map.IsCompressed); + } + return (name, false); + } + + public ContentNestedData Deserialize(string data) + { + // by default JsonConvert will deserialize our numeric values as Int64 + // which is bad, because they were Int32 in the database - take care + + var settings = new JsonSerializerSettings + { + Converters = new List { new ForceInt32Converter() }, + + }; + + return JsonConvert.DeserializeObject(data, settings); + } + + public string Serialize(ContentNestedData nestedData) + { + return JsonConvert.SerializeObject(nestedData); + } + } + + public class MappedPropertyDataContractResolver : DefaultContractResolver + { + private readonly IDictionary _serializeMap; + private readonly IDictionary _deserializeMap; + + + /// + /// Constructor + /// + /// Map for PropertData properties + /// + public MappedPropertyDataContractResolver(IDictionary serializeMap, IDictionary deserializeMap) + { + _serializeMap = serializeMap; + _deserializeMap = deserializeMap; + } + + public (string mappedName, bool isCompressed) ToSerializedProperty(string name) + { + if (_serializeMap.TryGetValue(name, out PropertyMap map)) + { + return (map.To, map.IsCompressed); + } + return (name, false); + } + public (string mappedName, bool isCompressed) ToDeserializedProperty(string name) + { + if (_deserializeMap.TryGetValue(name, out PropertyMap map)) + { + return (map.To, map.IsCompressed); + } + return (name, false); + } + + private readonly Type _type = typeof(PropertyData); + protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) + { + var property = base.CreateProperty(member, memberSerialization); + if (property.DeclaringType == _type) + { + if (property.PropertyName.Equals("LongPropertyName", StringComparison.OrdinalIgnoreCase)) + { + property.PropertyName = "Short"; + } + } + return property; + } + protected override string ResolvePropertyName(string propertyName) + { + return base.ResolvePropertyName(propertyName); + } + protected override JsonDictionaryContract CreateDictionaryContract(Type objectType) + { + JsonDictionaryContract contract = base.CreateDictionaryContract(objectType); + + contract.DictionaryKeyResolver = propertyName => propertyName; + + return contract; + } + } + + public class MappedNamingStrategy : NamingStrategy + { + public MappedNamingStrategy() + { + ProcessDictionaryKeys = true; + } + public override string GetDictionaryKey(string key) + { + return key; + } + + protected override string ResolvePropertyName(string name) + { + return name; + } + } + + + public class PropertyMap + { + /// + /// PropertyName + /// + public string To { get; set; } + /// + /// Whether the property is compressed + /// + public bool IsCompressed { get; set; } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs index 4abcbc7e6f..cf7ab95360 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs @@ -4,7 +4,7 @@ using Newtonsoft.Json; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class PropertyData + public class PropertyData { private string _culture; private string _segment; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index f748fd555c..f67256bb6b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -10,6 +10,9 @@ namespace Umbraco.Web.PublishedCache.NuCache { base.Compose(composition); + // register the NuCache NestedContentData serializer + composition.Register(); + // register the NuCache database data source composition.Register(); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index a39e26e2b1..b14deb7959 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -48,6 +48,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IPublishedModelFactory _publishedModelFactory; private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; + private readonly IContentNestedDataSerializer _contentNestedDataSerializer; // volatile because we read it with no lock private volatile bool _isReady; @@ -81,7 +82,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders) + UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -98,6 +99,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _defaultCultureAccessor = defaultCultureAccessor; _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; + _contentNestedDataSerializer = contentNestedDataSerializer; // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member @@ -1457,7 +1459,7 @@ namespace Umbraco.Web.PublishedCache.NuCache // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 - Data = JsonConvert.SerializeObject(nestedData) + Data = _contentNestedDataSerializer.Serialize(nestedData) }; //Core.Composing.Current.Logger.Debug(dto.Data); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index b1cbff0fef..01f81cfc83 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -245,6 +245,8 @@ + + From 7ad019c18a38265187bd1bb0ac9f745ce3f8d440 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Wed, 1 Jul 2020 17:49:34 +1200 Subject: [PATCH 002/147] remove example nested content serializer as it is not production ready. --- ...opertiesJsonContentNestedDataSerializer.cs | 157 ------------------ 1 file changed, 157 deletions(-) delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs deleted file mode 100644 index 4919404e11..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MappedPropertiesJsonContentNestedDataSerializer.cs +++ /dev/null @@ -1,157 +0,0 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Text; -using System.Threading.Tasks; -using Umbraco.Core.Serialization; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - public class MappedPropertiesJsonContentNestedDataSerializer : IContentNestedDataSerializer - { - private readonly IDictionary _serializeMap; - private readonly IDictionary _deserializeMap; - - - /// - /// Constructor - /// - /// Map for PropertData properties - /// - public MappedPropertiesJsonContentNestedDataSerializer(IDictionary serializeMap, IDictionary deserializeMap) - { - _serializeMap = serializeMap; - _deserializeMap = deserializeMap; - } - - public (string mappedName, bool isCompressed) ToSerializedProperty(string name) - { - if(_serializeMap.TryGetValue(name,out PropertyMap map)) - { - return (map.To,map.IsCompressed); - } - return (name,false); - } - public (string mappedName, bool isCompressed) ToDeserializedProperty(string name) - { - if (_deserializeMap.TryGetValue(name, out PropertyMap map)) - { - return (map.To, map.IsCompressed); - } - return (name, false); - } - - public ContentNestedData Deserialize(string data) - { - // by default JsonConvert will deserialize our numeric values as Int64 - // which is bad, because they were Int32 in the database - take care - - var settings = new JsonSerializerSettings - { - Converters = new List { new ForceInt32Converter() }, - - }; - - return JsonConvert.DeserializeObject(data, settings); - } - - public string Serialize(ContentNestedData nestedData) - { - return JsonConvert.SerializeObject(nestedData); - } - } - - public class MappedPropertyDataContractResolver : DefaultContractResolver - { - private readonly IDictionary _serializeMap; - private readonly IDictionary _deserializeMap; - - - /// - /// Constructor - /// - /// Map for PropertData properties - /// - public MappedPropertyDataContractResolver(IDictionary serializeMap, IDictionary deserializeMap) - { - _serializeMap = serializeMap; - _deserializeMap = deserializeMap; - } - - public (string mappedName, bool isCompressed) ToSerializedProperty(string name) - { - if (_serializeMap.TryGetValue(name, out PropertyMap map)) - { - return (map.To, map.IsCompressed); - } - return (name, false); - } - public (string mappedName, bool isCompressed) ToDeserializedProperty(string name) - { - if (_deserializeMap.TryGetValue(name, out PropertyMap map)) - { - return (map.To, map.IsCompressed); - } - return (name, false); - } - - private readonly Type _type = typeof(PropertyData); - protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) - { - var property = base.CreateProperty(member, memberSerialization); - if (property.DeclaringType == _type) - { - if (property.PropertyName.Equals("LongPropertyName", StringComparison.OrdinalIgnoreCase)) - { - property.PropertyName = "Short"; - } - } - return property; - } - protected override string ResolvePropertyName(string propertyName) - { - return base.ResolvePropertyName(propertyName); - } - protected override JsonDictionaryContract CreateDictionaryContract(Type objectType) - { - JsonDictionaryContract contract = base.CreateDictionaryContract(objectType); - - contract.DictionaryKeyResolver = propertyName => propertyName; - - return contract; - } - } - - public class MappedNamingStrategy : NamingStrategy - { - public MappedNamingStrategy() - { - ProcessDictionaryKeys = true; - } - public override string GetDictionaryKey(string key) - { - return key; - } - - protected override string ResolvePropertyName(string name) - { - return name; - } - } - - - public class PropertyMap - { - /// - /// PropertyName - /// - public string To { get; set; } - /// - /// Whether the property is compressed - /// - public bool IsCompressed { get; set; } - } -} From 3735b6d3919e2d778c53a4b44aa6a4ac173f7555 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 2 Jul 2020 16:21:54 +1000 Subject: [PATCH 003/147] Fixes paging size for querying the nucache table so it doesn't timeout --- .../NuCache/PublishedSnapshotService.cs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index a39e26e2b1..fcea6e6227 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Configuration; using System.Diagnostics; using System.Globalization; using System.IO; @@ -1469,6 +1470,14 @@ namespace Umbraco.Web.PublishedCache.NuCache #region Rebuild Database PreCache + private const int DefaultSqlPagingSize = 1000; + + private static int GetSqlPagingSize() + { + var appSetting = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.PublishedSnapshotService.SqlPageSize"]; + return appSetting != null && int.TryParse(appSetting, out var size) ? size : DefaultSqlPagingSize; + } + public override void Rebuild() { _logger.Debug("Rebuilding..."); @@ -1477,14 +1486,14 @@ namespace Umbraco.Web.PublishedCache.NuCache scope.ReadLock(Constants.Locks.ContentTree); scope.ReadLock(Constants.Locks.MediaTree); scope.ReadLock(Constants.Locks.MemberTree); - RebuildContentDbCacheLocked(scope, 5000, null); - RebuildMediaDbCacheLocked(scope, 5000, null); - RebuildMemberDbCacheLocked(scope, 5000, null); + RebuildContentDbCacheLocked(scope, GetSqlPagingSize(), null); + RebuildMediaDbCacheLocked(scope, GetSqlPagingSize(), null); + RebuildMemberDbCacheLocked(scope, GetSqlPagingSize(), null); scope.Complete(); } } - public void RebuildContentDbCache(int groupSize = 5000, IEnumerable contentTypeIds = null) + public void RebuildContentDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { @@ -1556,7 +1565,7 @@ WHERE cmsContentNu.nodeId IN ( } while (processed < total); } - public void RebuildMediaDbCache(int groupSize = 5000, IEnumerable contentTypeIds = null) + public void RebuildMediaDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { @@ -1615,7 +1624,7 @@ WHERE cmsContentNu.nodeId IN ( } while (processed < total); } - public void RebuildMemberDbCache(int groupSize = 5000, IEnumerable contentTypeIds = null) + public void RebuildMemberDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { From bb2fe5d2d6f41e06872bf2e806ed131afe9c52e7 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 2 Jul 2020 20:56:42 +1000 Subject: [PATCH 004/147] Adds notes about SQL call --- .../Repositories/Implement/ContentRepositoryBase.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs index 845006891d..d56724db9f 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs @@ -630,6 +630,13 @@ namespace Umbraco.Core.Persistence.Repositories.Implement } if (versions.Count == 0) return new Dictionary(); + // TODO: This is a bugger of a query and I believe is the main issue with regards to SQL performance drain when querying content + // which is done when rebuilding caches/indexes/etc... in bulk. We are using an "IN" query on umbracoPropertyData.VersionId + // which then performs a Clustered Index Scan on PK_umbracoPropertyData which means it iterates the entire table which can be enormous! + // especially if there are both a lot of content but worse if there is a lot of versions of that content. + // So is it possible to return this property data without doing an index scan on PK_umbracoPropertyData and without iterating every row + // in the table? + // get all PropertyDataDto for all definitions / versions var allPropertyDataDtos = Database.FetchByGroups(versions, 2000, batch => SqlContext.Sql() From d2042e28e13f427eb38d1b3f0b3eb9571db15252 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 2 Jul 2020 21:27:48 +1000 Subject: [PATCH 005/147] Adds string interning for reading of property alias, culture and segment when reading from the content cache --- .../DataSource/BTree.DictionaryOfPropertyDataSerializer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index aa5dc9eb30..15c6a9f1bd 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -19,7 +19,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource for (var i = 0; i < pcount; i++) { // read property alias - var key = PrimitiveSerializer.String.ReadFrom(stream); + var key = string.Intern(PrimitiveSerializer.String.ReadFrom(stream)); // read values count var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); @@ -38,8 +38,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // the 'current' value, and string.Empty should be used to represent the invariant or // neutral values - PropertyData throws when getting nulls, so falling back to // string.Empty here - what else? - pdata.Culture = ReadStringObject(stream) ?? string.Empty; - pdata.Segment = ReadStringObject(stream) ?? string.Empty; + pdata.Culture = string.Intern(ReadStringObject(stream)) ?? string.Empty; + pdata.Segment = string.Intern(ReadStringObject(stream)) ?? string.Empty; pdata.Value = ReadObject(stream); } From e75c9d2273232e7c4419172c0763cc18b483b494 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 3 Jul 2020 00:26:55 +1000 Subject: [PATCH 006/147] Interns strings for aliases, etc... for when content is deserialized from the contentNu table so we aren't duplicating strings when cold booting. --- .../AutoInterningStringConverter.cs | 38 +++++++++++ ...ngKeyCaseInsensitiveDictionaryConverter.cs | 54 +++++++++++++++ .../CaseInsensitiveDictionaryConverter.cs | 14 +++- src/Umbraco.Core/Umbraco.Core.csproj | 2 + .../AutoInterningStringConverterTests.cs | 67 +++++++++++++++++++ src/Umbraco.Tests/Umbraco.Tests.csproj | 1 + ....DictionaryOfCultureVariationSerializer.cs | 9 ++- ...Tree.DictionaryOfPropertyDataSerializer.cs | 4 +- .../NuCache/DataSource/ContentNestedData.cs | 4 +- .../NuCache/DataSource/PropertyData.cs | 4 +- .../NuCache/DataSource/SerializerBase.cs | 6 +- 11 files changed, 193 insertions(+), 10 deletions(-) create mode 100644 src/Umbraco.Core/Serialization/AutoInterningStringConverter.cs create mode 100644 src/Umbraco.Core/Serialization/AutoInterningStringKeyCaseInsensitiveDictionaryConverter.cs create mode 100644 src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs diff --git a/src/Umbraco.Core/Serialization/AutoInterningStringConverter.cs b/src/Umbraco.Core/Serialization/AutoInterningStringConverter.cs new file mode 100644 index 0000000000..2d1f12f5da --- /dev/null +++ b/src/Umbraco.Core/Serialization/AutoInterningStringConverter.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Umbraco.Core.Serialization +{ + + /// + /// When applied to a string or string collection field will ensure the deserialized strings are interned + /// + /// + /// Borrowed from https://stackoverflow.com/a/34906004/694494 + /// On the same page an interesting approach of using a local intern pool https://stackoverflow.com/a/39605620/694494 which re-uses .NET System.Xml.NameTable + /// + internal class AutoInterningStringConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + // CanConvert is not called when a converter is applied directly to a property. + throw new NotImplementedException($"{nameof(AutoInterningStringConverter)} should not be used globally"); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + return null; + // Check is in case the value is a non-string literal such as an integer. + var s = reader.TokenType == JsonToken.String + ? string.Intern((string)reader.Value) + : string.Intern((string)JToken.Load(reader)); + return s; + } + + public override bool CanWrite => false; + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) => throw new NotImplementedException(); + } +} diff --git a/src/Umbraco.Core/Serialization/AutoInterningStringKeyCaseInsensitiveDictionaryConverter.cs b/src/Umbraco.Core/Serialization/AutoInterningStringKeyCaseInsensitiveDictionaryConverter.cs new file mode 100644 index 0000000000..2076462f0c --- /dev/null +++ b/src/Umbraco.Core/Serialization/AutoInterningStringKeyCaseInsensitiveDictionaryConverter.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Umbraco.Core.Serialization +{ + /// + /// When applied to a dictionary with a string key, will ensure the deserialized string keys are interned + /// + /// + /// + /// borrowed from https://stackoverflow.com/a/36116462/694494 + /// + internal class AutoInterningStringKeyCaseInsensitiveDictionaryConverter : CaseInsensitiveDictionaryConverter + { + public AutoInterningStringKeyCaseInsensitiveDictionaryConverter() + { + } + public AutoInterningStringKeyCaseInsensitiveDictionaryConverter(StringComparer comparer) : base(comparer) + { + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.StartObject) + { + var dictionary = new Dictionary(); + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonToken.PropertyName: + var key = string.Intern(reader.Value.ToString()); + + if (!reader.Read()) + throw new Exception("Unexpected end when reading object."); + + var v = serializer.Deserialize(reader); + dictionary[key] = v; + break; + case JsonToken.Comment: + break; + case JsonToken.EndObject: + return dictionary; + } + } + } + return null; + } + + } +} diff --git a/src/Umbraco.Core/Serialization/CaseInsensitiveDictionaryConverter.cs b/src/Umbraco.Core/Serialization/CaseInsensitiveDictionaryConverter.cs index a92d562a52..d5cbc0da31 100644 --- a/src/Umbraco.Core/Serialization/CaseInsensitiveDictionaryConverter.cs +++ b/src/Umbraco.Core/Serialization/CaseInsensitiveDictionaryConverter.cs @@ -14,12 +14,24 @@ namespace Umbraco.Core.Serialization /// public class CaseInsensitiveDictionaryConverter : CustomCreationConverter { + private readonly StringComparer _comparer; + + public CaseInsensitiveDictionaryConverter() + : this(StringComparer.OrdinalIgnoreCase) + { + } + + public CaseInsensitiveDictionaryConverter(StringComparer comparer) + { + _comparer = comparer ?? throw new ArgumentNullException(nameof(comparer)); + } + public override bool CanWrite => false; public override bool CanRead => true; public override bool CanConvert(Type objectType) => typeof(IDictionary).IsAssignableFrom(objectType); - public override IDictionary Create(Type objectType) => new Dictionary(StringComparer.OrdinalIgnoreCase); + public override IDictionary Create(Type objectType) => new Dictionary(_comparer); } } diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index b2a8ea2a6d..c5a953631a 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -139,6 +139,8 @@ + + diff --git a/src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs b/src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs new file mode 100644 index 0000000000..bf99b68077 --- /dev/null +++ b/src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs @@ -0,0 +1,67 @@ +using Newtonsoft.Json; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Xml.Serialization; +using Umbraco.Core.Serialization; + +namespace Umbraco.Tests.Serialization +{ + [TestFixture] + public class AutoInterningStringConverterTests + { + [Test] + public void Intern_Property_String() + { + var str1 = "Hello"; + var obj = new Test + { + Name = str1 + " " + "there" + }; + + // ensure the raw value is not interned + Assert.IsNull(string.IsInterned(obj.Name)); + + var serialized = JsonConvert.SerializeObject(obj); + obj = JsonConvert.DeserializeObject(serialized); + + Assert.IsNotNull(string.IsInterned(obj.Name)); + } + + [Test] + public void Intern_Property_Dictionary() + { + var str1 = "key"; + var obj = new Test + { + Values = new Dictionary + { + [str1 + "1"] = 0, + [str1 + "2"] = 1 + } + }; + + // ensure the raw value is not interned + Assert.IsNull(string.IsInterned(obj.Values.Keys.First())); + Assert.IsNull(string.IsInterned(obj.Values.Keys.Last())); + + var serialized = JsonConvert.SerializeObject(obj); + obj = JsonConvert.DeserializeObject(serialized); + + Assert.IsNotNull(string.IsInterned(obj.Values.Keys.First())); + Assert.IsNotNull(string.IsInterned(obj.Values.Keys.Last())); + } + + public class Test + { + [JsonConverter(typeof(AutoInterningStringConverter))] + public string Name { get; set; } + + [JsonConverter(typeof(AutoInterningStringKeyCaseInsensitiveDictionaryConverter))] + public Dictionary Values = new Dictionary(); + } + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 731dc05363..d2ee8e83df 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -158,6 +158,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs index 4521311302..1fcbdbdae7 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs @@ -18,8 +18,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); for (var i = 0; i < pcount; i++) { - var languageId = PrimitiveSerializer.String.ReadFrom(stream); - var cultureVariation = new CultureVariation { Name = ReadStringObject(stream), UrlSegment = ReadStringObject(stream), Date = ReadDateTime(stream) }; + var languageId = string.Intern(PrimitiveSerializer.String.ReadFrom(stream)); + var cultureVariation = new CultureVariation + { + Name = ReadStringObject(stream), + UrlSegment = ReadStringObject(stream), + Date = ReadDateTime(stream) + }; dict[languageId] = cultureVariation; } return dict; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index 15c6a9f1bd..e3fa6597e4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -38,8 +38,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // the 'current' value, and string.Empty should be used to represent the invariant or // neutral values - PropertyData throws when getting nulls, so falling back to // string.Empty here - what else? - pdata.Culture = string.Intern(ReadStringObject(stream)) ?? string.Empty; - pdata.Segment = string.Intern(ReadStringObject(stream)) ?? string.Empty; + pdata.Culture = ReadStringObject(stream, true) ?? string.Empty; + pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; pdata.Value = ReadObject(stream); } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs index ec5424ad9a..751644c715 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs @@ -11,11 +11,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { //dont serialize empty properties [JsonProperty("pd")] - [JsonConverter(typeof(CaseInsensitiveDictionaryConverter))] + [JsonConverter(typeof(AutoInterningStringKeyCaseInsensitiveDictionaryConverter))] public Dictionary PropertyData { get; set; } [JsonProperty("cd")] - [JsonConverter(typeof(CaseInsensitiveDictionaryConverter))] + [JsonConverter(typeof(AutoInterningStringKeyCaseInsensitiveDictionaryConverter))] public Dictionary CultureData { get; set; } [JsonProperty("us")] diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs index 4abcbc7e6f..6ccb1dc210 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs @@ -1,6 +1,7 @@ using System; using System.ComponentModel; using Newtonsoft.Json; +using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { @@ -9,6 +10,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private string _culture; private string _segment; + [JsonConverter(typeof(AutoInterningStringConverter))] [DefaultValue("")] [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, PropertyName = "c")] public string Culture @@ -17,6 +19,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource set => _culture = value ?? throw new ArgumentNullException(nameof(value)); // TODO: or fallback to string.Empty? CANNOT be null } + [JsonConverter(typeof(AutoInterningStringConverter))] [DefaultValue("")] [JsonProperty(DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, PropertyName = "s")] public string Segment @@ -28,7 +31,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource [JsonProperty("v")] public object Value { get; set; } - //Legacy properties used to deserialize existing nucache db entries [JsonProperty("culture")] private string LegacyCulture diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index ed17420645..6196be9a3a 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -23,13 +23,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return read(stream); } - protected string ReadStringObject(Stream stream) // required 'cos string is not a struct + protected string ReadStringObject(Stream stream, bool intern = false) // required 'cos string is not a struct { var type = PrimitiveSerializer.Char.ReadFrom(stream); if (type == 'N') return null; if (type != 'S') throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'."); - return PrimitiveSerializer.String.ReadFrom(stream); + return intern + ? string.Intern(PrimitiveSerializer.String.ReadFrom(stream)) + : PrimitiveSerializer.String.ReadFrom(stream); } protected int? ReadIntObject(Stream stream) => ReadObject(stream, 'I', ReadInt); From c63bfb866bdf84600a607549f1f6a4b351c56db4 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 3 Jul 2020 12:11:05 +1000 Subject: [PATCH 007/147] Adds MessagePack serialization for nucache --- .../ContentSerializationTests.cs | 103 ++++++++++++++++++ src/Umbraco.Tests/Umbraco.Tests.csproj | 3 +- .../JsonContentNestedDataSerializer.cs | 14 ++- .../MsgPackContentNestedDataSerializer.cs | 42 +++++++ .../NuCache/DataSource/SerializerBase.cs | 71 ++++++++---- .../PublishedCache/NuCache/NuCacheComposer.cs | 3 +- .../NuCache/PublishedSnapshotService.cs | 4 - src/Umbraco.Web/Umbraco.Web.csproj | 6 +- 8 files changed, 215 insertions(+), 31 deletions(-) create mode 100644 src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs new file mode 100644 index 0000000000..debfddef98 --- /dev/null +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -0,0 +1,103 @@ +using NUnit.Framework; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Web.PublishedCache.NuCache.DataSource; + +namespace Umbraco.Tests.PublishedContent +{ + [TestFixture] + public class ContentSerializationTests + { + [Test] + public void Ensure_Same_Results() + { + var jsonSerializer = new JsonContentNestedDataSerializer(); + var msgPackSerializer = new MsgPackContentNestedDataSerializer(); + + var now = DateTime.Now; + var content = new ContentNestedData + { + PropertyData = new Dictionary + { + ["propertyOne"] = new[] + { + new PropertyData + { + Culture = "en-US", + Segment = "test", + Value = "hello world" + } + }, + ["propertyTwo"] = new[] + { + new PropertyData + { + Culture = "en-US", + Segment = "test", + Value = "Lorem ipsum" + } + } + }, + CultureData = new Dictionary + { + ["en-US"] = new CultureVariation + { + Date = now, + IsDraft = false, + Name = "Home", + UrlSegment = "home" + } + }, + UrlSegment = "home" + }; + + var json = jsonSerializer.Serialize(content); + var msgPack = msgPackSerializer.Serialize(content); + + Console.WriteLine(json); + Console.WriteLine(msgPackSerializer.ToJson(msgPack)); + + var jsonContent = jsonSerializer.Deserialize(json); + var msgPackContent = msgPackSerializer.Deserialize(msgPack); + + + CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); + CollectionAssert.AreEqual(jsonContent.PropertyData.Keys, msgPackContent.PropertyData.Keys); + CollectionAssert.AreEqual(jsonContent.CultureData.Values, msgPackContent.CultureData.Values, new CultureVariationComparer()); + CollectionAssert.AreEqual(jsonContent.PropertyData.Values, msgPackContent.PropertyData.Values, new PropertyDataComparer()); + Assert.AreEqual(jsonContent.UrlSegment, msgPackContent.UrlSegment); + } + + public class CultureVariationComparer : Comparer + { + public override int Compare(CultureVariation x, CultureVariation y) + { + if (x == null && y == null) return 0; + if (x == null && y != null) return -1; + if (x != null && y == null) return 1; + + return x.Date.CompareTo(y.Date) | x.IsDraft.CompareTo(y.IsDraft) | x.Name.CompareTo(y.Name) | x.UrlSegment.CompareTo(y.UrlSegment); + } + } + + public class PropertyDataComparer : Comparer + { + public override int Compare(PropertyData x, PropertyData y) + { + if (x == null && y == null) return 0; + if (x == null && y != null) return -1; + if (x != null && y == null) return 1; + + var xVal = x.Value?.ToString() ?? string.Empty; + var yVal = y.Value?.ToString() ?? string.Empty; + + return x.Culture.CompareTo(y.Culture) | x.Segment.CompareTo(y.Segment) | xVal.CompareTo(yVal); + } + } + + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index d2ee8e83df..c65110ab6b 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -109,7 +109,7 @@ - + @@ -149,6 +149,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 4ef63c09fb..6635fa7090 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,13 +1,14 @@ using Newtonsoft.Json; -using System; using System.Collections.Generic; using System.Linq; +using System.Reflection.Emit; using System.Text; using System.Threading.Tasks; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer { public ContentNestedData Deserialize(string data) @@ -17,7 +18,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var settings = new JsonSerializerSettings { - Converters = new List { new ForceInt32Converter() } + Converters = new List { new ForceInt32Converter() }, + + // Explicitly specify date handling so that it's consistent and follows the same date handling as MessagePack + DateParseHandling = DateParseHandling.DateTime, + DateFormatHandling = DateFormatHandling.IsoDateFormat, + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + DateFormatString = "o" }; return JsonConvert.DeserializeObject(data, settings); @@ -25,6 +32,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { + // note that numeric values (which are Int32) are serialized without their + // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 + return JsonConvert.SerializeObject(nestedData); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs new file mode 100644 index 0000000000..8ff49a9544 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -0,0 +1,42 @@ +using MessagePack; +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class MsgPackContentNestedDataSerializer : IContentNestedDataSerializer + { + private MessagePackSerializerOptions _options; + + public MsgPackContentNestedDataSerializer() + { + _options = MessagePack.Resolvers.ContractlessStandardResolver.Options.WithCompression(MessagePackCompression.Lz4BlockArray); + } + + public string ToJson(string serialized) + { + var bin = Convert.FromBase64String(serialized); + var json = MessagePackSerializer.ConvertToJson(bin, _options); + return json; + } + + // TODO: Instead of returning base64 it would be more ideal to avoid that translation entirely and just store/retrieve raw bytes + + // TODO: We need to write tests to serialize/deserialize between either of these serializers to ensure we end up with the same object + // i think this one is a bit quirky so far :) + + public ContentNestedData Deserialize(string data) + { + var bin = Convert.FromBase64String(data); + var obj = MessagePackSerializer.Deserialize(bin, _options); + return obj; + } + + public string Serialize(ContentNestedData nestedData) + { + var bin = MessagePackSerializer.Serialize( + nestedData, + _options); + return Convert.ToBase64String(bin); + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 6196be9a3a..0c89419b51 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -6,6 +6,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal abstract class SerializerBase { + private const char PrefixNull = 'N'; + private const char PrefixString = 'S'; + private const char PrefixInt32 = 'I'; + private const char PrefixUInt16 = 'H'; + private const char PrefixLong = 'L'; + private const char PrefixFloat = 'F'; + private const char PrefixDouble = 'B'; + private const char PrefixDateTime = 'D'; + private const char PrefixByte = 'O'; + protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); protected long ReadLong(Stream stream) => PrimitiveSerializer.Int64.ReadFrom(stream); @@ -17,7 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource where T : struct { var type = PrimitiveSerializer.Char.ReadFrom(stream); - if (type == 'N') return null; + if (type == PrefixNull) return null; if (type != t) throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{t}'."); return read(stream); @@ -26,40 +36,47 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected string ReadStringObject(Stream stream, bool intern = false) // required 'cos string is not a struct { var type = PrimitiveSerializer.Char.ReadFrom(stream); - if (type == 'N') return null; - if (type != 'S') + if (type == PrefixNull) return null; + if (type != PrefixString) throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'."); return intern ? string.Intern(PrimitiveSerializer.String.ReadFrom(stream)) : PrimitiveSerializer.String.ReadFrom(stream); } - protected int? ReadIntObject(Stream stream) => ReadObject(stream, 'I', ReadInt); - protected long? ReadLongObject(Stream stream) => ReadObject(stream, 'L', ReadLong); - protected float? ReadFloatObject(Stream stream) => ReadObject(stream, 'F', ReadFloat); - protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, 'B', ReadDouble); - protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, 'D', ReadDateTime); + protected int? ReadIntObject(Stream stream) => ReadObject(stream, PrefixInt32, ReadInt); + protected long? ReadLongObject(Stream stream) => ReadObject(stream, PrefixLong, ReadLong); + protected float? ReadFloatObject(Stream stream) => ReadObject(stream, PrefixFloat, ReadFloat); + protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble); + protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime); protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); protected object ReadObject(char type, Stream stream) { + // NOTE: There is going to be a ton of boxing going on here, but i'm not sure we can avoid that because innevitably with our + // current model structure the value will need to end up being 'object' at some point anyways. + switch (type) { - case 'N': + case PrefixNull: return null; - case 'S': + case PrefixString: return PrimitiveSerializer.String.ReadFrom(stream); - case 'I': + case PrefixInt32: return PrimitiveSerializer.Int32.ReadFrom(stream); - case 'L': + case PrefixUInt16: + return PrimitiveSerializer.UInt16.ReadFrom(stream); + case PrefixByte: + return PrimitiveSerializer.Byte.ReadFrom(stream); + case PrefixLong: return PrimitiveSerializer.Int64.ReadFrom(stream); - case 'F': + case PrefixFloat: return PrimitiveSerializer.Float.ReadFrom(stream); - case 'B': + case PrefixDouble: return PrimitiveSerializer.Double.ReadFrom(stream); - case 'D': + case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); @@ -70,36 +87,46 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { if (value == null) { - PrimitiveSerializer.Char.WriteTo('N', stream); + PrimitiveSerializer.Char.WriteTo(PrefixNull, stream); } else if (value is string stringValue) { - PrimitiveSerializer.Char.WriteTo('S', stream); + PrimitiveSerializer.Char.WriteTo(PrefixString, stream); PrimitiveSerializer.String.WriteTo(stringValue, stream); } else if (value is int intValue) { - PrimitiveSerializer.Char.WriteTo('I', stream); + PrimitiveSerializer.Char.WriteTo(PrefixInt32, stream); PrimitiveSerializer.Int32.WriteTo(intValue, stream); } + else if (value is byte byteValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixByte, stream); + PrimitiveSerializer.Byte.WriteTo(byteValue, stream); + } + else if (value is ushort ushortValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixUInt16, stream); + PrimitiveSerializer.UInt16.WriteTo(ushortValue, stream); + } else if (value is long longValue) { - PrimitiveSerializer.Char.WriteTo('L', stream); + PrimitiveSerializer.Char.WriteTo(PrefixLong, stream); PrimitiveSerializer.Int64.WriteTo(longValue, stream); } else if (value is float floatValue) { - PrimitiveSerializer.Char.WriteTo('F', stream); + PrimitiveSerializer.Char.WriteTo(PrefixFloat, stream); PrimitiveSerializer.Float.WriteTo(floatValue, stream); } else if (value is double doubleValue) { - PrimitiveSerializer.Char.WriteTo('B', stream); + PrimitiveSerializer.Char.WriteTo(PrefixDouble, stream); PrimitiveSerializer.Double.WriteTo(doubleValue, stream); } else if (value is DateTime dateValue) { - PrimitiveSerializer.Char.WriteTo('D', stream); + PrimitiveSerializer.Char.WriteTo(PrefixDateTime, stream); PrimitiveSerializer.DateTime.WriteTo(dateValue, stream); } else diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index f67256bb6b..6d01b34a76 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -11,7 +11,8 @@ namespace Umbraco.Web.PublishedCache.NuCache base.Compose(composition); // register the NuCache NestedContentData serializer - composition.Register(); + //composition.Register(); + composition.Register(); // register the NuCache database data source composition.Register(); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 5ce62271a7..6289667d3b 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1456,10 +1456,6 @@ namespace Umbraco.Web.PublishedCache.NuCache { NodeId = content.Id, Published = published, - - // note that numeric values (which are Int32) are serialized without their - // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 - Data = _contentNestedDataSerializer.Serialize(nestedData) }; diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 01f81cfc83..088fb0eeb3 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -73,6 +73,9 @@ + + 2.1.152 + @@ -247,6 +250,7 @@ + @@ -1266,7 +1270,7 @@ - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - + + + - - + + @@ -94,7 +94,11 @@ - - + + + + + + diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 5518fb5678..cd0868532c 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -78,7 +78,7 @@ - + 1.8.14 @@ -100,7 +100,9 @@ - + + 4.14.5 + @@ -109,7 +111,6 @@ - diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs index 0ff24980fe..1a49aaaf62 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs @@ -29,17 +29,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string UrlSegment { get; set; } //Legacy properties used to deserialize existing nucache db entries - [DataMember(Order = 3)] + [IgnoreDataMember] [JsonProperty("properties")] [JsonConverter(typeof(CaseInsensitiveDictionaryConverter))] private Dictionary LegacyPropertyData { set { PropertyData = value; } } - [DataMember(Order = 4)] + [IgnoreDataMember] [JsonProperty("cultureData")] [JsonConverter(typeof(CaseInsensitiveDictionaryConverter))] private Dictionary LegacyCultureData { set { CultureData = value; } } - [DataMember(Order = 5)] + [IgnoreDataMember] [JsonProperty("urlSegment")] private string LegacyUrlSegment { set { UrlSegment = value; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs index dd3323fa0c..daf60b66ec 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/CultureVariation.cs @@ -27,19 +27,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public bool IsDraft { get; set; } //Legacy properties used to deserialize existing nucache db entries - [DataMember(Order = 4)] + [IgnoreDataMember] [JsonProperty("name")] private string LegacyName { set { Name = value; } } - [DataMember(Order = 5)] + [IgnoreDataMember] [JsonProperty("urlSegment")] private string LegacyUrlSegment { set { UrlSegment = value; } } - [DataMember(Order = 6)] + [IgnoreDataMember] [JsonProperty("date")] private DateTime LegacyDate { set { Date = value; } } - [DataMember(Order = 7)] + [IgnoreDataMember] [JsonProperty("isDraft")] private bool LegacyIsDraft { set { IsDraft = value; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs index b49a781e0c..7a3cb67d50 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/PropertyData.cs @@ -38,21 +38,21 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public object Value { get; set; } //Legacy properties used to deserialize existing nucache db entries - [DataMember(Order = 3)] + [IgnoreDataMember] [JsonProperty("culture")] private string LegacyCulture { set => Culture = value; } - [DataMember(Order = 4)] + [IgnoreDataMember] [JsonProperty("seg")] private string LegacySegment { set => Segment = value; } - [DataMember(Order = 5)] + [IgnoreDataMember] [JsonProperty("val")] private object LegacyValue { From 40184c0c3c6723c3083d835c9e414aa6eec1ce64 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 3 Jul 2020 16:56:08 +1200 Subject: [PATCH 010/147] support uInt32 --- .../PublishedCache/NuCache/DataSource/SerializerBase.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 0c89419b51..afc0827ed3 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -10,6 +10,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixString = 'S'; private const char PrefixInt32 = 'I'; private const char PrefixUInt16 = 'H'; + private const char PrefixUInt32 = 'J'; private const char PrefixLong = 'L'; private const char PrefixFloat = 'F'; private const char PrefixDouble = 'B'; @@ -68,6 +69,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return PrimitiveSerializer.Int32.ReadFrom(stream); case PrefixUInt16: return PrimitiveSerializer.UInt16.ReadFrom(stream); + case PrefixUInt32: + return PrimitiveSerializer.UInt32.ReadFrom(stream); case PrefixByte: return PrimitiveSerializer.Byte.ReadFrom(stream); case PrefixLong: @@ -129,6 +132,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Char.WriteTo(PrefixDateTime, stream); PrimitiveSerializer.DateTime.WriteTo(dateValue, stream); } + else if (value is uint uInt32Value) + { + PrimitiveSerializer.Char.WriteTo(PrefixUInt32, stream); + PrimitiveSerializer.UInt32.WriteTo(uInt32Value, stream); + } else throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized."); } From 39625d94ddeb6fab6e5a5ce64aa369ea17e7352d Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 3 Jul 2020 15:41:25 +1000 Subject: [PATCH 011/147] Adds migration for binary column on cmsContentNu --- .../Migrations/Upgrade/UmbracoPlan.cs | 3 ++- .../V_8_7_0/AddCmsContentNuByteColumn.cs | 21 ++++++++++++++++++ .../Persistence/Dtos/ContentNuDto.cs | 5 +++++ src/Umbraco.Core/Umbraco.Core.csproj | 1 + .../NuCache/DataSource/ContentSourceDto.cs | 2 ++ .../NuCache/DataSource/DatabaseDataSource.cs | 22 ++++++++++++++----- .../IContentNestedDataSerializer.cs | 12 ++++++++-- .../MsgPackContentNestedDataSerializer.cs | 8 ++++--- .../NuCache/PublishedSnapshotService.cs | 6 +++-- 9 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index f0cfc08281..4a4260b683 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -191,8 +191,9 @@ namespace Umbraco.Core.Migrations.Upgrade To("{EE288A91-531B-4995-8179-1D62D9AA3E2E}"); To("{2AB29964-02A1-474D-BD6B-72148D2A53A2}"); - + // to 8.7.0... To("{a78e3369-8ea3-40ec-ad3f-5f76929d2b20}"); + To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); //FINAL } diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs new file mode 100644 index 0000000000..3cb51afb99 --- /dev/null +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs @@ -0,0 +1,21 @@ +using System.Linq; +using Umbraco.Core.Persistence.Dtos; + +namespace Umbraco.Core.Migrations.Upgrade.V_8_6_0 +{ + public class AddCmsContentNuByteColumn : MigrationBase + { + public AddCmsContentNuByteColumn(IMigrationContext context) + : base(context) + { + + } + + public override void Migrate() + { + var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToList(); + + AddColumnIfNotExists(columns, "dataRaw"); + } + } +} diff --git a/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs b/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs index c6269d5317..664d188a10 100644 --- a/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs @@ -25,8 +25,13 @@ namespace Umbraco.Core.Persistence.Dtos /// [Column("data")] [SpecialDbType(SpecialDbTypes.NTEXT)] + [NullSetting(NullSetting = NullSettings.Null)] public string Data { get; set; } + [Column("dataRaw")] + [NullSetting(NullSetting = NullSettings.Null)] + public byte[] RawData { get; set; } + [Column("rv")] public long Rv { get; set; } } diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index c5a953631a..706c2a3650 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -132,6 +132,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs index 37fb9df8bb..be2f9921d1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs @@ -27,6 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public int EditWriterId { get; set; } public int EditTemplateId { get; set; } public string EditData { get; set; } + public byte[] EditDataRaw { get; set; } // published data public int PublishedVersionId { get; set; } @@ -35,5 +36,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public int PubWriterId { get; set; } public int PubTemplateId { get; set; } public string PubData { get; set; } + public byte[] PubDataRaw { get; set; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index e39f649eaf..48ff3e558c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -49,6 +49,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .AndSelect("nuEdit", x => Alias(x.Data, "EditData")) .AndSelect("nuPub", x => Alias(x.Data, "PubData")) + .AndSelect("nuEdit", x => Alias(x.RawData, "EditDataRaw")) + .AndSelect("nuPub", x => Alias(x.RawData, "PubDataRaw")) + .From(); if (joins != null) @@ -136,6 +139,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) .AndSelect(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId")) .AndSelect("nuEdit", x => Alias(x.Data, "EditData")) + .AndSelect("nuEdit", x => Alias(x.RawData, "EditDataRaw")) .From(); if (joins != null) @@ -211,7 +215,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (dto.Edited) { - if (dto.EditData == null) + if (dto.EditData == null && dto.EditDataRaw == null) { if (Debugger.IsAttached) throw new Exception("Missing cmsContentNu edited content for node " + dto.Id + ", consider rebuilding."); @@ -219,7 +223,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer.Deserialize(dto.EditData); + var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer + ? byteSerializer.DeserializeBytes(dto.EditDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.EditData); d = new ContentData { @@ -238,7 +244,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (dto.Published) { - if (dto.PubData == null) + if (dto.PubData == null && dto.PubDataRaw == null) { if (Debugger.IsAttached) throw new Exception("Missing cmsContentNu published content for node " + dto.Id + ", consider rebuilding."); @@ -246,7 +252,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer.Deserialize(dto.PubData); + var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer + ? byteSerializer.DeserializeBytes(dto.PubDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.PubData); p = new ContentData { @@ -279,10 +287,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto) { - if (dto.EditData == null) + if (dto.EditData == null && dto.EditDataRaw == null) throw new Exception("No data for media " + dto.Id); - var nested = _contentNestedDataSerializer.Deserialize(dto.EditData); + var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer + ? byteSerializer.DeserializeBytes(dto.EditDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.EditData); var p = new ContentData { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index d9e2702d08..edd061fbe4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -6,9 +6,17 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + // TODO: We need a better name, not sure why the class is called ContentNested in the first place + public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer + { + ContentNestedData DeserializeBytes(byte[] data); + byte[] SerializeBytes(ContentNestedData nestedData); + } + + // TODO: We need a better name, not sure why the class is called ContentNested in the first place public interface IContentNestedDataSerializer { - ContentNestedData Deserialize(string data); - string Serialize(ContentNestedData nestedData); + ContentNestedData Deserialize(string data); + string Serialize(ContentNestedData nestedData); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 99a306fecb..1a093292b5 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class MsgPackContentNestedDataSerializer : IContentNestedDataSerializer + internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; @@ -39,8 +39,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - // TODO: Instead of returning base64 it would be more ideal to avoid that translation entirely and just store/retrieve raw bytes - public ContentNestedData Deserialize(string data) { var bin = Convert.FromBase64String(data); @@ -54,6 +52,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return Convert.ToBase64String(bin); } + public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); + + public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); + //private class ContentNestedDataResolver : IFormatterResolver //{ // // GetFormatter's get cost should be minimized so use type cache. diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 6289667d3b..99396b45c9 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1322,9 +1322,10 @@ namespace Umbraco.Web.PublishedCache.NuCache var dto = GetDto(content, published); db.InsertOrUpdate(dto, - "SET data=@data, rv=rv+1 WHERE nodeId=@id AND published=@published", + "SET data=@data, dataRaw=@dataRaw, rv=rv+1 WHERE nodeId=@id AND published=@published", new { + dataRaw = dto.RawData, data = dto.Data, id = dto.NodeId, published = dto.Published @@ -1456,7 +1457,8 @@ namespace Umbraco.Web.PublishedCache.NuCache { NodeId = content.Id, Published = published, - Data = _contentNestedDataSerializer.Serialize(nestedData) + Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(nestedData) : null, + RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(nestedData) : null }; //Core.Composing.Current.Logger.Debug(dto.Data); From c79b17cc48aa7f84c8c66ad06a36c7bd90a4c29a Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 3 Jul 2020 15:46:36 +1000 Subject: [PATCH 012/147] fixing tests --- .../Serialization/AutoInterningStringConverterTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs b/src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs index bf99b68077..f83ea940c9 100644 --- a/src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs +++ b/src/Umbraco.Tests/Serialization/AutoInterningStringConverterTests.cs @@ -19,7 +19,7 @@ namespace Umbraco.Tests.Serialization var str1 = "Hello"; var obj = new Test { - Name = str1 + " " + "there" + Name = str1 + Guid.NewGuid() }; // ensure the raw value is not interned @@ -39,8 +39,8 @@ namespace Umbraco.Tests.Serialization { Values = new Dictionary { - [str1 + "1"] = 0, - [str1 + "2"] = 1 + [str1 + Guid.NewGuid()] = 0, + [str1 + Guid.NewGuid()] = 1 } }; From e8717e6bf8b86230a4ba6d09f37444d9fcddce1c Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 3 Jul 2020 16:30:20 +1000 Subject: [PATCH 013/147] fixing tests and sqlce blob column --- .../Persistence/SqlSyntax/SqlCeSyntaxProvider.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs index 2ed0fb878c..8a42caaa03 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs @@ -14,6 +14,14 @@ namespace Umbraco.Core.Persistence.SqlSyntax /// public class SqlCeSyntaxProvider : MicrosoftSqlSyntaxProviderBase { + public SqlCeSyntaxProvider() + { + BlobColumnDefinition = "IMAGE"; + // This is silly to have to do this but the way these inherited classes are structured it's the easiest + // way without an overhaul in type map initialization + DbTypeMap.Set(DbType.Binary, BlobColumnDefinition); + } + public override Sql SelectTop(Sql sql, int top) { return new Sql(sql.SqlContext, sql.SQL.Insert(sql.SQL.IndexOf(' '), " TOP " + top), sql.Arguments); @@ -227,8 +235,6 @@ where table_name=@0 and column_name=@1", tableName, columnName).FirstOrDefault() } } - - public override string DropIndex { get { return "DROP INDEX {1}.{0}"; } } } From 1280235dcfe98651123db9ed14c9ca613cae6f03 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 6 Jul 2020 16:25:15 +1000 Subject: [PATCH 014/147] Adds custom mapper for SqlCe for Image column, hopefully tests pass now. --- .../Persistence/SqlCeImageMapper.cs | 59 +++++++++++++++++++ .../Persistence/UmbracoDatabase.cs | 17 ++++-- src/Umbraco.Core/Umbraco.Core.csproj | 1 + .../NuCache/PublishedSnapshotService.cs | 2 +- 4 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 src/Umbraco.Core/Persistence/SqlCeImageMapper.cs diff --git a/src/Umbraco.Core/Persistence/SqlCeImageMapper.cs b/src/Umbraco.Core/Persistence/SqlCeImageMapper.cs new file mode 100644 index 0000000000..33303c68f0 --- /dev/null +++ b/src/Umbraco.Core/Persistence/SqlCeImageMapper.cs @@ -0,0 +1,59 @@ +using System; +using System.Data; +using System.Data.Common; +using System.Data.SqlServerCe; +using System.Linq; +using System.Reflection; +using NPoco; +using Umbraco.Core.Composing; + +namespace Umbraco.Core.Persistence +{ + /// + /// Custom NPoco mapper for SqlCe + /// + /// + /// Work arounds to handle special columns + /// + internal class SqlCeImageMapper : DefaultMapper + { + public override Func GetToDbConverter(Type destType, MemberInfo sourceMemberInfo) + { + if (sourceMemberInfo.GetMemberInfoType() == typeof(byte[])) + { + return x => + { + var pd = Current.SqlContext.PocoDataFactory.ForType(sourceMemberInfo.DeclaringType); + if (pd == null) return null; + var col = pd.AllColumns.FirstOrDefault(x => x.MemberInfoData.MemberInfo == sourceMemberInfo); + if (col == null) return null; + + return new SqlCeParameter + { + SqlDbType = SqlDbType.Image, + Value = x ?? Array.Empty() + }; + }; + } + return base.GetToDbConverter(destType, sourceMemberInfo); + } + + public override Func GetParameterConverter(DbCommand dbCommand, Type sourceType) + { + if (sourceType == typeof(byte[])) + { + return x => + { + var param = new SqlCeParameter + { + SqlDbType = SqlDbType.Image, + Value = x + }; + return param; + }; + + } + return base.GetParameterConverter(dbCommand, sourceType); + } + } +} diff --git a/src/Umbraco.Core/Persistence/UmbracoDatabase.cs b/src/Umbraco.Core/Persistence/UmbracoDatabase.cs index a95d95ea08..ec09db690f 100644 --- a/src/Umbraco.Core/Persistence/UmbracoDatabase.cs +++ b/src/Umbraco.Core/Persistence/UmbracoDatabase.cs @@ -10,6 +10,7 @@ using Umbraco.Core.Persistence.FaultHandling; namespace Umbraco.Core.Persistence { + /// /// Extends NPoco Database for Umbraco. /// @@ -38,14 +39,10 @@ namespace Umbraco.Core.Persistence : base(connectionString, sqlContext.DatabaseType, provider, sqlContext.SqlSyntax.DefaultIsolationLevel) { SqlContext = sqlContext; - _logger = logger; _connectionRetryPolicy = connectionRetryPolicy; _commandRetryPolicy = commandRetryPolicy; - - EnableSqlTrace = EnableSqlTraceDefault; - - NPocoDatabaseExtensions.ConfigureNPocoBulkExtensions(); + Init(); } /// @@ -57,10 +54,17 @@ namespace Umbraco.Core.Persistence { SqlContext = sqlContext; _logger = logger; + Init(); + } + private void Init() + { EnableSqlTrace = EnableSqlTraceDefault; - NPocoDatabaseExtensions.ConfigureNPocoBulkExtensions(); + if (SqlContext.DatabaseType == DatabaseType.SQLCe) + { + Mappers.Add(new SqlCeImageMapper()); + } } #endregion @@ -257,5 +261,6 @@ namespace Umbraco.Core.Persistence } #endregion + } } diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 706c2a3650..5e2411db09 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -140,6 +140,7 @@ + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 99396b45c9..c87f035589 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1325,7 +1325,7 @@ namespace Umbraco.Web.PublishedCache.NuCache "SET data=@data, dataRaw=@dataRaw, rv=rv+1 WHERE nodeId=@id AND published=@published", new { - dataRaw = dto.RawData, + dataRaw = dto.RawData ?? Array.Empty(), data = dto.Data, id = dto.NodeId, published = dto.Published From c1c189d47f73af319e473c08b38bb79b7ec0951b Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Thu, 9 Jul 2020 00:17:31 +1200 Subject: [PATCH 015/147] Wip support for compressing/decompressing nucache documents on a per property basis. Option for compressing the properties in sql/nucache.db. Option for immediate/lazy decompression of properties. Mapping support for shorter property alias. TODO: config file for property map TODO: HasValue and IsValue on propertyvalueconverterbase --- .../AppSettingsNucachePropertyMapFactory.cs | 47 +++++ .../DataSource/BTree.ContentDataSerializer.cs | 13 +- .../BTree.ContentNodeKitSerializer.cs | 20 +- ...Tree.DictionaryOfPropertyDataSerializer.cs | 2 +- .../NuCache/DataSource/BTree.cs | 4 +- .../IDictionaryOfPropertyDataSerializer.cs | 11 ++ .../INucachePropertyOptionsFactory.cs | 13 ++ .../DataSource/LazyCompressedString.cs | 31 +++ .../Lz4DictionaryOfPropertyDataSerializer.cs | 178 ++++++++++++++++++ .../MsgPackContentNestedDataSerializer.cs | 41 +++- .../DataSource/NucachePropertyOptions.cs | 20 ++ .../NuCache/DataSource/SerializerBase.cs | 21 ++- .../PublishedCache/NuCache/NuCacheComposer.cs | 22 ++- .../NuCache/PublishedSnapshotService.cs | 8 +- src/Umbraco.Web/Umbraco.Web.csproj | 9 + 15 files changed, 422 insertions(+), 18 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs new file mode 100644 index 0000000000..ddc9bc9b1a --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class AppSettingsNucachePropertyMapFactory : INucachePropertyOptionsFactory + { + public NucachePropertyOptions GetNucachePropertyOptions() + { + NucachePropertyOptions options = new NucachePropertyOptions + { + PropertyMap = GetPropertyMap(), + LZ4CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT, + MinimumCompressibleStringLength = null + }; + return options; + } + + public IReadOnlyDictionary GetPropertyMap() + { + var propertyMap = new Dictionary(); + // TODO: Use xml/json/c# to define map + var propertyDictionarySerializerMap = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.PropertySerializationMap"]; + if (!string.IsNullOrWhiteSpace(propertyDictionarySerializerMap)) + { + //propertyAlias,CompressionLevel,DecompressionLevel,mappedAlias; + propertyDictionarySerializerMap.Split(';') + .Select(x => + { + var y = x.Split(','); + (string alias, NucachePropertyCompressionLevel compressionLevel, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) v = (y[0], + (NucachePropertyCompressionLevel)System.Enum.Parse(typeof(NucachePropertyCompressionLevel), y[1]), + (NucachePropertyDecompressionLevel)System.Enum.Parse(typeof(NucachePropertyDecompressionLevel), y[2]), + y[3] + ); + return v; + }) + .ToList().ForEach(x => propertyMap.Add(x.alias, (x.compressionLevel, x.decompressionLevel, x.mappedAlias))); + } + return propertyMap; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs index d02af375c6..9cc5d3a701 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs @@ -5,8 +5,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { class ContentDataSerializer : ISerializer { + public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null) + { + _dictionaryOfPropertyDataSerializer = dictionaryOfPropertyDataSerializer; + if(_dictionaryOfPropertyDataSerializer == null) + { + _dictionaryOfPropertyDataSerializer = PropertiesSerializer; + } + } private static readonly DictionaryOfPropertyDataSerializer PropertiesSerializer = new DictionaryOfPropertyDataSerializer(); private static readonly DictionaryOfCultureVariationSerializer CultureVariationsSerializer = new DictionaryOfCultureVariationSerializer(); + private readonly IDictionaryOfPropertyDataSerializer _dictionaryOfPropertyDataSerializer; public ContentData ReadFrom(Stream stream) { @@ -19,7 +28,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionDate = PrimitiveSerializer.DateTime.ReadFrom(stream), WriterId = PrimitiveSerializer.Int32.ReadFrom(stream), TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream), - Properties = PropertiesSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays + Properties = _dictionaryOfPropertyDataSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays CultureInfos = CultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays }; } @@ -36,7 +45,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { PrimitiveSerializer.Int32.WriteTo(value.TemplateId.Value, stream); } - PropertiesSerializer.WriteTo(value.Properties, stream); + _dictionaryOfPropertyDataSerializer.WriteTo(value.Properties, stream); CultureVariationsSerializer.WriteTo(value.CultureInfos, stream); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs index f799869850..1d668ba4fd 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentNodeKitSerializer.cs @@ -5,7 +5,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class ContentNodeKitSerializer : ISerializer { - static readonly ContentDataSerializer DataSerializer = new ContentDataSerializer(); + public ContentNodeKitSerializer(ContentDataSerializer contentDataSerializer = null) + { + _contentDataSerializer = contentDataSerializer; + if(_contentDataSerializer == null) + { + _contentDataSerializer = DefaultDataSerializer; + } + } + static readonly ContentDataSerializer DefaultDataSerializer = new ContentDataSerializer(); + private readonly ContentDataSerializer _contentDataSerializer; + //static readonly ListOfIntSerializer ChildContentIdsSerializer = new ListOfIntSerializer(); public ContentNodeKit ReadFrom(Stream stream) @@ -26,10 +36,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource }; var hasDraft = PrimitiveSerializer.Boolean.ReadFrom(stream); if (hasDraft) - kit.DraftData = DataSerializer.ReadFrom(stream); + kit.DraftData = _contentDataSerializer.ReadFrom(stream); var hasPublished = PrimitiveSerializer.Boolean.ReadFrom(stream); if (hasPublished) - kit.PublishedData = DataSerializer.ReadFrom(stream); + kit.PublishedData = _contentDataSerializer.ReadFrom(stream); return kit; } @@ -47,11 +57,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Boolean.WriteTo(value.DraftData != null, stream); if (value.DraftData != null) - DataSerializer.WriteTo(value.DraftData, stream); + _contentDataSerializer.WriteTo(value.DraftData, stream); PrimitiveSerializer.Boolean.WriteTo(value.PublishedData != null, stream); if (value.PublishedData != null) - DataSerializer.WriteTo(value.PublishedData, stream); + _contentDataSerializer.WriteTo(value.PublishedData, stream); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index e3fa6597e4..cb12164397 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -6,7 +6,7 @@ using Umbraco.Core; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer> + internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { public IDictionary ReadFrom(Stream stream) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs index 910c0ca737..80d8488495 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs @@ -6,10 +6,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class BTree { - public static BPlusTree GetTree(string filepath, bool exists) + public static BPlusTree GetTree(string filepath, bool exists, ContentDataSerializer contentDataSerializer = null) { var keySerializer = new PrimitiveSerializer(); - var valueSerializer = new ContentNodeKitSerializer(); + var valueSerializer = new ContentNodeKitSerializer(contentDataSerializer); var options = new BPlusTree.OptionsV2(keySerializer, valueSerializer) { CreateFile = exists ? CreatePolicy.IfNeeded : CreatePolicy.Always, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs new file mode 100644 index 0000000000..a086e3e2f3 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IDictionaryOfPropertyDataSerializer.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.IO; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal interface IDictionaryOfPropertyDataSerializer + { + IDictionary ReadFrom(Stream stream); + void WriteTo(IDictionary value, Stream stream); + } +} \ No newline at end of file diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs new file mode 100644 index 0000000000..82df43a5bb --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public interface INucachePropertyOptionsFactory + { + NucachePropertyOptions GetNucachePropertyOptions(); + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs new file mode 100644 index 0000000000..99bc2f5859 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -0,0 +1,31 @@ +using K4os.Compression.LZ4; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class LazyCompressedString + { + private byte[] _bytes; + private string _str; + + public LazyCompressedString(byte[] bytes) + { + _bytes = bytes; + } + public override string ToString() + { + return LazyInitializer.EnsureInitialized(ref _str, () => + { + var str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); + _bytes = null; + return str; + }); + } + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs new file mode 100644 index 0000000000..7ec2fb17b6 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using CSharpTest.Net.Serialization; +using Umbraco.Core; +using System.Text; +using K4os.Compression.LZ4; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// If/where to compress custom properties for nucache + /// + public enum NucachePropertyCompressionLevel + { + None = 0, + SQLDatabase = 1, + NucacheDatabase = 2 + } + /// + /// If/where to decompress custom properties for nucache + /// + public enum NucachePropertyDecompressionLevel + { + NotCompressed = 0, + Immediate = 1, + Lazy = 2 + } + + + internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer + { + private readonly IReadOnlyDictionary _compressProperties; + private readonly LZ4Level _compressionLevel; + private readonly long? _minimumStringLengthForCompression; + private readonly IReadOnlyDictionary _uncompressProperties; + + + public Lz4DictionaryOfPropertyDataSerializer(NucachePropertyOptions nucachePropertyOptions) + { + _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Value.mappedAlias), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); + _minimumStringLengthForCompression = nucachePropertyOptions.MinimumCompressibleStringLength; + _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Value.mappedAlias)); + + _compressionLevel = nucachePropertyOptions.LZ4CompressionLevel; + } + + + public IDictionary ReadFrom(Stream stream) + { + var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + + // read properties count + var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + + // read each property + for (var i = 0; i < pcount; i++) + { + + // read property alias + var alias = PrimitiveSerializer.String.ReadFrom(stream); + var map = GetDeSerializationMap(alias); + var key = string.Intern(map.MappedAlias ?? alias); + + // read values count + var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); + + // create pdata and add to the dictionary + var pdatas = new List(); + + // for each value, read and add to pdata + for (var j = 0; j < vcount; j++) + { + var pdata = new PropertyData(); + pdatas.Add(pdata); + + // everything that can be null is read/written as object + // even though - culture and segment should never be null here, as 'null' represents + // the 'current' value, and string.Empty should be used to represent the invariant or + // neutral values - PropertyData throws when getting nulls, so falling back to + // string.Empty here - what else? + pdata.Culture = ReadStringObject(stream, true) ?? string.Empty; + pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; + pdata.Value = ReadObject(stream); + + var decompressionLevel = NucachePropertyDecompressionLevel.Immediate; + if ((map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) || map.Compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + && pdata.Value != null && pdata.Value is byte[] byteArrayValue) + { + //Compressed string + switch (decompressionLevel) + { + case NucachePropertyDecompressionLevel.Lazy: + pdata.Value = new LazyCompressedString(byteArrayValue); + break; + case NucachePropertyDecompressionLevel.NotCompressed: + break;//Shouldn't be any not compressed + case NucachePropertyDecompressionLevel.Immediate: + default: + pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); + break; + } + } + } + + dict[key] = pdatas.ToArray(); + } + return dict; + } + + public void WriteTo(IDictionary value, Stream stream) + { + // write properties count + PrimitiveSerializer.Int32.WriteTo(value.Count, stream); + + // write each property + foreach (var (alias, values) in value) + { + var map = GetSerializationMap(alias); + + // write alias + PrimitiveSerializer.String.WriteTo(map.MappedAlias ?? alias, stream); + + // write values count + PrimitiveSerializer.Int32.WriteTo(values.Length, stream); + + // write each value + foreach (var pdata in values) + { + // everything that can be null is read/written as object + // even though - culture and segment should never be null here, + // see note in ReadFrom() method above + WriteObject(pdata.Culture ?? string.Empty, stream); + WriteObject(pdata.Segment ?? string.Empty, stream); + + //Only compress strings + if (pdata.Value is string stringValue && pdata.Value != null && map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) + && (_minimumStringLengthForCompression == null + || !_minimumStringLengthForCompression.HasValue + || stringValue.Length > _minimumStringLengthForCompression.Value)) + { + var stringBytes = Encoding.UTF8.GetBytes(stringValue); + var compressedBytes = LZ4Pickler.Pickle(stringBytes, _compressionLevel); + WriteObject(compressedBytes, stream); + } + WriteObject(pdata.Value, stream); + } + } + } + private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) + { + if (_compressProperties == null) + { + return DEFAULT_MAP; + } + if (_compressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + { + return map; + } + + return DEFAULT_MAP; + } + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetDeSerializationMap(string propertyAlias) + { + if (_uncompressProperties == null) + { + return DEFAULT_MAP; + } + if (_uncompressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + { + return map; + } + return DEFAULT_MAP; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 1a093292b5..be4e8a9832 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,16 +1,20 @@ -using MessagePack; +using K4os.Compression.LZ4; +using MessagePack; using MessagePack.Formatters; using MessagePack.Resolvers; using System; using System.Collections.Generic; +using System.Linq; +using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; + private readonly NucachePropertyOptions _propertyOptions; - public MsgPackContentNestedDataSerializer() + public MsgPackContentNestedDataSerializer(NucachePropertyOptions propertyOptions = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -30,6 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); + _propertyOptions = propertyOptions ?? new NucachePropertyOptions(); } public string ToJson(string serialized) @@ -48,10 +53,42 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { + Optimize(nestedData); + var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + /// + /// Compress properties and map property names to shorter names + /// + /// + private void Optimize(ContentNestedData nestedData) + { + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Any()) + { + foreach (var map in _propertyOptions.PropertyMap) + { + if (map.Value.compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + { + if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) + { + foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) + { + property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); + } + } + } + if (map.Value.mappedAlias != null && !map.Key.Equals(map.Value.mappedAlias) + && nestedData.PropertyData.Remove(map.Key) && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) + { + nestedData.PropertyData.Remove(map.Key); + nestedData.PropertyData.Add(map.Value.mappedAlias, properties2); + } + } + } + } + public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs new file mode 100644 index 0000000000..f10787646a --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public class NucachePropertyOptions + { + public IReadOnlyDictionary PropertyMap + { get; set; } = new Dictionary(); + + public K4os.Compression.LZ4.LZ4Level LZ4CompressionLevel { get; set; } = K4os.Compression.LZ4.LZ4Level.L00_FAST; + + public long? MinimumCompressibleStringLength { get; set; } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index afc0827ed3..885b5cf80e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -16,6 +16,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixDouble = 'B'; private const char PrefixDateTime = 'D'; private const char PrefixByte = 'O'; + private const char PrefixByteArray = 'A'; protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); @@ -23,6 +24,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected float ReadFloat(Stream stream) => PrimitiveSerializer.Float.ReadFrom(stream); protected double ReadDouble(Stream stream) => PrimitiveSerializer.Double.ReadFrom(stream); protected DateTime ReadDateTime(Stream stream) => PrimitiveSerializer.DateTime.ReadFrom(stream); + protected byte[] ReadByteArray(Stream stream) => PrimitiveSerializer.Bytes.ReadFrom(stream); private T? ReadObject(Stream stream, char t, Func read) where T : struct @@ -39,7 +41,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var type = PrimitiveSerializer.Char.ReadFrom(stream); if (type == PrefixNull) return null; if (type != PrefixString) - throw new NotSupportedException($"Cannot deserialize type '{type}', expected 'S'."); + throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixString}'."); return intern ? string.Intern(PrimitiveSerializer.String.ReadFrom(stream)) : PrimitiveSerializer.String.ReadFrom(stream); @@ -51,6 +53,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble); protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime); + protected byte[] ReadByteArrayObject(Stream stream) // required 'cos byte[] is not a struct + { + var type = PrimitiveSerializer.Char.ReadFrom(stream); + if (type == PrefixNull) return null; + if (type != PrefixByteArray) + throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixByteArray}'."); + return PrimitiveSerializer.Bytes.ReadFrom(stream); + } + + protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); @@ -81,6 +93,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return PrimitiveSerializer.Double.ReadFrom(stream); case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); + case PrefixByteArray: + return PrimitiveSerializer.Bytes.ReadFrom(stream); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); } @@ -137,6 +151,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Char.WriteTo(PrefixUInt32, stream); PrimitiveSerializer.UInt32.WriteTo(uInt32Value, stream); } + else if (value is byte[] byteArrayValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); + PrimitiveSerializer.Bytes.WriteTo(byteArrayValue, stream); + } else throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized."); } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 205ac55cdc..95ff91304d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,4 +1,6 @@ -using System.Configuration; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -15,11 +17,26 @@ namespace Umbraco.Web.PublishedCache.NuCache if (serializer == "MsgPack") { - composition.Register(); + var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; + if (propertyDictionarySerializer == "LZ4Map") + { + composition.Register(); + composition.Register(factory => + { + var lz4Serializer = factory.GetInstance(); + return new ContentDataSerializer(lz4Serializer); + }); + } + else + { + composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + } + composition.Register(); } else { composition.Register(); + composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); } // register the NuCache database data source @@ -34,5 +51,6 @@ namespace Umbraco.Web.PublishedCache.NuCache // TODO: no NuCache health check yet //composition.HealthChecks().Add(); } + } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index c87f035589..0135f204c7 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -50,6 +50,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly ContentDataSerializer _contentDataSerializer; // volatile because we read it with no lock private volatile bool _isReady; @@ -83,7 +84,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer) + UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -101,6 +102,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; _contentNestedDataSerializer = contentNestedDataSerializer; + _contentDataSerializer = contentDataSerializer; // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member @@ -182,8 +184,8 @@ namespace Umbraco.Web.PublishedCache.NuCache _localMediaDbExists = File.Exists(localMediaDbPath); // if both local databases exist then GetTree will open them, else new databases will be created - _localContentDb = BTree.GetTree(localContentDbPath, _localContentDbExists); - _localMediaDb = BTree.GetTree(localMediaDbPath, _localMediaDbExists); + _localContentDb = BTree.GetTree(localContentDbPath, _localContentDbExists, _contentDataSerializer); + _localMediaDb = BTree.GetTree(localMediaDbPath, _localMediaDbExists, _contentDataSerializer); _logger.Info("Registered with MainDom, localContentDbExists? {LocalContentDbExists}, localMediaDbExists? {LocalMediaDbExists}", _localContentDbExists, _localMediaDbExists); } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 088fb0eeb3..6aae3f5ed9 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -68,6 +68,9 @@ 2.7.0.100 + + 1.1.11 + @@ -248,9 +251,15 @@ + + + + + + From d1449a0f5cb38a94ddb836085446826dc3aae8b4 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Thu, 9 Jul 2020 11:50:15 +1200 Subject: [PATCH 016/147] Fix key mapping --- .../DataSource/Lz4DictionaryOfPropertyDataSerializer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index 7ec2fb17b6..b1e913bdec 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -39,9 +39,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public Lz4DictionaryOfPropertyDataSerializer(NucachePropertyOptions nucachePropertyOptions) { - _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Value.mappedAlias), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); + _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); _minimumStringLengthForCompression = nucachePropertyOptions.MinimumCompressibleStringLength; - _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Value.mappedAlias)); + _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Key)); _compressionLevel = nucachePropertyOptions.LZ4CompressionLevel; } From d4276dff58dd9ed69afbf1e3af34da775f80bde2 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 4 Aug 2020 18:01:06 +1200 Subject: [PATCH 017/147] Fix composition. Store compression options in map --- .../AppSettingsNucachePropertyMapFactory.cs | 4 ++-- .../INucachePropertyOptionsFactory.cs | 4 ++-- .../DataSource/LazyCompressedString.cs | 10 +++++++- .../Lz4DictionaryOfPropertyDataSerializer.cs | 21 ++++++++-------- .../MsgPackContentNestedDataSerializer.cs | 4 ++-- .../PublishedCache/NuCache/NuCacheComposer.cs | 5 ++-- src/Umbraco.Web/Umbraco.Web.csproj | 2 +- src/umbraco.sln | 24 +++++++++++++++++-- 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs index ddc9bc9b1a..cf44290bd4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -7,9 +7,9 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public class AppSettingsNucachePropertyMapFactory : INucachePropertyOptionsFactory + public class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory { - public NucachePropertyOptions GetNucachePropertyOptions() + public NucachePropertyOptions GetNuCachePropertyOptions() { NucachePropertyOptions options = new NucachePropertyOptions { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs index 82df43a5bb..efc5e599ea 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -6,8 +6,8 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public interface INucachePropertyOptionsFactory + public interface INuCachePropertyOptionsFactory { - NucachePropertyOptions GetNucachePropertyOptions(); + NucachePropertyOptions GetNuCachePropertyOptions(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 99bc2f5859..3d6e70c7b2 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -8,15 +8,23 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public class LazyCompressedString + /// + /// Lazily decompresses a LZ4 Pickler compressed UTF8 string + /// + internal class LazyCompressedString { private byte[] _bytes; private string _str; + /// + /// Constructor + /// + /// LZ4 Pickle compressed UTF8 String public LazyCompressedString(byte[] bytes) { _bytes = bytes; } + public override string ToString() { return LazyInitializer.EnsureInitialized(ref _str, () => diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index b1e913bdec..a5e3034872 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -32,18 +32,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { private readonly IReadOnlyDictionary _compressProperties; - private readonly LZ4Level _compressionLevel; - private readonly long? _minimumStringLengthForCompression; private readonly IReadOnlyDictionary _uncompressProperties; - public Lz4DictionaryOfPropertyDataSerializer(NucachePropertyOptions nucachePropertyOptions) + public Lz4DictionaryOfPropertyDataSerializer(INuCachePropertyOptionsFactory nucachePropertyOptionsFactory) { + var nucachePropertyOptions = nucachePropertyOptionsFactory.GetNuCachePropertyOptions(); _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); - _minimumStringLengthForCompression = nucachePropertyOptions.MinimumCompressibleStringLength; _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Key)); - _compressionLevel = nucachePropertyOptions.LZ4CompressionLevel; + _nucachePropertyOptions = nucachePropertyOptions; } @@ -84,12 +82,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; pdata.Value = ReadObject(stream); - var decompressionLevel = NucachePropertyDecompressionLevel.Immediate; if ((map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) || map.Compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) && pdata.Value != null && pdata.Value is byte[] byteArrayValue) { //Compressed string - switch (decompressionLevel) + switch (map.decompressionLevel) { case NucachePropertyDecompressionLevel.Lazy: pdata.Value = new LazyCompressedString(byteArrayValue); @@ -136,12 +133,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource //Only compress strings if (pdata.Value is string stringValue && pdata.Value != null && map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) - && (_minimumStringLengthForCompression == null - || !_minimumStringLengthForCompression.HasValue - || stringValue.Length > _minimumStringLengthForCompression.Value)) + && (_nucachePropertyOptions.MinimumCompressibleStringLength == null + || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue + || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) { var stringBytes = Encoding.UTF8.GetBytes(stringValue); - var compressedBytes = LZ4Pickler.Pickle(stringBytes, _compressionLevel); + var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); WriteObject(compressedBytes, stream); } WriteObject(pdata.Value, stream); @@ -149,6 +146,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); + private readonly NucachePropertyOptions _nucachePropertyOptions; + public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) { if (_compressProperties == null) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index be4e8a9832..c83bc3e973 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -14,7 +14,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private MessagePackSerializerOptions _options; private readonly NucachePropertyOptions _propertyOptions; - public MsgPackContentNestedDataSerializer(NucachePropertyOptions propertyOptions = null) + public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -34,7 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptions ?? new NucachePropertyOptions(); + _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NucachePropertyOptions(); } public string ToJson(string serialized) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 95ff91304d..1ee6b96eb1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -14,13 +14,14 @@ namespace Umbraco.Web.PublishedCache.NuCache base.Compose(composition); var serializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.Serializer"]; - + composition.Register(); + composition.Register(); if (serializer == "MsgPack") { var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; if (propertyDictionarySerializer == "LZ4Map") { - composition.Register(); + composition.Register(factory => { var lz4Serializer = factory.GetInstance(); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 6aae3f5ed9..549492603b 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -251,7 +251,7 @@ - + diff --git a/src/umbraco.sln b/src/umbraco.sln index 63fb856b5d..da9d7ae557 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -58,8 +58,24 @@ Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Web.UI.Client", "ht StartServerOnDebug = "false" EndProjectSection EndProject -Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest\", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" +Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" ProjectSection(WebsiteProperties) = preProject + TargetFrameworkMoniker = ".NETFramework,Version%3Dv4.0" + Debug.AspNetCompiler.VirtualPath = "/localhost_49800" + Debug.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" + Debug.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" + Debug.AspNetCompiler.Updateable = "true" + Debug.AspNetCompiler.ForceOverwrite = "true" + Debug.AspNetCompiler.FixedNames = "false" + Debug.AspNetCompiler.Debug = "True" + Release.AspNetCompiler.VirtualPath = "/localhost_49800" + Release.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" + Release.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" + Release.AspNetCompiler.Updateable = "true" + Release.AspNetCompiler.ForceOverwrite = "true" + Release.AspNetCompiler.FixedNames = "false" + Release.AspNetCompiler.Debug = "False" + VWDPort = "49800" SlnRelativePath = "Umbraco.Tests.AcceptanceTest\" EndProjectSection EndProject @@ -123,6 +139,10 @@ Global {4C4C194C-B5E4-4991-8F87-4373E24CC19F}.Release|Any CPU.Build.0 = Release|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Release|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -157,6 +177,7 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {227C3B55-80E5-4E7E-A802-BE16C5128B9D} = {2849E9D4-3B4E-40A3-A309-F3CB4F0E125F} + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {5D3B8245-ADA6-453F-A008-50ED04BFE770} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {E3F9F378-AFE1-40A5-90BD-82833375DBFE} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} {5B03EF4E-E0AC-4905-861B-8C3EC1A0D458} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} @@ -164,7 +185,6 @@ Global {3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {C7311C00-2184-409B-B506-52A5FAEA8736} = {FD962632-184C-4005-A5F3-E705D92FC645} {FB5676ED-7A69-492C-B802-E7B24144C0FC} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7A0F2E34-D2AF-4DAB-86A0-7D7764B3D0EC} From 451eacf7cbcee1c175fe676f5e1f9e237d406253 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 4 Aug 2020 18:03:08 +1200 Subject: [PATCH 018/147] Fix NuCache spelling --- .../DataSource/AppSettingsNucachePropertyMapFactory.cs | 4 ++-- .../NuCache/DataSource/INucachePropertyOptionsFactory.cs | 2 +- .../DataSource/Lz4DictionaryOfPropertyDataSerializer.cs | 2 +- .../NuCache/DataSource/MsgPackContentNestedDataSerializer.cs | 4 ++-- .../NuCache/DataSource/NucachePropertyOptions.cs | 2 +- src/Umbraco.Web/Umbraco.Web.csproj | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs index cf44290bd4..6ca4c3e666 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -9,9 +9,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory { - public NucachePropertyOptions GetNuCachePropertyOptions() + public NuCachePropertyOptions GetNuCachePropertyOptions() { - NucachePropertyOptions options = new NucachePropertyOptions + NuCachePropertyOptions options = new NuCachePropertyOptions { PropertyMap = GetPropertyMap(), LZ4CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs index efc5e599ea..0cb694e1c1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -8,6 +8,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public interface INuCachePropertyOptionsFactory { - NucachePropertyOptions GetNuCachePropertyOptions(); + NuCachePropertyOptions GetNuCachePropertyOptions(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index a5e3034872..501ead8386 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -146,7 +146,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); - private readonly NucachePropertyOptions _nucachePropertyOptions; + private readonly NuCachePropertyOptions _nucachePropertyOptions; public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index c83bc3e973..31421e7177 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -12,7 +12,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; - private readonly NucachePropertyOptions _propertyOptions; + private readonly NuCachePropertyOptions _propertyOptions; public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) { @@ -34,7 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NucachePropertyOptions(); + _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NuCachePropertyOptions(); } public string ToJson(string serialized) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs index f10787646a..be0118f563 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public class NucachePropertyOptions + public class NuCachePropertyOptions { public IReadOnlyDictionary PropertyMap diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 549492603b..a4e5640d36 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -254,12 +254,12 @@ - + - + From e65f3a7e80a91e5296e7f5e8e1ad5d70a8545ef0 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 4 Aug 2020 18:21:16 +1200 Subject: [PATCH 019/147] undo solution changes --- src/umbraco.sln | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/src/umbraco.sln b/src/umbraco.sln index da9d7ae557..63fb856b5d 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -58,24 +58,8 @@ Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Web.UI.Client", "ht StartServerOnDebug = "false" EndProjectSection EndProject -Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" +Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest\", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" ProjectSection(WebsiteProperties) = preProject - TargetFrameworkMoniker = ".NETFramework,Version%3Dv4.0" - Debug.AspNetCompiler.VirtualPath = "/localhost_49800" - Debug.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" - Debug.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" - Debug.AspNetCompiler.Updateable = "true" - Debug.AspNetCompiler.ForceOverwrite = "true" - Debug.AspNetCompiler.FixedNames = "false" - Debug.AspNetCompiler.Debug = "True" - Release.AspNetCompiler.VirtualPath = "/localhost_49800" - Release.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" - Release.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_49800\" - Release.AspNetCompiler.Updateable = "true" - Release.AspNetCompiler.ForceOverwrite = "true" - Release.AspNetCompiler.FixedNames = "false" - Release.AspNetCompiler.Debug = "False" - VWDPort = "49800" SlnRelativePath = "Umbraco.Tests.AcceptanceTest\" EndProjectSection EndProject @@ -139,10 +123,6 @@ Global {4C4C194C-B5E4-4991-8F87-4373E24CC19F}.Release|Any CPU.Build.0 = Release|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -177,7 +157,6 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {227C3B55-80E5-4E7E-A802-BE16C5128B9D} = {2849E9D4-3B4E-40A3-A309-F3CB4F0E125F} - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {5D3B8245-ADA6-453F-A008-50ED04BFE770} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {E3F9F378-AFE1-40A5-90BD-82833375DBFE} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} {5B03EF4E-E0AC-4905-861B-8C3EC1A0D458} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} @@ -185,6 +164,7 @@ Global {3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {C7311C00-2184-409B-B506-52A5FAEA8736} = {FD962632-184C-4005-A5F3-E705D92FC645} {FB5676ED-7A69-492C-B802-E7B24144C0FC} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7A0F2E34-D2AF-4DAB-86A0-7D7764B3D0EC} From 0d159751b99eb5bc3946e69e58271ce1bb5d507a Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 13 Aug 2020 19:15:13 +1000 Subject: [PATCH 020/147] Cleans up code - removes tuples everywhere --- .../AppSettingsNucachePropertyMapFactory.cs | 9 ++-- .../Lz4DictionaryOfPropertyDataSerializer.cs | 42 ++++++++-------- .../MsgPackContentNestedDataSerializer.cs | 6 +-- .../DataSource/NuCacheCompressionOptions.cs | 50 +++++++++++++++++++ .../DataSource/NucachePropertyOptions.cs | 9 ++-- src/Umbraco.Web/Umbraco.Web.csproj | 1 + 6 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs index 6ca4c3e666..271695d250 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -7,7 +7,8 @@ using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory + // TODO: We'll remove this when the responsibility for compressing property data is at the property editor level + internal class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory { public NuCachePropertyOptions GetNuCachePropertyOptions() { @@ -20,9 +21,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return options; } - public IReadOnlyDictionary GetPropertyMap() + public IReadOnlyDictionary GetPropertyMap() { - var propertyMap = new Dictionary(); + var propertyMap = new Dictionary(); // TODO: Use xml/json/c# to define map var propertyDictionarySerializerMap = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.PropertySerializationMap"]; if (!string.IsNullOrWhiteSpace(propertyDictionarySerializerMap)) @@ -39,7 +40,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource ); return v; }) - .ToList().ForEach(x => propertyMap.Add(x.alias, (x.compressionLevel, x.decompressionLevel, x.mappedAlias))); + .ToList().ForEach(x => propertyMap.Add(x.alias, new NuCacheCompressionOptions(x.compressionLevel, x.decompressionLevel, x.mappedAlias))); } return propertyMap; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index 501ead8386..c62af1293f 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -16,7 +16,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { None = 0, SQLDatabase = 1, - NucacheDatabase = 2 + NuCacheDatabase = 2 } /// /// If/where to decompress custom properties for nucache @@ -28,22 +28,22 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource Lazy = 2 } - + internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { - private readonly IReadOnlyDictionary _compressProperties; - private readonly IReadOnlyDictionary _uncompressProperties; + private readonly IReadOnlyDictionary _compressProperties; + private readonly IReadOnlyDictionary _uncompressProperties; public Lz4DictionaryOfPropertyDataSerializer(INuCachePropertyOptionsFactory nucachePropertyOptionsFactory) { var nucachePropertyOptions = nucachePropertyOptionsFactory.GetNuCachePropertyOptions(); - _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => (x.Value.compress,x.Value.decompressionLevel, string.Intern(x.Value.mappedAlias))); - _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.mappedAlias, x => (x.Value.compress, x.Value.decompressionLevel, x.Key)); + _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => new NuCacheCompressionOptions(x.Value.CompressLevel, x.Value.DecompressLevel, string.Intern(x.Value.MappedAlias))); + _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.MappedAlias, x => new NuCacheCompressionOptions(x.Value.CompressLevel, x.Value.DecompressLevel, x.Key)); _nucachePropertyOptions = nucachePropertyOptions; } - + public IDictionary ReadFrom(Stream stream) { @@ -82,11 +82,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; pdata.Value = ReadObject(stream); - if ((map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) || map.Compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + if ((map.CompressLevel.Equals(NucachePropertyCompressionLevel.NuCacheDatabase) || map.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) && pdata.Value != null && pdata.Value is byte[] byteArrayValue) { //Compressed string - switch (map.decompressionLevel) + switch (map.DecompressLevel) { case NucachePropertyDecompressionLevel.Lazy: pdata.Value = new LazyCompressedString(byteArrayValue); @@ -132,7 +132,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource WriteObject(pdata.Segment ?? string.Empty, stream); //Only compress strings - if (pdata.Value is string stringValue && pdata.Value != null && map.Compress.Equals(NucachePropertyCompressionLevel.NucacheDatabase) + if (pdata.Value is string stringValue && pdata.Value != null && map.CompressLevel.Equals(NucachePropertyCompressionLevel.NuCacheDatabase) && (_nucachePropertyOptions.MinimumCompressibleStringLength == null || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) @@ -145,33 +145,33 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } } - private readonly (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) DEFAULT_MAP =(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); + private static readonly NuCacheCompressionOptions DefaultMap = new NuCacheCompressionOptions(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); private readonly NuCachePropertyOptions _nucachePropertyOptions; - public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetSerializationMap(string propertyAlias) + public NuCacheCompressionOptions GetSerializationMap(string propertyAlias) { if (_compressProperties == null) { - return DEFAULT_MAP; + return DefaultMap; } - if (_compressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + if (_compressProperties.TryGetValue(propertyAlias, out var map1)) { - return map; + return map1; } - return DEFAULT_MAP; + return DefaultMap; } - public (NucachePropertyCompressionLevel Compress, NucachePropertyDecompressionLevel decompressionLevel, string MappedAlias) GetDeSerializationMap(string propertyAlias) + public NuCacheCompressionOptions GetDeSerializationMap(string propertyAlias) { if (_uncompressProperties == null) { - return DEFAULT_MAP; + return DefaultMap; } - if (_uncompressProperties.TryGetValue(propertyAlias, out (NucachePropertyCompressionLevel compress, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) map)) + if (_uncompressProperties.TryGetValue(propertyAlias, out var map2)) { - return map; + return map2; } - return DEFAULT_MAP; + return DefaultMap; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 31421e7177..133586382c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -69,7 +69,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { foreach (var map in _propertyOptions.PropertyMap) { - if (map.Value.compress.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) { if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) { @@ -79,11 +79,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } } - if (map.Value.mappedAlias != null && !map.Key.Equals(map.Value.mappedAlias) + if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) && nestedData.PropertyData.Remove(map.Key) && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) { nestedData.PropertyData.Remove(map.Key); - nestedData.PropertyData.Add(map.Value.mappedAlias, properties2); + nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs new file mode 100644 index 0000000000..64f1b675bc --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public struct NuCacheCompressionOptions : IEquatable + { + public NuCacheCompressionOptions(NucachePropertyCompressionLevel compressLevel, NucachePropertyDecompressionLevel decompressLevel, string mappedAlias) + { + CompressLevel = compressLevel; + DecompressLevel = decompressLevel; + MappedAlias = mappedAlias ?? throw new ArgumentNullException(nameof(mappedAlias)); + } + + public NucachePropertyCompressionLevel CompressLevel { get; private set; } + public NucachePropertyDecompressionLevel DecompressLevel { get; private set; } + public string MappedAlias { get; private set; } + + public override bool Equals(object obj) + { + return obj is NuCacheCompressionOptions options && Equals(options); + } + + public bool Equals(NuCacheCompressionOptions other) + { + return CompressLevel == other.CompressLevel && + DecompressLevel == other.DecompressLevel && + MappedAlias == other.MappedAlias; + } + + public override int GetHashCode() + { + var hashCode = 961370163; + hashCode = hashCode * -1521134295 + CompressLevel.GetHashCode(); + hashCode = hashCode * -1521134295 + DecompressLevel.GetHashCode(); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(MappedAlias); + return hashCode; + } + + public static bool operator ==(NuCacheCompressionOptions left, NuCacheCompressionOptions right) + { + return left.Equals(right); + } + + public static bool operator !=(NuCacheCompressionOptions left, NuCacheCompressionOptions right) + { + return !(left == right); + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs index be0118f563..f88f30ccd8 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs @@ -1,17 +1,14 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + public class NuCachePropertyOptions { - public IReadOnlyDictionary PropertyMap - { get; set; } = new Dictionary(); + public IReadOnlyDictionary PropertyMap { get; set; } = new Dictionary(); public K4os.Compression.LZ4.LZ4Level LZ4CompressionLevel { get; set; } = K4os.Compression.LZ4.LZ4Level.L00_FAST; diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index a4e5640d36..18c4a8ed93 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -259,6 +259,7 @@ + From cb691926787c57fc87d009f120409248d3525921 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Thu, 13 Aug 2020 23:11:03 +1200 Subject: [PATCH 021/147] Support include in sql server indexes --- .../DatabaseAnnotations/IndexAttribute.cs | 5 +++++ .../DefinitionFactory.cs | 8 ++++++++ .../IndexDefinition.cs | 1 + .../SqlSyntax/SqlServerSyntaxProvider.cs | 19 +++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/src/Umbraco.Core/Persistence/DatabaseAnnotations/IndexAttribute.cs b/src/Umbraco.Core/Persistence/DatabaseAnnotations/IndexAttribute.cs index 138dceff09..5aaabbfa6f 100644 --- a/src/Umbraco.Core/Persistence/DatabaseAnnotations/IndexAttribute.cs +++ b/src/Umbraco.Core/Persistence/DatabaseAnnotations/IndexAttribute.cs @@ -31,5 +31,10 @@ namespace Umbraco.Core.Persistence.DatabaseAnnotations /// Gets or sets the column name(s) for the current index /// public string ForColumns { get; set; } + + /// + /// Gets or sets the column name(s) for the columns to include in the index + /// + public string IncludeColumns { get; set; } } } diff --git a/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/DefinitionFactory.cs b/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/DefinitionFactory.cs index 5925e58afc..129b200c17 100644 --- a/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/DefinitionFactory.cs +++ b/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/DefinitionFactory.cs @@ -166,6 +166,14 @@ namespace Umbraco.Core.Persistence.DatabaseModelDefinitions definition.Columns.Add(new IndexColumnDefinition {Name = column, Direction = Direction.Ascending}); } } + if (string.IsNullOrEmpty(attribute.IncludeColumns) == false) + { + var columns = attribute.IncludeColumns.Split(',').Select(p => p.Trim()); + foreach (var column in columns) + { + definition.IncludeColumns.Add(new IndexColumnDefinition { Name = column, Direction = Direction.Ascending }); + } + } return definition; } } diff --git a/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs b/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs index 582f9a40f7..8efb795948 100644 --- a/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs +++ b/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs @@ -17,6 +17,7 @@ namespace Umbraco.Core.Persistence.DatabaseModelDefinitions public virtual string ColumnName { get; set; } public virtual ICollection Columns { get; set; } + public virtual ICollection IncludeColumns { get; set; } public IndexTypes IndexType { get; set; } } } diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs index bb50fa98a1..6c53a94f90 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs @@ -336,5 +336,24 @@ where tbl.[name]=@0 and col.[name]=@1;", tableName, columnName) public override string DropIndex => "DROP INDEX {0} ON {1}"; public override string RenameColumn => "sp_rename '{0}.{1}', '{2}', 'COLUMN'"; + + public override string CreateIndex => "CREATE {0}{1}INDEX {2} ON {3} ({4}){5}"; + public override string Format(IndexDefinition index) + { + var name = string.IsNullOrEmpty(index.Name) + ? $"IX_{index.TableName}_{index.ColumnName}" + : index.Name; + + var columns = index.Columns.Any() + ? string.Join(",", index.Columns.Select(x => GetQuotedColumnName(x.Name))) + : GetQuotedColumnName(index.ColumnName); + + var includeColumns = index.IncludeColumns.Any() + ? " INCLUDE " + string.Join(",", index.IncludeColumns.Select(x => GetQuotedColumnName(x.Name))) + : GetQuotedColumnName(index.ColumnName); + + return string.Format(CreateIndex, GetIndexType(index.IndexType), " ", GetQuotedName(name), + GetQuotedTableName(index.TableName), columns, includeColumns); + } } } From 07de0876e776bca8ac1b43a46d3ea662d10125c2 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 13 Aug 2020 22:15:09 +1000 Subject: [PATCH 022/147] Cleans up some code, adds code comments --- .../DataSource/BTree.ContentDataSerializer.cs | 15 +-- ....DictionaryOfCultureVariationSerializer.cs | 3 + ...Tree.DictionaryOfPropertyDataSerializer.cs | 3 + .../NuCache/DataSource/ContentData.cs | 4 +- .../IContentNestedDataSerializer.cs | 18 ++-- .../DataSource/LazyCompressedString.cs | 2 + .../Lz4DictionaryOfPropertyDataSerializer.cs | 102 +++++++++--------- .../MsgPackContentNestedDataSerializer.cs | 12 ++- .../DataSource/NuCacheCompressionOptions.cs | 7 ++ .../NucachePropertyCompressionLevel.cs | 23 ++++ .../NucachePropertyDecompressionLevel.cs | 12 +++ src/Umbraco.Web/Umbraco.Web.csproj | 2 + 12 files changed, 135 insertions(+), 68 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs index 9cc5d3a701..1192c892d4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.ContentDataSerializer.cs @@ -3,18 +3,21 @@ using CSharpTest.Net.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - class ContentDataSerializer : ISerializer + /// + /// Serializes/Deserializes data to BTree data source for + /// + internal class ContentDataSerializer : ISerializer { public ContentDataSerializer(IDictionaryOfPropertyDataSerializer dictionaryOfPropertyDataSerializer = null) { _dictionaryOfPropertyDataSerializer = dictionaryOfPropertyDataSerializer; if(_dictionaryOfPropertyDataSerializer == null) { - _dictionaryOfPropertyDataSerializer = PropertiesSerializer; + _dictionaryOfPropertyDataSerializer = DefaultPropertiesSerializer; } } - private static readonly DictionaryOfPropertyDataSerializer PropertiesSerializer = new DictionaryOfPropertyDataSerializer(); - private static readonly DictionaryOfCultureVariationSerializer CultureVariationsSerializer = new DictionaryOfCultureVariationSerializer(); + private static readonly DictionaryOfPropertyDataSerializer DefaultPropertiesSerializer = new DictionaryOfPropertyDataSerializer(); + private static readonly DictionaryOfCultureVariationSerializer DefaultCultureVariationsSerializer = new DictionaryOfCultureVariationSerializer(); private readonly IDictionaryOfPropertyDataSerializer _dictionaryOfPropertyDataSerializer; public ContentData ReadFrom(Stream stream) @@ -29,7 +32,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource WriterId = PrimitiveSerializer.Int32.ReadFrom(stream), TemplateId = PrimitiveSerializer.Int32.ReadFrom(stream), Properties = _dictionaryOfPropertyDataSerializer.ReadFrom(stream), // TODO: We don't want to allocate empty arrays - CultureInfos = CultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays + CultureInfos = DefaultCultureVariationsSerializer.ReadFrom(stream) // TODO: We don't want to allocate empty arrays }; } @@ -46,7 +49,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Int32.WriteTo(value.TemplateId.Value, stream); } _dictionaryOfPropertyDataSerializer.WriteTo(value.Properties, stream); - CultureVariationsSerializer.WriteTo(value.CultureInfos, stream); + DefaultCultureVariationsSerializer.WriteTo(value.CultureInfos, stream); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs index 1fcbdbdae7..b3a6ef2d05 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfCultureVariationSerializer.cs @@ -6,6 +6,9 @@ using Umbraco.Core; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// + /// Serializes/Deserializes culture variant data as a dictionary for BTree + /// internal class DictionaryOfCultureVariationSerializer : SerializerBase, ISerializer> { public IReadOnlyDictionary ReadFrom(Stream stream) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index cb12164397..0b15c0ba4b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -6,6 +6,9 @@ using Umbraco.Core; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// + /// Serializes/Deserializes property data as a dictionary for BTree + /// internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { public IDictionary ReadFrom(Stream stream) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs index 36586acd12..5f252591e0 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentData.cs @@ -3,7 +3,9 @@ using System.Collections.Generic; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - // represents everything that is specific to edited or published version + /// + /// Represents everything that is specific to an edited or published content version + /// internal class ContentData { public string Name { get; set; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index edd061fbe4..32eb388bee 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -1,19 +1,19 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource +namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - // TODO: We need a better name, not sure why the class is called ContentNested in the first place + // TODO: We need better names if possible, not sure why the class is called ContentNested in the first place + + /// + /// Serializes/Deserializes document to the SQL Database as bytes + /// public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer { ContentNestedData DeserializeBytes(byte[] data); byte[] SerializeBytes(ContentNestedData nestedData); } - // TODO: We need a better name, not sure why the class is called ContentNested in the first place + /// + /// Serializes/Deserializes document to the SQL Database as a string + /// public interface IContentNestedDataSerializer { ContentNestedData Deserialize(string data); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 3d6e70c7b2..55469ad791 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -13,6 +13,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// internal class LazyCompressedString { + // TODO: This could be a struct + private byte[] _bytes; private string _str; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index c62af1293f..31f7763e04 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -9,26 +9,10 @@ using K4os.Compression.LZ4; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - /// - /// If/where to compress custom properties for nucache - /// - public enum NucachePropertyCompressionLevel - { - None = 0, - SQLDatabase = 1, - NuCacheDatabase = 2 - } - /// - /// If/where to decompress custom properties for nucache - /// - public enum NucachePropertyDecompressionLevel - { - NotCompressed = 0, - Immediate = 1, - Lazy = 2 - } - + /// + /// Serializes/Deserializes property data as a dictionary for BTree with Lz4 compression options + /// internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer { private readonly IReadOnlyDictionary _compressProperties; @@ -58,7 +42,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // read property alias var alias = PrimitiveSerializer.String.ReadFrom(stream); - var map = GetDeSerializationMap(alias); + var map = GetDeserializationMap(alias); var key = string.Intern(map.MappedAlias ?? alias); // read values count @@ -82,23 +66,30 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; pdata.Value = ReadObject(stream); - if ((map.CompressLevel.Equals(NucachePropertyCompressionLevel.NuCacheDatabase) || map.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) - && pdata.Value != null && pdata.Value is byte[] byteArrayValue) - { - //Compressed string - switch (map.DecompressLevel) - { - case NucachePropertyDecompressionLevel.Lazy: - pdata.Value = new LazyCompressedString(byteArrayValue); - break; - case NucachePropertyDecompressionLevel.NotCompressed: - break;//Shouldn't be any not compressed - case NucachePropertyDecompressionLevel.Immediate: - default: - pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); - break; - } - } + switch (map.CompressLevel) + { + case NucachePropertyCompressionLevel.SQLDatabase: + case NucachePropertyCompressionLevel.NuCacheDatabase: + if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) + { + //Compressed string + switch (map.DecompressLevel) + { + case NucachePropertyDecompressionLevel.Lazy: + pdata.Value = new LazyCompressedString(byteArrayValue); + break; + case NucachePropertyDecompressionLevel.NotCompressed: + //Shouldn't be any not compressed + // TODO: Do we need to throw here? + break; + case NucachePropertyDecompressionLevel.Immediate: + default: + pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); + break; + } + } + break; + } } dict[key] = pdatas.ToArray(); @@ -131,17 +122,30 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource WriteObject(pdata.Culture ?? string.Empty, stream); WriteObject(pdata.Segment ?? string.Empty, stream); - //Only compress strings - if (pdata.Value is string stringValue && pdata.Value != null && map.CompressLevel.Equals(NucachePropertyCompressionLevel.NuCacheDatabase) - && (_nucachePropertyOptions.MinimumCompressibleStringLength == null - || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue - || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) + //Only compress strings + switch (map.CompressLevel) { - var stringBytes = Encoding.UTF8.GetBytes(stringValue); - var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); - WriteObject(compressedBytes, stream); + // If we're compressing into btree at the property level + case NucachePropertyCompressionLevel.NuCacheDatabase: + + if (pdata.Value is string stringValue && !(pdata.Value is null) + && (_nucachePropertyOptions.MinimumCompressibleStringLength is null + || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue + || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) + { + var stringBytes = Encoding.UTF8.GetBytes(stringValue); + var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); + WriteObject(compressedBytes, stream); + } + else + { + WriteObject(pdata.Value, stream); + } + break; + default: + WriteObject(pdata.Value, stream); + break; } - WriteObject(pdata.Value, stream); } } } @@ -150,7 +154,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public NuCacheCompressionOptions GetSerializationMap(string propertyAlias) { - if (_compressProperties == null) + if (_compressProperties is null) { return DefaultMap; } @@ -161,9 +165,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return DefaultMap; } - public NuCacheCompressionOptions GetDeSerializationMap(string propertyAlias) + public NuCacheCompressionOptions GetDeserializationMap(string propertyAlias) { - if (_uncompressProperties == null) + if (_uncompressProperties is null) { return DefaultMap; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 133586382c..206051f839 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -9,6 +9,9 @@ using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// + /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack + /// internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; @@ -65,7 +68,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// private void Optimize(ContentNestedData nestedData) { - if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Any()) + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) { foreach (var map in _propertyOptions.PropertyMap) { @@ -79,10 +82,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } } + + // if there is an alias map for this property then use that instead of the real property alias + // (used to save memory, the mapped alias is normally a single char or at least a smaller string) if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.Remove(map.Key) && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) + && nestedData.PropertyData.Remove(map.Key) + && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) { - nestedData.PropertyData.Remove(map.Key); nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs index 64f1b675bc..36f1606008 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs @@ -14,6 +14,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public NucachePropertyCompressionLevel CompressLevel { get; private set; } public NucachePropertyDecompressionLevel DecompressLevel { get; private set; } + + /// + /// Used to map a real property alias to a shorter moniker in memory + /// + /// + /// This is simply a memory saving mechanism + /// public string MappedAlias { get; private set; } public override bool Equals(object obj) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs new file mode 100644 index 0000000000..2f24a203ca --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs @@ -0,0 +1,23 @@ +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// If/where to compress custom properties for nucache + /// + public enum NucachePropertyCompressionLevel + { + None = 0, + + /// + /// Compress property data at the nucache SQL DB table level + /// + /// + /// Only necessary if the document in the nucache SQL DB table isn't stored as compressed bytes + /// + SQLDatabase = 1, + + /// + /// Compress property data at the nucache BTree level + /// + NuCacheDatabase = 2 + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs new file mode 100644 index 0000000000..335ae6ff4a --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs @@ -0,0 +1,12 @@ +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + /// + /// If/where to decompress custom properties for nucache + /// + public enum NucachePropertyDecompressionLevel + { + NotCompressed = 0, + Immediate = 1, + Lazy = 2 + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 18c4a8ed93..5715a5e060 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -260,6 +260,8 @@ + + From 044585a4f3a0e77a3a73b2bd45e06abf2b0d2db3 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 13 Aug 2020 22:22:06 +1000 Subject: [PATCH 023/147] removes the SQLDatabase enum value --- .../IContentNestedDataSerializer.cs | 2 +- .../Lz4DictionaryOfPropertyDataSerializer.cs | 1 - .../MsgPackContentNestedDataSerializer.cs | 50 ++----------------- .../NucachePropertyCompressionLevel.cs | 8 --- 4 files changed, 5 insertions(+), 56 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index 32eb388bee..8b1b3e1496 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -5,7 +5,7 @@ /// /// Serializes/Deserializes document to the SQL Database as bytes /// - public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer + public interface IContentNestedDataByteSerializer { ContentNestedData DeserializeBytes(byte[] data); byte[] SerializeBytes(ContentNestedData nestedData); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index 31f7763e04..d71f0e2184 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -68,7 +68,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource switch (map.CompressLevel) { - case NucachePropertyCompressionLevel.SQLDatabase: case NucachePropertyCompressionLevel.NuCacheDatabase: if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 206051f839..33181ebd5e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,23 +1,17 @@ -using K4os.Compression.LZ4; -using MessagePack; -using MessagePack.Formatters; +using MessagePack; using MessagePack.Resolvers; using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack /// - internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer + internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer, IContentNestedDataSerializer { - private MessagePackSerializerOptions _options; - private readonly NuCachePropertyOptions _propertyOptions; + private readonly MessagePackSerializerOptions _options; - public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) + public MsgPackContentNestedDataSerializer() { var defaultOptions = ContractlessStandardResolver.Options; @@ -37,7 +31,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? new NuCachePropertyOptions(); } public string ToJson(string serialized) @@ -56,45 +49,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { - Optimize(nestedData); - var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } - /// - /// Compress properties and map property names to shorter names - /// - /// - private void Optimize(ContentNestedData nestedData) - { - if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) - { - foreach (var map in _propertyOptions.PropertyMap) - { - if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) - { - if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) - { - foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) - { - property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); - } - } - } - - // if there is an alias map for this property then use that instead of the real property alias - // (used to save memory, the mapped alias is normally a single char or at least a smaller string) - if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.Remove(map.Key) - && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) - { - nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); - } - } - } - } - public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs index 2f24a203ca..8a82dcb888 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs @@ -7,14 +7,6 @@ { None = 0, - /// - /// Compress property data at the nucache SQL DB table level - /// - /// - /// Only necessary if the document in the nucache SQL DB table isn't stored as compressed bytes - /// - SQLDatabase = 1, - /// /// Compress property data at the nucache BTree level /// From 9ac2c271105fffc2de0046f42a233073dfc2184b Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 13 Aug 2020 23:09:40 +1000 Subject: [PATCH 024/147] Revert "removes the SQLDatabase enum value" and renames some stuff --- .../AppSettingsNucachePropertyMapFactory.cs | 13 +++-- .../IContentNestedDataSerializer.cs | 2 +- .../INucachePropertyOptionsFactory.cs | 2 +- .../Lz4DictionaryOfPropertyDataSerializer.cs | 3 +- .../MsgPackContentNestedDataSerializer.cs | 50 +++++++++++++++++-- .../NuCachePropertyCompressionOptions.cs | 32 ++++++++++++ .../NucachePropertyCompressionLevel.cs | 8 +++ .../DataSource/NucachePropertyOptions.cs | 17 ------- src/Umbraco.Web/Umbraco.Web.csproj | 2 +- 9 files changed, 97 insertions(+), 32 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs index 271695d250..544231bd32 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs @@ -10,14 +10,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // TODO: We'll remove this when the responsibility for compressing property data is at the property editor level internal class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory { - public NuCachePropertyOptions GetNuCachePropertyOptions() + public NuCachePropertyCompressionOptions GetNuCachePropertyOptions() { - NuCachePropertyOptions options = new NuCachePropertyOptions - { - PropertyMap = GetPropertyMap(), - LZ4CompressionLevel = K4os.Compression.LZ4.LZ4Level.L10_OPT, - MinimumCompressibleStringLength = null - }; + var options = new NuCachePropertyCompressionOptions( + GetPropertyMap(), + K4os.Compression.LZ4.LZ4Level.L10_OPT, + null); + return options; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index 8b1b3e1496..32eb388bee 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -5,7 +5,7 @@ /// /// Serializes/Deserializes document to the SQL Database as bytes /// - public interface IContentNestedDataByteSerializer + public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer { ContentNestedData DeserializeBytes(byte[] data); byte[] SerializeBytes(ContentNestedData nestedData); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs index 0cb694e1c1..d423499744 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs @@ -8,6 +8,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public interface INuCachePropertyOptionsFactory { - NuCachePropertyOptions GetNuCachePropertyOptions(); + NuCachePropertyCompressionOptions GetNuCachePropertyOptions(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index d71f0e2184..752cedc18b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -68,6 +68,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource switch (map.CompressLevel) { + case NucachePropertyCompressionLevel.SQLDatabase: case NucachePropertyCompressionLevel.NuCacheDatabase: if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) { @@ -149,7 +150,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } private static readonly NuCacheCompressionOptions DefaultMap = new NuCacheCompressionOptions(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); - private readonly NuCachePropertyOptions _nucachePropertyOptions; + private readonly NuCachePropertyCompressionOptions _nucachePropertyOptions; public NuCacheCompressionOptions GetSerializationMap(string propertyAlias) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 33181ebd5e..24e2bc7b27 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,17 +1,23 @@ -using MessagePack; +using K4os.Compression.LZ4; +using MessagePack; +using MessagePack.Formatters; using MessagePack.Resolvers; using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack /// - internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer, IContentNestedDataSerializer + internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { - private readonly MessagePackSerializerOptions _options; + private MessagePackSerializerOptions _options; + private readonly NuCachePropertyCompressionOptions _propertyOptions; - public MsgPackContentNestedDataSerializer() + public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -31,6 +37,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); + _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? NuCachePropertyCompressionOptions.Empty; } public string ToJson(string serialized) @@ -49,10 +56,45 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public string Serialize(ContentNestedData nestedData) { + Optimize(nestedData); + var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + /// + /// Compress properties and map property names to shorter names + /// + /// + private void Optimize(ContentNestedData nestedData) + { + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + { + foreach (var map in _propertyOptions.PropertyMap) + { + if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + { + if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) + { + foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) + { + property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); + } + } + } + + // if there is an alias map for this property then use that instead of the real property alias + // (used to save memory, the mapped alias is normally a single char or at least a smaller string) + if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) + && nestedData.PropertyData.Remove(map.Key) + && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) + { + nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); + } + } + } + } + public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs new file mode 100644 index 0000000000..50fb20cadc --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs @@ -0,0 +1,32 @@ +using K4os.Compression.LZ4; +using System; +using System.Collections.Generic; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + + public class NuCachePropertyCompressionOptions + { + /// + /// Returns empty options + /// + public static NuCachePropertyCompressionOptions Empty { get; } = new NuCachePropertyCompressionOptions(); + + private NuCachePropertyCompressionOptions() + { + } + + public NuCachePropertyCompressionOptions(IReadOnlyDictionary propertyMap, LZ4Level lZ4CompressionLevel, long? minimumCompressibleStringLength) + { + PropertyMap = propertyMap ?? throw new ArgumentNullException(nameof(propertyMap)); + LZ4CompressionLevel = lZ4CompressionLevel; + MinimumCompressibleStringLength = minimumCompressibleStringLength; + } + + public IReadOnlyDictionary PropertyMap { get; } = new Dictionary(); + + public LZ4Level LZ4CompressionLevel { get; } = LZ4Level.L00_FAST; + + public long? MinimumCompressibleStringLength { get; } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs index 8a82dcb888..2f24a203ca 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs @@ -7,6 +7,14 @@ { None = 0, + /// + /// Compress property data at the nucache SQL DB table level + /// + /// + /// Only necessary if the document in the nucache SQL DB table isn't stored as compressed bytes + /// + SQLDatabase = 1, + /// /// Compress property data at the nucache BTree level /// diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs deleted file mode 100644 index f88f30ccd8..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyOptions.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - - public class NuCachePropertyOptions - { - public IReadOnlyDictionary PropertyMap { get; set; } = new Dictionary(); - - public K4os.Compression.LZ4.LZ4Level LZ4CompressionLevel { get; set; } = K4os.Compression.LZ4.LZ4Level.L00_FAST; - - public long? MinimumCompressibleStringLength { get; set; } - } -} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 5715a5e060..fd54f550da 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -262,7 +262,7 @@ - + From 7d689a6e1133628b700ee9b9ee5432e75542c9a8 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 13 Aug 2020 23:32:05 +1000 Subject: [PATCH 025/147] adds notes, ensures that we optimize property data when using msgpack binary serialization too! --- .../DataSource/Lz4DictionaryOfPropertyDataSerializer.cs | 6 +++--- .../DataSource/MsgPackContentNestedDataSerializer.cs | 7 ++++++- .../PublishedCache/NuCache/DataSource/SerializerBase.cs | 4 ++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index 752cedc18b..b3d7f7dcbc 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -67,7 +67,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Value = ReadObject(stream); switch (map.CompressLevel) - { + { + // If the property is compressed at either the DB or Nucache level, it means it's compressed here and we need to decompress case NucachePropertyCompressionLevel.SQLDatabase: case NucachePropertyCompressionLevel.NuCacheDatabase: if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) @@ -80,8 +81,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource break; case NucachePropertyDecompressionLevel.NotCompressed: //Shouldn't be any not compressed - // TODO: Do we need to throw here? - break; + throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {alias} since it's compresion option is {map.CompressLevel}"); case NucachePropertyDecompressionLevel.Immediate: default: pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 24e2bc7b27..09ca1278b4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -97,7 +97,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); - public byte[] SerializeBytes(ContentNestedData nestedData) => MessagePackSerializer.Serialize(nestedData, _options); + public byte[] SerializeBytes(ContentNestedData nestedData) + { + Optimize(nestedData); + + return MessagePackSerializer.Serialize(nestedData, _options); + } //private class ContentNestedDataResolver : IFormatterResolver //{ diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 885b5cf80e..4a11a8f0e5 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -16,6 +16,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixDouble = 'B'; private const char PrefixDateTime = 'D'; private const char PrefixByte = 'O'; + + // TODO: It might make sense to have another prefix for an LZ4 compressed byte array. + // Would be an improvement for the SQLDatabase compression option because then you could mix compressed and decompressed properties with the same alias. + // For example, don't compress recent content, but compress older content. private const char PrefixByteArray = 'A'; protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); From c619e5a96a1011c5c78a51ba9749297259dabead Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 14 Aug 2020 00:45:58 +1000 Subject: [PATCH 026/147] Fixes the property map, adds notes, updates tests data to include larger paragraphs --- src/Umbraco.TestData/UmbracoTestDataController.cs | 6 ++++-- .../DataSource/MsgPackContentNestedDataSerializer.cs | 2 +- .../DataSource/NucachePropertyCompressionLevel.cs | 10 +++++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.TestData/UmbracoTestDataController.cs b/src/Umbraco.TestData/UmbracoTestDataController.cs index 02949d5345..35a578d5d9 100644 --- a/src/Umbraco.TestData/UmbracoTestDataController.cs +++ b/src/Umbraco.TestData/UmbracoTestDataController.cs @@ -208,7 +208,8 @@ namespace Umbraco.TestData var docType = GetOrCreateContentType(); var parent = Services.ContentService.Create(company, -1, docType.Alias); - parent.SetValue("review", faker.Rant.Review()); + // give it some reasonable data (100 reviews) + parent.SetValue("review", string.Join(" ", Enumerable.Range(0, 100).Select(x => faker.Rant.Review()))); parent.SetValue("desc", company); parent.SetValue("media", imageIds[random.Next(0, imageIds.Count - 1)]); Services.ContentService.Save(parent); @@ -218,7 +219,8 @@ namespace Umbraco.TestData return CreateHierarchy(parent, count, depth, currParent => { var content = Services.ContentService.Create(faker.Commerce.ProductName(), currParent, docType.Alias); - content.SetValue("review", faker.Rant.Review()); + // give it some reasonable data (100 reviews) + content.SetValue("review", string.Join(" ", Enumerable.Range(0, 100).Select(x => faker.Rant.Review()))); content.SetValue("desc", string.Join(", ", Enumerable.Range(0, 5).Select(x => faker.Commerce.ProductAdjective()))); content.SetValue("media", imageIds[random.Next(0, imageIds.Count - 1)]); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 09ca1278b4..eb17bed858 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -86,9 +86,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // if there is an alias map for this property then use that instead of the real property alias // (used to save memory, the mapped alias is normally a single char or at least a smaller string) if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.Remove(map.Key) && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) { + nestedData.PropertyData.Remove(map.Key); nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs index 2f24a203ca..23826fd722 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs @@ -11,13 +11,21 @@ /// Compress property data at the nucache SQL DB table level /// /// - /// Only necessary if the document in the nucache SQL DB table isn't stored as compressed bytes + /// Idea being we only compress this once. + /// All the records in cmsContentNu need to be rebuilt when this gets enabled. + /// Good option as then we don't use up memory / cpu to compress at boot. /// SQLDatabase = 1, /// /// Compress property data at the nucache BTree level /// + /// + /// Compress the property when writing to nucache bplustree after reading from the database. + /// Idea being we compress this at rebuild / boot. + /// This option supports older items not being compressed already, at the expense of doing this compression at boot. + /// But it also means you can easily switch between None and NuCacheDatabase if performance is worse. + /// NuCacheDatabase = 2 } } From 23880b596f56257c8eb9fac63b974dce01493b61 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 14 Aug 2020 10:02:22 +1200 Subject: [PATCH 027/147] Fix formatting --- .../Persistence/SqlSyntax/SqlServerSyntaxProvider.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs index 6c53a94f90..db199ad9dc 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs @@ -349,8 +349,8 @@ where tbl.[name]=@0 and col.[name]=@1;", tableName, columnName) : GetQuotedColumnName(index.ColumnName); var includeColumns = index.IncludeColumns.Any() - ? " INCLUDE " + string.Join(",", index.IncludeColumns.Select(x => GetQuotedColumnName(x.Name))) - : GetQuotedColumnName(index.ColumnName); + ? $" INCLUDE ({string.Join(",", index.IncludeColumns.Select(x => GetQuotedColumnName(x.Name)))})" + : string.Empty; return string.Format(CreateIndex, GetIndexType(index.IndexType), " ", GetQuotedName(name), GetQuotedTableName(index.TableName), columns, includeColumns); From 7379846597fe012cd43815ab825fecf8c18e235e Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 14 Aug 2020 18:03:51 +1200 Subject: [PATCH 028/147] SQL CE no support for includes. Add null check. --- .../Persistence/SqlSyntax/SqlCeSyntaxProvider.cs | 14 ++++++++++++++ .../SqlSyntax/SqlServerSyntaxProvider.cs | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs index 8a42caaa03..3ab4b884b6 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs @@ -236,6 +236,20 @@ where table_name=@0 and column_name=@1", tableName, columnName).FirstOrDefault() } public override string DropIndex { get { return "DROP INDEX {1}.{0}"; } } + public override string CreateIndex => "CREATE {0}{1}INDEX {2} ON {3} ({4})"; + public override string Format(IndexDefinition index) + { + var name = string.IsNullOrEmpty(index.Name) + ? $"IX_{index.TableName}_{index.ColumnName}" + : index.Name; + var columns = index.Columns.Any() + ? string.Join(",", index.Columns.Select(x => GetQuotedColumnName(x.Name))) + : GetQuotedColumnName(index.ColumnName); + + + return string.Format(CreateIndex, GetIndexType(index.IndexType), " ", GetQuotedName(name), + GetQuotedTableName(index.TableName), columns); + } } } diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs index db199ad9dc..772d9e89c5 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlServerSyntaxProvider.cs @@ -348,7 +348,7 @@ where tbl.[name]=@0 and col.[name]=@1;", tableName, columnName) ? string.Join(",", index.Columns.Select(x => GetQuotedColumnName(x.Name))) : GetQuotedColumnName(index.ColumnName); - var includeColumns = index.IncludeColumns.Any() + var includeColumns = index.IncludeColumns?.Any() ?? false ? $" INCLUDE ({string.Join(",", index.IncludeColumns.Select(x => GetQuotedColumnName(x.Name)))})" : string.Empty; From 6f2dccc2e2d4278fcf748986f0f7beefffbd95a3 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 10:14:36 +1000 Subject: [PATCH 029/147] Uses a more optimized COUNT query when rebuilding the in memory cache --- src/Umbraco.Core/Constants-SqlTemplates.cs | 6 +++ .../Persistence/NPocoDatabaseExtensions.cs | 23 +++++++-- .../NuCache/DataSource/DatabaseDataSource.cs | 47 ++++++++++++++++--- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Core/Constants-SqlTemplates.cs b/src/Umbraco.Core/Constants-SqlTemplates.cs index 984bc495b0..6940539cb6 100644 --- a/src/Umbraco.Core/Constants-SqlTemplates.cs +++ b/src/Umbraco.Core/Constants-SqlTemplates.cs @@ -15,6 +15,12 @@ public const string GetReservedId = "Umbraco.Core.VersionableRepository.GetReservedId"; } + + internal static class NuCacheDatabaseDataSource + { + public const string ContentSourcesSelect1 = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelect1"; + public const string ContentSourcesCount = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesCount"; + } } } } diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs index 152dcbe6d3..c2100d97ad 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs @@ -27,12 +27,13 @@ namespace Umbraco.Core.Persistence /// The number of rows to load per page /// /// + /// Specify a custom Sql command to get the total count, if null is specified than the auto-generated sql count will be used /// /// /// NPoco's normal Page returns a List{T} but sometimes we don't want all that in memory and instead want to /// iterate over each row with a reader using Query vs Fetch. /// - internal static IEnumerable QueryPaged(this IDatabase database, long pageSize, Sql sql) + internal static IEnumerable QueryPaged(this IDatabase database, long pageSize, Sql sql, Sql sqlCount) { var sqlString = sql.SQL; var sqlArgs = sql.Arguments; @@ -42,12 +43,12 @@ namespace Umbraco.Core.Persistence do { // Get the paged queries - database.BuildPageQueries(pageIndex * pageSize, pageSize, sqlString, ref sqlArgs, out var sqlCount, out var sqlPage); + database.BuildPageQueries(pageIndex * pageSize, pageSize, sqlString, ref sqlArgs, out var generatedSqlCount, out var sqlPage); // get the item count once if (itemCount == null) { - itemCount = database.ExecuteScalar(sqlCount, sqlArgs); + itemCount = database.ExecuteScalar(sqlCount?.SQL ?? generatedSqlCount, sqlCount?.Arguments ?? sqlArgs); } pageIndex++; @@ -60,6 +61,22 @@ namespace Umbraco.Core.Persistence } while ((pageIndex * pageSize) < itemCount); } + /// + /// Iterates over the result of a paged data set with a db reader + /// + /// + /// + /// + /// The number of rows to load per page + /// + /// + /// + /// + /// NPoco's normal Page returns a List{T} but sometimes we don't want all that in memory and instead want to + /// iterate over each row with a reader using Query vs Fetch. + /// + internal static IEnumerable QueryPaged(this IDatabase database, long pageSize, Sql sql) => database.QueryPaged(pageSize, sql, null); + // NOTE // // proper way to do it with TSQL and SQLCE diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 19998c7956..77cb82b226 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -32,9 +32,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private Sql ContentSourcesSelect(IScope scope, Func, Sql> joins = null) { - var sql = scope.SqlContext.Sql() - - .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), + var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelect1, tsql => + tsql.Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) @@ -52,7 +51,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .AndSelect("nuEdit", x => Alias(x.RawData, "EditDataRaw")) .AndSelect("nuPub", x => Alias(x.RawData, "PubDataRaw")) - .From(); + .From()); + + var sql = sqlTemplate.Sql(); + + // TODO: I'm unsure how we can format the below into SQL templates also because right.Current and right.Published end up being parameters if (joins != null) sql = joins(sql); @@ -74,6 +77,32 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } + /// + /// Returns a slightly more optimized query to use for the document counting when paging over the content sources + /// + /// + /// + private Sql ContentSourcesCount(IScope scope) + { + var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesCount, tsql => + tsql.Select(x => Alias(x.NodeId, "Id")) + .From() + .InnerJoin().On((left, right) => left.NodeId == right.NodeId) + .InnerJoin().On((left, right) => left.NodeId == right.NodeId)); + + var sql = sqlTemplate.Sql(); + + // TODO: We can't use a template with this one because of the 'right.Current' and 'right.Published' ends up being a parameter so not sure how we can do that + sql = sql + .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current) + .InnerJoin().On((left, right) => left.Id == right.Id) + .LeftJoin(j => + j.InnerJoin("pdver").On((left, right) => left.Id == right.Id && right.Published, "pcver", "pdver"), "pcver") + .On((left, right) => left.NodeId == right.NodeId, aliasRight: "pcver"); + + return sql; + } + public ContentNodeKit GetContentSource(IScope scope, int id) { var sql = ContentSourcesSelect(scope) @@ -86,14 +115,20 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetAllContentSources(IScope scope) { + // Create a different query for the SQL vs the COUNT Sql since the auto-generated COUNT Sql will be inneficient var sql = ContentSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + // create a more efficient COUNT query without the join on the cmsContentNu table + var sqlCountQuery = ContentSourcesCount(scope) + .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed); + var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) yield return CreateContentNodeKit(row); } @@ -319,6 +354,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return s; } - + } } From d84d305ae9abb864d8b6ab8efb928f2c458d1351 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 10:38:49 +1000 Subject: [PATCH 030/147] Changes our Expression visitor to not output NOT for boolean values --- .../Persistence/Querying/ExpressionVisitorBase.cs | 7 +++++-- .../NuCache/DataSource/DatabaseDataSource.cs | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs index d04930fa92..4316023b03 100644 --- a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs +++ b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs @@ -328,8 +328,11 @@ namespace Umbraco.Core.Persistence.Querying { case ExpressionType.MemberAccess: // false property , i.e. x => !Trashed - SqlParameters.Add(true); - return Visited ? string.Empty : $"NOT ({o} = @{SqlParameters.Count - 1})"; + // BUT we don't want to do a NOT SQL statement since this generally results in indexes not being used + // so we want to do an == false + SqlParameters.Add(false); + return Visited ? string.Empty : $"({o} = @{SqlParameters.Count - 1})"; + //return Visited ? string.Empty : $"NOT ({o} = @{SqlParameters.Count - 1})"; default: // could be anything else, such as: x => !x.Path.StartsWith("-20") return Visited ? string.Empty : string.Concat("NOT (", o, ")"); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 77cb82b226..3cad7171be 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -68,11 +68,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .InnerJoin().On((left, right) => left.Id == right.Id) .LeftJoin(j => - j.InnerJoin("pdver").On((left, right) => left.Id == right.Id && right.Published, "pcver", "pdver"), "pcver") + j.InnerJoin("pdver").On((left, right) => left.Id == right.Id && right.Published == true, "pcver", "pdver"), "pcver") .On((left, right) => left.NodeId == right.NodeId, aliasRight: "pcver") - .LeftJoin("nuEdit").On((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit") - .LeftJoin("nuPub").On((left, right) => left.NodeId == right.NodeId && right.Published, aliasRight: "nuPub"); + .LeftJoin("nuEdit").On((left, right) => left.NodeId == right.NodeId && right.Published == false, aliasRight: "nuEdit") + .LeftJoin("nuPub").On((left, right) => left.NodeId == right.NodeId && right.Published == true, aliasRight: "nuPub"); return sql; } From 77e5a41077ea85ed1b8d9a9c2ace687275741d1e Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 12:26:24 +1000 Subject: [PATCH 031/147] Updates indexes on umbracoNode and umbracoContentVersion which uses INCLUDE columns --- .../Migrations/Upgrade/UmbracoPlan.cs | 6 +- .../Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs | 56 +++++++++++++++++++ .../IndexDefinition.cs | 9 +-- .../Persistence/Dtos/ContentVersionDto.cs | 2 +- src/Umbraco.Core/Persistence/Dtos/NodeDto.cs | 4 +- src/Umbraco.Core/Umbraco.Core.csproj | 1 + 6 files changed, 66 insertions(+), 12 deletions(-) create mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index 0557312e90..a15dcfb23b 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -7,6 +7,7 @@ using Umbraco.Core.Migrations.Upgrade.V_8_0_0; using Umbraco.Core.Migrations.Upgrade.V_8_0_1; using Umbraco.Core.Migrations.Upgrade.V_8_1_0; using Umbraco.Core.Migrations.Upgrade.V_8_6_0; +using Umbraco.Core.Migrations.Upgrade.V_8_8_0; namespace Umbraco.Core.Migrations.Upgrade { @@ -194,7 +195,10 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.7.0... To("{a78e3369-8ea3-40ec-ad3f-5f76929d2b20}"); To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); - + + // to 8.8.0... + To("{4695D0C9-0729-4976-985B-048D503665D8}"); + //FINAL } } diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs new file mode 100644 index 0000000000..fb9e6e7f23 --- /dev/null +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs @@ -0,0 +1,56 @@ +using System.Linq; +using Umbraco.Core.Migrations.Expressions.Execute.Expressions; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; +using Umbraco.Core.Persistence.Dtos; +using Umbraco.Core.Persistence.SqlSyntax; +using Umbraco.Core.PropertyEditors; + +namespace Umbraco.Core.Migrations.Upgrade.V_8_8_0 +{ + public class UpgradedIncludeIndexes : MigrationBase + { + public UpgradedIncludeIndexes(IMigrationContext context) + : base(context) + { + + } + + public override void Migrate() + { + // Rebuild keys and indexes for these tables, unfortunately we cannot use the Delete.KeysAndIndexes + // procedure since for some reason that tries to drop the PK and we don't want that and it would be a breaking + // change to add another parameter to that method so we'll just manually do it. + + var nodeDtoObjectTypeIndex = $"IX_{NodeDto.TableName}_ObjectType"; // this is the one we'll rebuild + // delete existing ones + DeleteIndexes($"IX_{NodeDto.TableName}_ParentId", $"IX_{NodeDto.TableName}_Trashed", nodeDtoObjectTypeIndex); + CreateIndexes(nodeDtoObjectTypeIndex); + + var contentVersionNodeIdIndex = $"IX_{ContentVersionDto.TableName}_NodeId"; + // delete existing ones + DeleteIndexes(contentVersionNodeIdIndex); + CreateIndexes(contentVersionNodeIdIndex); + } + + private void DeleteIndexes(params string[] toDelete) + { + var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); + + foreach (var i in toDelete) + Delete.Index(i).OnTable(tableDef.Name).Do(); + } + + private void CreateIndexes(params string[] toCreate) + { + var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); + + foreach(var c in toCreate) + { + // get the definition by name + var index = tableDef.Indexes.First(x => x.Name == c); + new ExecuteSqlStatementExpression(Context) { SqlStatement = Context.SqlContext.SqlSyntax.Format(index) }.Execute(); + } + + } + } +} diff --git a/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs b/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs index 8efb795948..20f75d38c8 100644 --- a/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs +++ b/src/Umbraco.Core/Persistence/DatabaseModelDefinitions/IndexDefinition.cs @@ -6,18 +6,13 @@ namespace Umbraco.Core.Persistence.DatabaseModelDefinitions { public class IndexDefinition { - public IndexDefinition() - { - Columns = new List(); - } - public virtual string Name { get; set; } public virtual string SchemaName { get; set; } public virtual string TableName { get; set; } public virtual string ColumnName { get; set; } - public virtual ICollection Columns { get; set; } - public virtual ICollection IncludeColumns { get; set; } + public virtual ICollection Columns { get; set; } = new List(); + public virtual ICollection IncludeColumns { get; set; } = new List(); public IndexTypes IndexType { get; set; } } } diff --git a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs index 4b203c128f..020b4a9e04 100644 --- a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs @@ -19,7 +19,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("nodeId")] [ForeignKey(typeof(ContentDto))] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_NodeId", ForColumns = "nodeId,current")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_NodeId", ForColumns = "nodeId,current", IncludeColumns = "id,versionDate,text,userId")] public int NodeId { get; set; } [Column("versionDate")] // TODO: db rename to 'updateDate' diff --git a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs index 5800efb97a..3fd9a9f1d4 100644 --- a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs @@ -26,7 +26,6 @@ namespace Umbraco.Core.Persistence.Dtos [Column("parentId")] [ForeignKey(typeof(NodeDto))] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ParentId")] public int ParentId { get; set; } [Column("level")] @@ -42,7 +41,6 @@ namespace Umbraco.Core.Persistence.Dtos [Column("trashed")] [Constraint(Default = "0")] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_Trashed")] public bool Trashed { get; set; } [Column("nodeUser")] // TODO: db rename to 'createUserId' @@ -56,7 +54,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("nodeObjectType")] // TODO: db rename to 'objectType' [NullSetting(NullSetting = NullSettings.Null)] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType", ForColumns = "level,parentId,sortOrder,nodeObjectType,trashed", IncludeColumns = "nodeUser,path,uniqueId,createDate")] public Guid? NodeObjectType { get; set; } [Column("createDate")] diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 7756fd06f5..7bfbecc84d 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -133,6 +133,7 @@ + From aec828098d5e4086e1e1d54257a6801e8dd4f760 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 12:33:47 +1000 Subject: [PATCH 032/147] adds MessagePack to nuspec --- build/NuSpecs/UmbracoCms.Web.nuspec | 1 + 1 file changed, 1 insertion(+) diff --git a/build/NuSpecs/UmbracoCms.Web.nuspec b/build/NuSpecs/UmbracoCms.Web.nuspec index 72619db02e..d45b65af09 100644 --- a/build/NuSpecs/UmbracoCms.Web.nuspec +++ b/build/NuSpecs/UmbracoCms.Web.nuspec @@ -42,6 +42,7 @@ + From a078a30990154f96d27955dc566acb32f49c5451 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 14:19:44 +1000 Subject: [PATCH 033/147] Re-adds the indexes removed --- .../Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs | 19 +++++-------------- src/Umbraco.Core/Persistence/Dtos/NodeDto.cs | 8 ++++++-- .../PublishedCache/NuCache/NuCacheComposer.cs | 2 ++ 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs index fb9e6e7f23..50c0a48c91 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs @@ -2,8 +2,6 @@ using Umbraco.Core.Migrations.Expressions.Execute.Expressions; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Dtos; -using Umbraco.Core.Persistence.SqlSyntax; -using Umbraco.Core.PropertyEditors; namespace Umbraco.Core.Migrations.Upgrade.V_8_8_0 { @@ -16,20 +14,13 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_8_0 } public override void Migrate() - { - // Rebuild keys and indexes for these tables, unfortunately we cannot use the Delete.KeysAndIndexes - // procedure since for some reason that tries to drop the PK and we don't want that and it would be a breaking - // change to add another parameter to that method so we'll just manually do it. - - var nodeDtoObjectTypeIndex = $"IX_{NodeDto.TableName}_ObjectType"; // this is the one we'll rebuild - // delete existing ones - DeleteIndexes($"IX_{NodeDto.TableName}_ParentId", $"IX_{NodeDto.TableName}_Trashed", nodeDtoObjectTypeIndex); - CreateIndexes(nodeDtoObjectTypeIndex); + { + var nodeDtoLevelIndex = $"IX_{NodeDto.TableName}_Level"; + CreateIndexes(nodeDtoLevelIndex); // add the new definition var contentVersionNodeIdIndex = $"IX_{ContentVersionDto.TableName}_NodeId"; - // delete existing ones - DeleteIndexes(contentVersionNodeIdIndex); - CreateIndexes(contentVersionNodeIdIndex); + DeleteIndexes(contentVersionNodeIdIndex); // delete existing ones + CreateIndexes(contentVersionNodeIdIndex); // add the updated definition } private void DeleteIndexes(params string[] toDelete) diff --git a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs index 3fd9a9f1d4..6797cd2c98 100644 --- a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs @@ -26,9 +26,12 @@ namespace Umbraco.Core.Persistence.Dtos [Column("parentId")] [ForeignKey(typeof(NodeDto))] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ParentId")] public int ParentId { get; set; } + // NOTE: This index is primarily for the nucache data lookup, see https://github.com/umbraco/Umbraco-CMS/pull/8365#issuecomment-673404177 [Column("level")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_Level", ForColumns = "level,parentId,sortOrder,nodeObjectType,trashed", IncludeColumns = "nodeUser,path,uniqueId,createDate")] public short Level { get; set; } [Column("path")] @@ -41,6 +44,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("trashed")] [Constraint(Default = "0")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_Trashed")] public bool Trashed { get; set; } [Column("nodeUser")] // TODO: db rename to 'createUserId' @@ -53,8 +57,8 @@ namespace Umbraco.Core.Persistence.Dtos public string Text { get; set; } [Column("nodeObjectType")] // TODO: db rename to 'objectType' - [NullSetting(NullSetting = NullSettings.Null)] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType", ForColumns = "level,parentId,sortOrder,nodeObjectType,trashed", IncludeColumns = "nodeUser,path,uniqueId,createDate")] + [NullSetting(NullSetting = NullSettings.Null)] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType")] public Guid? NodeObjectType { get; set; } [Column("createDate")] diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 1ee6b96eb1..c098b516a0 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -16,6 +16,8 @@ namespace Umbraco.Web.PublishedCache.NuCache var serializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.Serializer"]; composition.Register(); composition.Register(); + + // TODO: Based on our findings it seems like this should not be configurable, we should just be using this because it's better if (serializer == "MsgPack") { var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; From 68c4e4f36b1cbbb6fd5b4f190e6b9f86c77a7108 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 14:49:13 +1000 Subject: [PATCH 034/147] ObjectType and UniqueId indexes now cover all fields since anytime these are queried we return all data. --- .../Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs | 12 ++++++++---- src/Umbraco.Core/Persistence/Dtos/NodeDto.cs | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs index 50c0a48c91..11de5bdf32 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs @@ -14,13 +14,17 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_8_0 } public override void Migrate() - { - var nodeDtoLevelIndex = $"IX_{NodeDto.TableName}_Level"; - CreateIndexes(nodeDtoLevelIndex); // add the new definition + { + var indexesToReplace = new[] { $"IX_{NodeDto.TableName}_UniqueId", $"IX_{NodeDto.TableName}_ObjectType" }; + DeleteIndexes(indexesToReplace); // delete existing ones + // add the new definitions + CreateIndexes($"IX_{NodeDto.TableName}_Level"); + CreateIndexes(indexesToReplace); + var contentVersionNodeIdIndex = $"IX_{ContentVersionDto.TableName}_NodeId"; DeleteIndexes(contentVersionNodeIdIndex); // delete existing ones - CreateIndexes(contentVersionNodeIdIndex); // add the updated definition + CreateIndexes(contentVersionNodeIdIndex); // add the updated definitions } private void DeleteIndexes(params string[] toDelete) diff --git a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs index 6797cd2c98..62475af833 100644 --- a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs @@ -20,7 +20,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("uniqueId")] [NullSetting(NullSetting = NullSettings.NotNull)] - [Index(IndexTypes.UniqueNonClustered, Name = "IX_" + TableName + "_UniqueId")] + [Index(IndexTypes.UniqueNonClustered, Name = "IX_" + TableName + "_UniqueId", IncludeColumns = "parentId,level,path,sortOrder,trashed,nodeUser,text,createDate")] [Constraint(Default = SystemMethods.NewGuid)] public Guid UniqueId { get; set; } @@ -58,7 +58,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("nodeObjectType")] // TODO: db rename to 'objectType' [NullSetting(NullSetting = NullSettings.Null)] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType", IncludeColumns = "parentId,level,path,sortOrder,trashed,nodeUser,text,createDate")] public Guid? NodeObjectType { get; set; } [Column("createDate")] From 8ab5899db2c860752cc73f5fb3f1a85120988aad Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 15:18:23 +1000 Subject: [PATCH 035/147] Adds another required DB index --- .../Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs | 13 ++++++------- .../Persistence/Dtos/ContentVersionDto.cs | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs index 11de5bdf32..d4dd783876 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs @@ -16,15 +16,14 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_8_0 public override void Migrate() { var indexesToReplace = new[] { $"IX_{NodeDto.TableName}_UniqueId", $"IX_{NodeDto.TableName}_ObjectType" }; - DeleteIndexes(indexesToReplace); // delete existing ones - // add the new definitions - CreateIndexes($"IX_{NodeDto.TableName}_Level"); - CreateIndexes(indexesToReplace); - + DeleteIndexes(indexesToReplace); // delete existing ones + CreateIndexes(indexesToReplace); // replace + CreateIndexes($"IX_{NodeDto.TableName}_Level"); // add the new definitions var contentVersionNodeIdIndex = $"IX_{ContentVersionDto.TableName}_NodeId"; - DeleteIndexes(contentVersionNodeIdIndex); // delete existing ones - CreateIndexes(contentVersionNodeIdIndex); // add the updated definitions + DeleteIndexes(contentVersionNodeIdIndex); // delete existing ones + CreateIndexes(contentVersionNodeIdIndex); // replace + CreateIndexes($"IX_{ContentVersionDto.TableName}_Current"); // add the new definitions } private void DeleteIndexes(params string[] toDelete) diff --git a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs index 020b4a9e04..f9bf283be9 100644 --- a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs @@ -32,6 +32,7 @@ namespace Umbraco.Core.Persistence.Dtos public int? UserId { get => _userId == 0 ? null : _userId; set => _userId = value; } //return null if zero [Column("current")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_Current", IncludeColumns = "nodeId")] public bool Current { get; set; } // about current: From ba8da3850ede43d0c2706fa556f9d52aa403ff83 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Aug 2020 11:43:43 +1000 Subject: [PATCH 036/147] This gets things working with all compression levels - going to cleanup/simplify --- .../DataSource/LazyCompressedString.cs | 14 ++-- .../Lz4DictionaryOfPropertyDataSerializer.cs | 6 +- .../MsgPackContentNestedDataSerializer.cs | 76 ++++++++++++++++--- .../NuCachePropertyCompressionOptions.cs | 1 + .../NucachePropertyDecompressionLevel.cs | 3 + .../NuCache/DataSource/SerializerBase.cs | 9 ++- 6 files changed, 88 insertions(+), 21 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 55469ad791..9df40daf76 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -1,10 +1,7 @@ using K4os.Compression.LZ4; using System; -using System.Collections.Generic; -using System.Linq; using System.Text; using System.Threading; -using System.Threading.Tasks; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { @@ -13,10 +10,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// internal class LazyCompressedString { - // TODO: This could be a struct - - private byte[] _bytes; private string _str; + private byte[] _bytes; /// /// Constructor @@ -27,6 +22,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _bytes = bytes; } + public byte[] GetBytes() + { + if (_bytes == null) + throw new InvalidOperationException("The bytes have already been expanded"); + return _bytes; + } + public override string ToString() { return LazyInitializer.EnsureInitialized(ref _str, () => diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs index b3d7f7dcbc..1a2d26f4b1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs @@ -88,8 +88,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource break; } } - break; - } + break; + } } dict[key] = pdatas.ToArray(); @@ -122,7 +122,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource WriteObject(pdata.Culture ?? string.Empty, stream); WriteObject(pdata.Segment ?? string.Empty, stream); - //Only compress strings + //Only compress strings (if NucachePropertyCompressionLevel.SQLDatabase has been specified then the property value will already be compressed) switch (map.CompressLevel) { // If we're compressing into btree at the property level diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index eb17bed858..0475810422 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -50,23 +50,36 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public ContentNestedData Deserialize(string data) { var bin = Convert.FromBase64String(data); - var obj = MessagePackSerializer.Deserialize(bin, _options); - return obj; + var nestedData = MessagePackSerializer.Deserialize(bin, _options); + Expand(nestedData); + return nestedData; } public string Serialize(ContentNestedData nestedData) { - Optimize(nestedData); - + Compress(nestedData); var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } + public ContentNestedData DeserializeBytes(byte[] data) + { + var nestedData = MessagePackSerializer.Deserialize(data, _options); + Expand(nestedData); + return nestedData; + } + + public byte[] SerializeBytes(ContentNestedData nestedData) + { + Compress(nestedData); + return MessagePackSerializer.Serialize(nestedData, _options); + } + /// - /// Compress properties and map property names to shorter names + /// Used during serialization to compress properties and map property names to shorter names /// /// - private void Optimize(ContentNestedData nestedData) + private void Compress(ContentNestedData nestedData) { if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) { @@ -95,13 +108,54 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentNestedData DeserializeBytes(byte[] data) => MessagePackSerializer.Deserialize(data, _options); - - public byte[] SerializeBytes(ContentNestedData nestedData) + /// + /// Used during deserialization to map the property data as lazy or expand the value and re-map back to the true property aliases + /// + /// + private void Expand(ContentNestedData nestedData) { - Optimize(nestedData); + if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + { + foreach (var map in _propertyOptions.PropertyMap) + { + if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + { + // if there is an alias map for this property then re-map to the real property alias + if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) + && nestedData.PropertyData.TryGetValue(map.Value.MappedAlias, out PropertyData[] properties2)) + { + nestedData.PropertyData.Remove(map.Value.MappedAlias); + nestedData.PropertyData.Add(map.Key, properties2); + } - return MessagePackSerializer.Serialize(nestedData, _options); + if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) + { + foreach (var pdata in properties) + { + if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) + { + //Compressed string + switch (map.Value.DecompressLevel) + { + case NucachePropertyDecompressionLevel.Lazy: + pdata.Value = new LazyCompressedString(byteArrayValue); + break; + case NucachePropertyDecompressionLevel.NotCompressed: + //Shouldn't be any not compressed + throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {map.Key} since it's compresion option is {map.Value.CompressLevel}"); + case NucachePropertyDecompressionLevel.Immediate: + default: + pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); + break; + } + } + } + } + } + + + } + } } //private class ContentNestedDataResolver : IFormatterResolver diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs index 50fb20cadc..55ab813783 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs @@ -27,6 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public LZ4Level LZ4CompressionLevel { get; } = LZ4Level.L00_FAST; + // TODO: Unsure if we really want to keep this public long? MinimumCompressibleStringLength { get; } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs index 335ae6ff4a..f4d485be71 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs @@ -6,7 +6,10 @@ public enum NucachePropertyDecompressionLevel { NotCompressed = 0, + + // TODO: I'm unsure if this will ever be necessary, lazy seems good and deserialization would only occur once Immediate = 1, + Lazy = 2 } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 4a11a8f0e5..b90c418750 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -98,7 +98,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); case PrefixByteArray: - return PrimitiveSerializer.Bytes.ReadFrom(stream); + // When it's a byte array always return as a LazyCompressedString + // TODO: Else we need to make a different prefix for lazy vs eager loading + return new LazyCompressedString(PrimitiveSerializer.Bytes.ReadFrom(stream)); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); } @@ -160,6 +162,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); PrimitiveSerializer.Bytes.WriteTo(byteArrayValue, stream); } + else if (value is LazyCompressedString lazyCompressedString) + { + PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); + PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); + } else throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized."); } From 9a06b6291ae0a636108727cc4af6fe1f28009d3c Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Aug 2020 15:57:13 +1000 Subject: [PATCH 037/147] Simplifies compression implementation, adds compression options to target all complex editors, ensures the options are per document type/property type --- .../ContentSerializationTests.cs | 8 +- ...ComplexEditorPropertyCompressionOptions.cs | 48 +++++ .../IPropertyCompressionOptions.cs | 10 + .../NoopPropertyCompressionOptions.cs | 10 + .../AppSettingsNucachePropertyMapFactory.cs | 47 ----- .../NuCache/DataSource/DatabaseDataSource.cs | 12 +- .../IContentNestedDataSerializer.cs | 8 +- .../INucachePropertyOptionsFactory.cs | 13 -- .../JsonContentNestedDataSerializer.cs | 4 +- .../DataSource/LazyCompressedString.cs | 22 ++- .../Lz4DictionaryOfPropertyDataSerializer.cs | 181 ------------------ .../MsgPackContentNestedDataSerializer.cs | 94 +++------ .../DataSource/NuCacheCompressionOptions.cs | 57 ------ .../NuCachePropertyCompressionOptions.cs | 33 ---- .../NucachePropertyCompressionLevel.cs | 31 --- .../NucachePropertyDecompressionLevel.cs | 15 -- .../NuCache/DataSource/SerializerBase.cs | 40 ++-- .../PublishedCache/NuCache/NuCacheComposer.cs | 33 +--- .../NuCache/PublishedSnapshotService.cs | 4 +- src/Umbraco.Web/Umbraco.Web.csproj | 10 +- 20 files changed, 155 insertions(+), 525 deletions(-) create mode 100644 src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs create mode 100644 src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs create mode 100644 src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index debfddef98..c85973f4b0 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -55,14 +55,14 @@ namespace Umbraco.Tests.PublishedContent UrlSegment = "home" }; - var json = jsonSerializer.Serialize(content); - var msgPack = msgPackSerializer.Serialize(content); + var json = jsonSerializer.Serialize(1, content); + var msgPack = msgPackSerializer.Serialize(1, content); Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(json); - var msgPackContent = msgPackSerializer.Deserialize(msgPack); + var jsonContent = jsonSerializer.Deserialize(1, json); + var msgPackContent = msgPackSerializer.Deserialize(1, msgPack); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs new file mode 100644 index 0000000000..f4776f652d --- /dev/null +++ b/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs @@ -0,0 +1,48 @@ +using K4os.Compression.LZ4; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Services; + +namespace Umbraco.Web.PropertyEditors +{ + + /// + /// Ensures all property types that have an editor storing a complex value are compressed + /// + public class ComplexEditorPropertyCompressionOptions : IPropertyCompressionOptions + { + private readonly IContentTypeService _contentTypeService; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); + + public ComplexEditorPropertyCompressionOptions(IContentTypeService contentTypeService, PropertyEditorCollection propertyEditors) + { + _contentTypeService = contentTypeService; + _propertyEditors = propertyEditors; + } + + public bool IsCompressed(int contentTypeId, string alias) + { + var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => + { + var ct = _contentTypeService.Get(contentTypeId); + if (ct == null) return null; + + var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + if (propertyType == null) return null; + + if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; + + var editor = propertyEditor.GetValueEditor(); + if (editor == null) return null; + + return editor.ValueType; + }); + + return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; + } + } +} diff --git a/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs new file mode 100644 index 0000000000..70f3e7c6f0 --- /dev/null +++ b/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Web.PropertyEditors +{ + /// + /// Determines if a property type's value should be compressed + /// + public interface IPropertyCompressionOptions + { + bool IsCompressed(int contentTypeId, string alias); + } +} diff --git a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs new file mode 100644 index 0000000000..638306d7ca --- /dev/null +++ b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs @@ -0,0 +1,10 @@ +namespace Umbraco.Web.PropertyEditors +{ + /// + /// Disables all compression for all properties + /// + internal class NoopPropertyCompressionOptions : IPropertyCompressionOptions + { + public bool IsCompressed(int contentTypeId, string alias) => false; + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs deleted file mode 100644 index 544231bd32..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/AppSettingsNucachePropertyMapFactory.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Configuration; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - // TODO: We'll remove this when the responsibility for compressing property data is at the property editor level - internal class AppSettingsNuCachePropertyMapFactory : INuCachePropertyOptionsFactory - { - public NuCachePropertyCompressionOptions GetNuCachePropertyOptions() - { - var options = new NuCachePropertyCompressionOptions( - GetPropertyMap(), - K4os.Compression.LZ4.LZ4Level.L10_OPT, - null); - - return options; - } - - public IReadOnlyDictionary GetPropertyMap() - { - var propertyMap = new Dictionary(); - // TODO: Use xml/json/c# to define map - var propertyDictionarySerializerMap = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.PropertySerializationMap"]; - if (!string.IsNullOrWhiteSpace(propertyDictionarySerializerMap)) - { - //propertyAlias,CompressionLevel,DecompressionLevel,mappedAlias; - propertyDictionarySerializerMap.Split(';') - .Select(x => - { - var y = x.Split(','); - (string alias, NucachePropertyCompressionLevel compressionLevel, NucachePropertyDecompressionLevel decompressionLevel, string mappedAlias) v = (y[0], - (NucachePropertyCompressionLevel)System.Enum.Parse(typeof(NucachePropertyCompressionLevel), y[1]), - (NucachePropertyDecompressionLevel)System.Enum.Parse(typeof(NucachePropertyDecompressionLevel), y[2]), - y[3] - ); - return v; - }) - .ToList().ForEach(x => propertyMap.Add(x.alias, new NuCacheCompressionOptions(x.compressionLevel, x.decompressionLevel, x.mappedAlias))); - } - return propertyMap; - } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 3cad7171be..a52fde2f0a 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -259,8 +259,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource else { var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.EditData); + ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); d = new ContentData { @@ -288,8 +288,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource else { var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.PubDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.PubData); + ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.PubDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.PubData); p = new ContentData { @@ -326,8 +326,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource throw new InvalidOperationException("No data for media " + dto.Id); var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.EditData); + ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) + : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); var p = new ContentData { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs index 32eb388bee..09933d735d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs @@ -7,8 +7,8 @@ /// public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer { - ContentNestedData DeserializeBytes(byte[] data); - byte[] SerializeBytes(ContentNestedData nestedData); + ContentNestedData DeserializeBytes(int contentTypeId, byte[] data); + byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData); } /// @@ -16,7 +16,7 @@ /// public interface IContentNestedDataSerializer { - ContentNestedData Deserialize(string data); - string Serialize(ContentNestedData nestedData); + ContentNestedData Deserialize(int contentTypeId, string data); + string Serialize(int contentTypeId, ContentNestedData nestedData); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs deleted file mode 100644 index d423499744..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/INucachePropertyOptionsFactory.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - public interface INuCachePropertyOptionsFactory - { - NuCachePropertyCompressionOptions GetNuCachePropertyOptions(); - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 6635fa7090..d4f11591c1 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -11,7 +11,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer { - public ContentNestedData Deserialize(string data) + public ContentNestedData Deserialize(int contentTypeId, string data) { // by default JsonConvert will deserialize our numeric values as Int64 // which is bad, because they were Int32 in the database - take care @@ -30,7 +30,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return JsonConvert.DeserializeObject(data, settings); } - public string Serialize(ContentNestedData nestedData) + public string Serialize(int contentTypeId, ContentNestedData nestedData) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 9df40daf76..3e0e796d36 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -1,17 +1,18 @@ using K4os.Compression.LZ4; using System; using System.Text; -using System.Threading; +using Umbraco.Core.Exceptions; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// /// Lazily decompresses a LZ4 Pickler compressed UTF8 string /// - internal class LazyCompressedString + internal struct LazyCompressedString { - private string _str; private byte[] _bytes; + private string _str; + private readonly object _locker; /// /// Constructor @@ -19,7 +20,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// LZ4 Pickle compressed UTF8 String public LazyCompressedString(byte[] bytes) { + _locker = new object(); _bytes = bytes; + _str = null; } public byte[] GetBytes() @@ -31,13 +34,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public override string ToString() { - return LazyInitializer.EnsureInitialized(ref _str, () => + if (_str != null) return _str; + lock (_locker) { - var str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); + if (_str != null) return _str; // double check + if (_bytes == null) throw new PanicException("Bytes have already been cleared"); + _str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); _bytes = null; - return str; - }); + } + return _str; } + + public static implicit operator string(LazyCompressedString l) => l.ToString(); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs deleted file mode 100644 index 1a2d26f4b1..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/Lz4DictionaryOfPropertyDataSerializer.cs +++ /dev/null @@ -1,181 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using CSharpTest.Net.Serialization; -using Umbraco.Core; -using System.Text; -using K4os.Compression.LZ4; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - - /// - /// Serializes/Deserializes property data as a dictionary for BTree with Lz4 compression options - /// - internal class Lz4DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer>, IDictionaryOfPropertyDataSerializer - { - private readonly IReadOnlyDictionary _compressProperties; - private readonly IReadOnlyDictionary _uncompressProperties; - - - public Lz4DictionaryOfPropertyDataSerializer(INuCachePropertyOptionsFactory nucachePropertyOptionsFactory) - { - var nucachePropertyOptions = nucachePropertyOptionsFactory.GetNuCachePropertyOptions(); - _compressProperties = nucachePropertyOptions.PropertyMap.ToList().ToDictionary(x => string.Intern(x.Key), x => new NuCacheCompressionOptions(x.Value.CompressLevel, x.Value.DecompressLevel, string.Intern(x.Value.MappedAlias))); - _uncompressProperties = _compressProperties.ToList().ToDictionary(x => x.Value.MappedAlias, x => new NuCacheCompressionOptions(x.Value.CompressLevel, x.Value.DecompressLevel, x.Key)); - - _nucachePropertyOptions = nucachePropertyOptions; - } - - - public IDictionary ReadFrom(Stream stream) - { - var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); - - // read properties count - var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); - - // read each property - for (var i = 0; i < pcount; i++) - { - - // read property alias - var alias = PrimitiveSerializer.String.ReadFrom(stream); - var map = GetDeserializationMap(alias); - var key = string.Intern(map.MappedAlias ?? alias); - - // read values count - var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); - - // create pdata and add to the dictionary - var pdatas = new List(); - - // for each value, read and add to pdata - for (var j = 0; j < vcount; j++) - { - var pdata = new PropertyData(); - pdatas.Add(pdata); - - // everything that can be null is read/written as object - // even though - culture and segment should never be null here, as 'null' represents - // the 'current' value, and string.Empty should be used to represent the invariant or - // neutral values - PropertyData throws when getting nulls, so falling back to - // string.Empty here - what else? - pdata.Culture = ReadStringObject(stream, true) ?? string.Empty; - pdata.Segment = ReadStringObject(stream, true) ?? string.Empty; - pdata.Value = ReadObject(stream); - - switch (map.CompressLevel) - { - // If the property is compressed at either the DB or Nucache level, it means it's compressed here and we need to decompress - case NucachePropertyCompressionLevel.SQLDatabase: - case NucachePropertyCompressionLevel.NuCacheDatabase: - if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) - { - //Compressed string - switch (map.DecompressLevel) - { - case NucachePropertyDecompressionLevel.Lazy: - pdata.Value = new LazyCompressedString(byteArrayValue); - break; - case NucachePropertyDecompressionLevel.NotCompressed: - //Shouldn't be any not compressed - throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {alias} since it's compresion option is {map.CompressLevel}"); - case NucachePropertyDecompressionLevel.Immediate: - default: - pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); - break; - } - } - break; - } - } - - dict[key] = pdatas.ToArray(); - } - return dict; - } - - public void WriteTo(IDictionary value, Stream stream) - { - // write properties count - PrimitiveSerializer.Int32.WriteTo(value.Count, stream); - - // write each property - foreach (var (alias, values) in value) - { - var map = GetSerializationMap(alias); - - // write alias - PrimitiveSerializer.String.WriteTo(map.MappedAlias ?? alias, stream); - - // write values count - PrimitiveSerializer.Int32.WriteTo(values.Length, stream); - - // write each value - foreach (var pdata in values) - { - // everything that can be null is read/written as object - // even though - culture and segment should never be null here, - // see note in ReadFrom() method above - WriteObject(pdata.Culture ?? string.Empty, stream); - WriteObject(pdata.Segment ?? string.Empty, stream); - - //Only compress strings (if NucachePropertyCompressionLevel.SQLDatabase has been specified then the property value will already be compressed) - switch (map.CompressLevel) - { - // If we're compressing into btree at the property level - case NucachePropertyCompressionLevel.NuCacheDatabase: - - if (pdata.Value is string stringValue && !(pdata.Value is null) - && (_nucachePropertyOptions.MinimumCompressibleStringLength is null - || !_nucachePropertyOptions.MinimumCompressibleStringLength.HasValue - || stringValue.Length > _nucachePropertyOptions.MinimumCompressibleStringLength.Value)) - { - var stringBytes = Encoding.UTF8.GetBytes(stringValue); - var compressedBytes = LZ4Pickler.Pickle(stringBytes, _nucachePropertyOptions.LZ4CompressionLevel); - WriteObject(compressedBytes, stream); - } - else - { - WriteObject(pdata.Value, stream); - } - break; - default: - WriteObject(pdata.Value, stream); - break; - } - } - } - } - private static readonly NuCacheCompressionOptions DefaultMap = new NuCacheCompressionOptions(NucachePropertyCompressionLevel.None, NucachePropertyDecompressionLevel.NotCompressed, null); - private readonly NuCachePropertyCompressionOptions _nucachePropertyOptions; - - public NuCacheCompressionOptions GetSerializationMap(string propertyAlias) - { - if (_compressProperties is null) - { - return DefaultMap; - } - if (_compressProperties.TryGetValue(propertyAlias, out var map1)) - { - return map1; - } - - return DefaultMap; - } - public NuCacheCompressionOptions GetDeserializationMap(string propertyAlias) - { - if (_uncompressProperties is null) - { - return DefaultMap; - } - if (_uncompressProperties.TryGetValue(propertyAlias, out var map2)) - { - return map2; - } - return DefaultMap; - } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 0475810422..0ea2b96fbe 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -2,10 +2,12 @@ using MessagePack; using MessagePack.Formatters; using MessagePack.Resolvers; +using NPoco.FluentMappings; using System; using System.Collections.Generic; using System.Linq; using System.Text; +using Umbraco.Web.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { @@ -15,9 +17,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer { private MessagePackSerializerOptions _options; - private readonly NuCachePropertyCompressionOptions _propertyOptions; + private readonly IPropertyCompressionOptions _propertyOptions; - public MsgPackContentNestedDataSerializer(INuCachePropertyOptionsFactory propertyOptionsFactory = null) + public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions = null) { var defaultOptions = ContractlessStandardResolver.Options; @@ -37,7 +39,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptionsFactory?.GetNuCachePropertyOptions() ?? NuCachePropertyCompressionOptions.Empty; + _propertyOptions = propertyOptions ?? new NoopPropertyCompressionOptions(); } public string ToJson(string serialized) @@ -47,113 +49,69 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentNestedData Deserialize(string data) + public ContentNestedData Deserialize(int contentTypeId, string data) { var bin = Convert.FromBase64String(data); var nestedData = MessagePackSerializer.Deserialize(bin, _options); - Expand(nestedData); + Expand(contentTypeId, nestedData); return nestedData; } - public string Serialize(ContentNestedData nestedData) + public string Serialize(int contentTypeId, ContentNestedData nestedData) { - Compress(nestedData); + Compress(contentTypeId, nestedData); var bin = MessagePackSerializer.Serialize(nestedData, _options); return Convert.ToBase64String(bin); } - public ContentNestedData DeserializeBytes(byte[] data) + public ContentNestedData DeserializeBytes(int contentTypeId, byte[] data) { var nestedData = MessagePackSerializer.Deserialize(data, _options); - Expand(nestedData); + Expand(contentTypeId, nestedData); return nestedData; } - public byte[] SerializeBytes(ContentNestedData nestedData) + public byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData) { - Compress(nestedData); + Compress(contentTypeId, nestedData); return MessagePackSerializer.Serialize(nestedData, _options); } /// - /// Used during serialization to compress properties and map property names to shorter names + /// Used during serialization to compress properties /// /// - private void Compress(ContentNestedData nestedData) + private void Compress(int contentTypeId, ContentNestedData nestedData) { - if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + foreach(var propertyAliasToData in nestedData.PropertyData) { - foreach (var map in _propertyOptions.PropertyMap) + if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) { - if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { - if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) - { - foreach (var property in properties.Where(x => x.Value != null && x.Value is string)) - { - property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes(property.Value as string), _propertyOptions.LZ4CompressionLevel); - } - } - } - - // if there is an alias map for this property then use that instead of the real property alias - // (used to save memory, the mapped alias is normally a single char or at least a smaller string) - if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties2)) - { - nestedData.PropertyData.Remove(map.Key); - nestedData.PropertyData.Add(map.Value.MappedAlias, properties2); + property.Value = LZ4Pickler.Pickle(Encoding.UTF8.GetBytes((string)property.Value), LZ4Level.L00_FAST); } } } } /// - /// Used during deserialization to map the property data as lazy or expand the value and re-map back to the true property aliases + /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(ContentNestedData nestedData) + private void Expand(int contentTypeId, ContentNestedData nestedData) { - if (_propertyOptions.PropertyMap != null && _propertyOptions.PropertyMap.Count > 0) + foreach (var propertyAliasToData in nestedData.PropertyData) { - foreach (var map in _propertyOptions.PropertyMap) + if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) { - if (map.Value.CompressLevel.Equals(NucachePropertyCompressionLevel.SQLDatabase)) + foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null)) { - // if there is an alias map for this property then re-map to the real property alias - if (map.Value.MappedAlias != null && !map.Key.Equals(map.Value.MappedAlias) - && nestedData.PropertyData.TryGetValue(map.Value.MappedAlias, out PropertyData[] properties2)) + if (property.Value is byte[] byteArrayValue) { - nestedData.PropertyData.Remove(map.Value.MappedAlias); - nestedData.PropertyData.Add(map.Key, properties2); - } - - if (nestedData.PropertyData.TryGetValue(map.Key, out PropertyData[] properties)) - { - foreach (var pdata in properties) - { - if (!(pdata.Value is null) && pdata.Value is byte[] byteArrayValue) - { - //Compressed string - switch (map.Value.DecompressLevel) - { - case NucachePropertyDecompressionLevel.Lazy: - pdata.Value = new LazyCompressedString(byteArrayValue); - break; - case NucachePropertyDecompressionLevel.NotCompressed: - //Shouldn't be any not compressed - throw new InvalidOperationException($"{NucachePropertyDecompressionLevel.NotCompressed} cannot be a decompression option for property {map.Key} since it's compresion option is {map.Value.CompressLevel}"); - case NucachePropertyDecompressionLevel.Immediate: - default: - pdata.Value = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(byteArrayValue)); - break; - } - } - } + property.Value = new LazyCompressedString(byteArrayValue); } } - - } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs deleted file mode 100644 index 36f1606008..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCacheCompressionOptions.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - public struct NuCacheCompressionOptions : IEquatable - { - public NuCacheCompressionOptions(NucachePropertyCompressionLevel compressLevel, NucachePropertyDecompressionLevel decompressLevel, string mappedAlias) - { - CompressLevel = compressLevel; - DecompressLevel = decompressLevel; - MappedAlias = mappedAlias ?? throw new ArgumentNullException(nameof(mappedAlias)); - } - - public NucachePropertyCompressionLevel CompressLevel { get; private set; } - public NucachePropertyDecompressionLevel DecompressLevel { get; private set; } - - /// - /// Used to map a real property alias to a shorter moniker in memory - /// - /// - /// This is simply a memory saving mechanism - /// - public string MappedAlias { get; private set; } - - public override bool Equals(object obj) - { - return obj is NuCacheCompressionOptions options && Equals(options); - } - - public bool Equals(NuCacheCompressionOptions other) - { - return CompressLevel == other.CompressLevel && - DecompressLevel == other.DecompressLevel && - MappedAlias == other.MappedAlias; - } - - public override int GetHashCode() - { - var hashCode = 961370163; - hashCode = hashCode * -1521134295 + CompressLevel.GetHashCode(); - hashCode = hashCode * -1521134295 + DecompressLevel.GetHashCode(); - hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(MappedAlias); - return hashCode; - } - - public static bool operator ==(NuCacheCompressionOptions left, NuCacheCompressionOptions right) - { - return left.Equals(right); - } - - public static bool operator !=(NuCacheCompressionOptions left, NuCacheCompressionOptions right) - { - return !(left == right); - } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs deleted file mode 100644 index 55ab813783..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NuCachePropertyCompressionOptions.cs +++ /dev/null @@ -1,33 +0,0 @@ -using K4os.Compression.LZ4; -using System; -using System.Collections.Generic; - -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - - public class NuCachePropertyCompressionOptions - { - /// - /// Returns empty options - /// - public static NuCachePropertyCompressionOptions Empty { get; } = new NuCachePropertyCompressionOptions(); - - private NuCachePropertyCompressionOptions() - { - } - - public NuCachePropertyCompressionOptions(IReadOnlyDictionary propertyMap, LZ4Level lZ4CompressionLevel, long? minimumCompressibleStringLength) - { - PropertyMap = propertyMap ?? throw new ArgumentNullException(nameof(propertyMap)); - LZ4CompressionLevel = lZ4CompressionLevel; - MinimumCompressibleStringLength = minimumCompressibleStringLength; - } - - public IReadOnlyDictionary PropertyMap { get; } = new Dictionary(); - - public LZ4Level LZ4CompressionLevel { get; } = LZ4Level.L00_FAST; - - // TODO: Unsure if we really want to keep this - public long? MinimumCompressibleStringLength { get; } - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs deleted file mode 100644 index 23826fd722..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyCompressionLevel.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - /// - /// If/where to compress custom properties for nucache - /// - public enum NucachePropertyCompressionLevel - { - None = 0, - - /// - /// Compress property data at the nucache SQL DB table level - /// - /// - /// Idea being we only compress this once. - /// All the records in cmsContentNu need to be rebuilt when this gets enabled. - /// Good option as then we don't use up memory / cpu to compress at boot. - /// - SQLDatabase = 1, - - /// - /// Compress property data at the nucache BTree level - /// - /// - /// Compress the property when writing to nucache bplustree after reading from the database. - /// Idea being we compress this at rebuild / boot. - /// This option supports older items not being compressed already, at the expense of doing this compression at boot. - /// But it also means you can easily switch between None and NuCacheDatabase if performance is worse. - /// - NuCacheDatabase = 2 - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs deleted file mode 100644 index f4d485be71..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/NucachePropertyDecompressionLevel.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - /// - /// If/where to decompress custom properties for nucache - /// - public enum NucachePropertyDecompressionLevel - { - NotCompressed = 0, - - // TODO: I'm unsure if this will ever be necessary, lazy seems good and deserialization would only occur once - Immediate = 1, - - Lazy = 2 - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index b90c418750..8b02946fc2 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -16,11 +16,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixDouble = 'B'; private const char PrefixDateTime = 'D'; private const char PrefixByte = 'O'; - - // TODO: It might make sense to have another prefix for an LZ4 compressed byte array. - // Would be an improvement for the SQLDatabase compression option because then you could mix compressed and decompressed properties with the same alias. - // For example, don't compress recent content, but compress older content. private const char PrefixByteArray = 'A'; + private const char PrefixCompressedStringByteArray = 'C'; protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); @@ -30,7 +27,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected DateTime ReadDateTime(Stream stream) => PrimitiveSerializer.DateTime.ReadFrom(stream); protected byte[] ReadByteArray(Stream stream) => PrimitiveSerializer.Bytes.ReadFrom(stream); - private T? ReadObject(Stream stream, char t, Func read) + private T? ReadStruct(Stream stream, char t, Func read) where T : struct { var type = PrimitiveSerializer.Char.ReadFrom(stream); @@ -51,29 +48,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource : PrimitiveSerializer.String.ReadFrom(stream); } - protected int? ReadIntObject(Stream stream) => ReadObject(stream, PrefixInt32, ReadInt); - protected long? ReadLongObject(Stream stream) => ReadObject(stream, PrefixLong, ReadLong); - protected float? ReadFloatObject(Stream stream) => ReadObject(stream, PrefixFloat, ReadFloat); - protected double? ReadDoubleObject(Stream stream) => ReadObject(stream, PrefixDouble, ReadDouble); - protected DateTime? ReadDateTimeObject(Stream stream) => ReadObject(stream, PrefixDateTime, ReadDateTime); - - protected byte[] ReadByteArrayObject(Stream stream) // required 'cos byte[] is not a struct - { - var type = PrimitiveSerializer.Char.ReadFrom(stream); - if (type == PrefixNull) return null; - if (type != PrefixByteArray) - throw new NotSupportedException($"Cannot deserialize type '{type}', expected '{PrefixByteArray}'."); - return PrimitiveSerializer.Bytes.ReadFrom(stream); - } - + protected int? ReadIntObject(Stream stream) => ReadStruct(stream, PrefixInt32, ReadInt); + protected long? ReadLongObject(Stream stream) => ReadStruct(stream, PrefixLong, ReadLong); + protected float? ReadFloatObject(Stream stream) => ReadStruct(stream, PrefixFloat, ReadFloat); + protected double? ReadDoubleObject(Stream stream) => ReadStruct(stream, PrefixDouble, ReadDouble); + protected DateTime? ReadDateTimeObject(Stream stream) => ReadStruct(stream, PrefixDateTime, ReadDateTime); protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); protected object ReadObject(char type, Stream stream) { - // NOTE: There is going to be a ton of boxing going on here, but i'm not sure we can avoid that because innevitably with our - // current model structure the value will need to end up being 'object' at some point anyways. + // NOTE: This method is only called when reading property data, some boxing may occur but all other reads for structs are + // done with ReadStruct to reduce all boxing. switch (type) { @@ -98,8 +85,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource case PrefixDateTime: return PrimitiveSerializer.DateTime.ReadFrom(stream); case PrefixByteArray: - // When it's a byte array always return as a LazyCompressedString - // TODO: Else we need to make a different prefix for lazy vs eager loading + return PrimitiveSerializer.Bytes.ReadFrom(stream); + case PrefixCompressedStringByteArray: return new LazyCompressedString(PrimitiveSerializer.Bytes.ReadFrom(stream)); default: throw new NotSupportedException($"Cannot deserialize unknown type '{type}'."); @@ -108,6 +95,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected void WriteObject(object value, Stream stream) { + // NOTE: This method is only currently used to write 'string' information, all other writes are done directly with the PrimitiveSerializer + // so no boxing occurs. Though potentially we should write everything via this class just like we do for reads. + if (value == null) { PrimitiveSerializer.Char.WriteTo(PrefixNull, stream); @@ -164,7 +154,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else if (value is LazyCompressedString lazyCompressedString) { - PrimitiveSerializer.Char.WriteTo(PrefixByteArray, stream); + PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream); PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); } else diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index c098b516a0..c5109db027 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -3,6 +3,7 @@ using System.Configuration; using System.Linq; using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Web.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Web.PublishedCache.NuCache @@ -14,36 +15,22 @@ namespace Umbraco.Web.PublishedCache.NuCache base.Compose(composition); var serializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.Serializer"]; - composition.Register(); - composition.Register(); - - // TODO: Based on our findings it seems like this should not be configurable, we should just be using this because it's better - if (serializer == "MsgPack") + if (serializer != "MsgPack") { - var propertyDictionarySerializer = ConfigurationManager.AppSettings["Umbraco.Web.PublishedCache.NuCache.DictionaryOfPropertiesSerializer"]; - if (propertyDictionarySerializer == "LZ4Map") - { - - composition.Register(factory => - { - var lz4Serializer = factory.GetInstance(); - return new ContentDataSerializer(lz4Serializer); - }); - } - else - { - composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); - } - composition.Register(); + // TODO: This allows people to revert to the legacy serializer, by default it will be MessagePack + composition.RegisterUnique(); + composition.RegisterUnique(); } else { - composition.Register(); - composition.Register(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + composition.RegisterUnique(); + composition.RegisterUnique(); } + + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); // register the NuCache database data source - composition.Register(); + composition.RegisterUnique(); // register the NuCache published snapshot service // must register default options, required in the service ctor diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 0135f204c7..e670cb75f6 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1459,8 +1459,8 @@ namespace Umbraco.Web.PublishedCache.NuCache { NodeId = content.Id, Published = published, - Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(nestedData) : null, - RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(nestedData) : null + Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(content.ContentTypeId, nestedData) : null, + RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(content.ContentTypeId, nestedData) : null }; //Core.Composing.Current.Logger.Debug(dto.Data); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index fac5a90b0d..cd689664a1 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -256,20 +256,16 @@ + + - - - - - - - + From 880e5179bfffb030e56a4207b5b7d17c10dc7860 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 26 Aug 2020 16:11:58 +1000 Subject: [PATCH 038/147] Fixing tests --- .../Persistence/Querying/ExpressionVisitorBase.cs | 2 +- src/Umbraco.Tests/Persistence/NPocoTests/NPocoSqlTests.cs | 8 ++++---- .../Persistence/Querying/QueryBuilderTests.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs index 4316023b03..dd8ef75178 100644 --- a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs +++ b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs @@ -331,7 +331,7 @@ namespace Umbraco.Core.Persistence.Querying // BUT we don't want to do a NOT SQL statement since this generally results in indexes not being used // so we want to do an == false SqlParameters.Add(false); - return Visited ? string.Empty : $"({o} = @{SqlParameters.Count - 1})"; + return Visited ? string.Empty : $"{o} = @{SqlParameters.Count - 1}"; //return Visited ? string.Empty : $"NOT ({o} = @{SqlParameters.Count - 1})"; default: // could be anything else, such as: x => !x.Path.StartsWith("-20") diff --git a/src/Umbraco.Tests/Persistence/NPocoTests/NPocoSqlTests.cs b/src/Umbraco.Tests/Persistence/NPocoTests/NPocoSqlTests.cs index a04984eb64..b4210ee31a 100644 --- a/src/Umbraco.Tests/Persistence/NPocoTests/NPocoSqlTests.cs +++ b/src/Umbraco.Tests/Persistence/NPocoTests/NPocoSqlTests.cs @@ -82,9 +82,9 @@ namespace Umbraco.Tests.Persistence.NPocoTests var sql = Sql().SelectAll().From() .Where(x => x.Trashed == false); - Assert.AreEqual("SELECT * FROM [umbracoNode] WHERE (NOT ([umbracoNode].[trashed] = @0))", sql.SQL.Replace("\n", " ")); + Assert.AreEqual("SELECT * FROM [umbracoNode] WHERE ([umbracoNode].[trashed] = @0)", sql.SQL.Replace("\n", " ")); Assert.AreEqual(1, sql.Arguments.Length); - Assert.AreEqual(true, sql.Arguments[0]); + Assert.AreEqual(false, sql.Arguments[0]); } [Test] @@ -92,9 +92,9 @@ namespace Umbraco.Tests.Persistence.NPocoTests { var sql = Sql().SelectAll().From().Where(x => x.Trashed == false); - Assert.AreEqual("SELECT * FROM [umbracoNode] WHERE (NOT ([umbracoNode].[trashed] = @0))", sql.SQL.Replace("\n", " ")); + Assert.AreEqual("SELECT * FROM [umbracoNode] WHERE ([umbracoNode].[trashed] = @0)", sql.SQL.Replace("\n", " ")); Assert.AreEqual(1, sql.Arguments.Length); - Assert.AreEqual(true, sql.Arguments[0]); + Assert.AreEqual(false, sql.Arguments[0]); } [Test] diff --git a/src/Umbraco.Tests/Persistence/Querying/QueryBuilderTests.cs b/src/Umbraco.Tests/Persistence/Querying/QueryBuilderTests.cs index ca6b4cd5f0..2b43b13a79 100644 --- a/src/Umbraco.Tests/Persistence/Querying/QueryBuilderTests.cs +++ b/src/Umbraco.Tests/Persistence/Querying/QueryBuilderTests.cs @@ -110,7 +110,7 @@ namespace Umbraco.Tests.Persistence.Querying Assert.AreEqual("-1,1046,1076,1089%", result.Arguments[0]); Assert.AreEqual(1046, result.Arguments[1]); Assert.AreEqual(true, result.Arguments[2]); - Assert.AreEqual(true, result.Arguments[3]); + Assert.AreEqual(false, result.Arguments[3]); } } } From 6bd6d97bdaccacaf34fe83261e7fe434a11a64e7 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 24 Sep 2020 18:37:24 +1000 Subject: [PATCH 039/147] WIP (fixes migrations) --- .../Migrations/Upgrade/UmbracoPlan.cs | 6 +- .../Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs | 50 -------------- .../AddCmsContentNuByteColumn.cs | 2 +- .../Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs | 69 +++++++++++++++++++ .../CompressedStorageAttribute.cs | 21 ++++++ ...StoragePropertyEditorCompressionOptions.cs | 44 ++++++++++++ .../PropertyEditors/DataEditorAttribute.cs | 1 + .../IPropertyCompressionOptions.cs | 2 +- src/Umbraco.Core/Umbraco.Core.csproj | 7 +- ...ComplexEditorPropertyCompressionOptions.cs | 48 ------------- .../NoopPropertyCompressionOptions.cs | 4 +- .../NuCache/DataSource/DatabaseDataSource.cs | 6 ++ .../MsgPackContentNestedDataSerializer.cs | 1 + .../PublishedCache/NuCache/NuCacheComposer.cs | 3 +- .../NuCache/PublishedSnapshotService.cs | 5 +- src/Umbraco.Web/Umbraco.Web.csproj | 2 - 16 files changed, 159 insertions(+), 112 deletions(-) delete mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs rename src/Umbraco.Core/Migrations/Upgrade/{V_8_7_0 => V_8_9_0}/AddCmsContentNuByteColumn.cs (90%) create mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs create mode 100644 src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs create mode 100644 src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs rename src/{Umbraco.Web => Umbraco.Core}/PropertyEditors/IPropertyCompressionOptions.cs (84%) delete mode 100644 src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index a15dcfb23b..cb35783ffa 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -7,7 +7,7 @@ using Umbraco.Core.Migrations.Upgrade.V_8_0_0; using Umbraco.Core.Migrations.Upgrade.V_8_0_1; using Umbraco.Core.Migrations.Upgrade.V_8_1_0; using Umbraco.Core.Migrations.Upgrade.V_8_6_0; -using Umbraco.Core.Migrations.Upgrade.V_8_8_0; +using Umbraco.Core.Migrations.Upgrade.V_8_9_0; namespace Umbraco.Core.Migrations.Upgrade { @@ -194,9 +194,9 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.7.0... To("{a78e3369-8ea3-40ec-ad3f-5f76929d2b20}"); - To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); - // to 8.8.0... + // to 8.9.0... + To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); To("{4695D0C9-0729-4976-985B-048D503665D8}"); //FINAL diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs deleted file mode 100644 index d4dd783876..0000000000 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Linq; -using Umbraco.Core.Migrations.Expressions.Execute.Expressions; -using Umbraco.Core.Persistence.DatabaseModelDefinitions; -using Umbraco.Core.Persistence.Dtos; - -namespace Umbraco.Core.Migrations.Upgrade.V_8_8_0 -{ - public class UpgradedIncludeIndexes : MigrationBase - { - public UpgradedIncludeIndexes(IMigrationContext context) - : base(context) - { - - } - - public override void Migrate() - { - var indexesToReplace = new[] { $"IX_{NodeDto.TableName}_UniqueId", $"IX_{NodeDto.TableName}_ObjectType" }; - DeleteIndexes(indexesToReplace); // delete existing ones - CreateIndexes(indexesToReplace); // replace - CreateIndexes($"IX_{NodeDto.TableName}_Level"); // add the new definitions - - var contentVersionNodeIdIndex = $"IX_{ContentVersionDto.TableName}_NodeId"; - DeleteIndexes(contentVersionNodeIdIndex); // delete existing ones - CreateIndexes(contentVersionNodeIdIndex); // replace - CreateIndexes($"IX_{ContentVersionDto.TableName}_Current"); // add the new definitions - } - - private void DeleteIndexes(params string[] toDelete) - { - var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); - - foreach (var i in toDelete) - Delete.Index(i).OnTable(tableDef.Name).Do(); - } - - private void CreateIndexes(params string[] toCreate) - { - var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); - - foreach(var c in toCreate) - { - // get the definition by name - var index = tableDef.Indexes.First(x => x.Name == c); - new ExecuteSqlStatementExpression(Context) { SqlStatement = Context.SqlContext.SqlSyntax.Format(index) }.Execute(); - } - - } - } -} diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs similarity index 90% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs index 3cb51afb99..c295e13051 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs @@ -1,7 +1,7 @@ using System.Linq; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_6_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_9_0 { public class AddCmsContentNuByteColumn : MigrationBase { diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs new file mode 100644 index 0000000000..083294d1a5 --- /dev/null +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs @@ -0,0 +1,69 @@ +using System.Linq; +using Umbraco.Core.Migrations.Expressions.Execute.Expressions; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; +using Umbraco.Core.Persistence.Dtos; + +namespace Umbraco.Core.Migrations.Upgrade.V_8_9_0 +{ + public class UpgradedIncludeIndexes : MigrationBase + { + public UpgradedIncludeIndexes(IMigrationContext context) + : base(context) + { + + } + + public override void Migrate() + { + // Need to drop the FK for the redirect table before modifying the unique id index + Delete.ForeignKey() + .FromTable(Constants.DatabaseSchema.Tables.RedirectUrl) + .ForeignColumn("contentKey") + .ToTable(NodeDto.TableName) + .PrimaryColumn("uniqueID") + .Do(); + var nodeDtoIndexes = new[] { $"IX_{NodeDto.TableName}_UniqueId", $"IX_{NodeDto.TableName}_ObjectType", $"IX_{NodeDto.TableName}_Level" }; + DeleteIndexes(nodeDtoIndexes); // delete existing ones + CreateIndexes(nodeDtoIndexes); // update/add + // Now re-create the FK for the redirect table + Create.ForeignKey() + .FromTable(Constants.DatabaseSchema.Tables.RedirectUrl) + .ForeignColumn("contentKey") + .ToTable(NodeDto.TableName) + .PrimaryColumn("uniqueID") + .Do(); + + + var contentVersionIndexes = new[] { $"IX_{ContentVersionDto.TableName}_NodeId", $"IX_{ContentVersionDto.TableName}_Current" }; + DeleteIndexes(contentVersionIndexes); // delete existing ones + CreateIndexes(contentVersionIndexes); // update/add + } + + private void DeleteIndexes(params string[] toDelete) + { + var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); + + foreach (var i in toDelete) + { + if (IndexExists(i)) + { + Delete.Index(i).OnTable(tableDef.Name).Do(); + } + } + + } + + private void CreateIndexes(params string[] toCreate) + { + var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); + + foreach (var c in toCreate) + { + // get the definition by name + var index = tableDef.Indexes.First(x => x.Name == c); + new ExecuteSqlStatementExpression(Context) { SqlStatement = Context.SqlContext.SqlSyntax.Format(index) }.Execute(); + } + + } + } +} diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs b/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs new file mode 100644 index 0000000000..31689c4ee9 --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs @@ -0,0 +1,21 @@ +using System; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// When assigned to a DataEditor it indicates that the values it generates can be compressed + /// + /// + /// Used in conjunction with + /// + [AttributeUsage(AttributeTargets.Class)] + public sealed class CompressedStorageAttribute : Attribute + { + public CompressedStorageAttribute(bool isCompressed = true) + { + IsCompressed = isCompressed; + } + + public bool IsCompressed { get; } + } +} diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs new file mode 100644 index 0000000000..15795bb61c --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs @@ -0,0 +1,44 @@ +using System.Collections.Concurrent; +using System.Linq; +using Umbraco.Core.Services; + +namespace Umbraco.Core.PropertyEditors +{ + + /// + /// Ensures all property types that have an editor storing a complex value are compressed + /// + public class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions + { + private readonly IContentTypeService _contentTypeService; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); + + public CompressedStoragePropertyEditorCompressionOptions(PropertyEditorCollection propertyEditors) + { + _propertyEditors = propertyEditors; + } + + public bool IsCompressed(int contentTypeId, string alias) + { + return false; + //var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => + //{ + // var ct = _contentTypeService.Get(contentTypeId); + // if (ct == null) return null; + + // var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + // if (propertyType == null) return null; + + // if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; + + // var editor = propertyEditor.GetValueEditor(); + // if (editor == null) return null; + + // return editor.ValueType; + //}); + + //return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; + } + } +} diff --git a/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs b/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs index 7b3be7ea5f..0375509797 100644 --- a/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs +++ b/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs @@ -2,6 +2,7 @@ namespace Umbraco.Core.PropertyEditors { + /// /// Marks a class that represents a data editor. /// diff --git a/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs similarity index 84% rename from src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs rename to src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs index 70f3e7c6f0..f5aaf3dc57 100644 --- a/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs @@ -1,4 +1,4 @@ -namespace Umbraco.Web.PropertyEditors +namespace Umbraco.Core.PropertyEditors { /// /// Determines if a property type's value should be compressed diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 4a25863158..444ef1a464 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -132,8 +132,8 @@ - - + + @@ -152,6 +152,9 @@ + + + diff --git a/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs deleted file mode 100644 index f4776f652d..0000000000 --- a/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs +++ /dev/null @@ -1,48 +0,0 @@ -using K4os.Compression.LZ4; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using Umbraco.Core.PropertyEditors; -using Umbraco.Core.Services; - -namespace Umbraco.Web.PropertyEditors -{ - - /// - /// Ensures all property types that have an editor storing a complex value are compressed - /// - public class ComplexEditorPropertyCompressionOptions : IPropertyCompressionOptions - { - private readonly IContentTypeService _contentTypeService; - private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); - - public ComplexEditorPropertyCompressionOptions(IContentTypeService contentTypeService, PropertyEditorCollection propertyEditors) - { - _contentTypeService = contentTypeService; - _propertyEditors = propertyEditors; - } - - public bool IsCompressed(int contentTypeId, string alias) - { - var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => - { - var ct = _contentTypeService.Get(contentTypeId); - if (ct == null) return null; - - var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); - if (propertyType == null) return null; - - if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; - - var editor = propertyEditor.GetValueEditor(); - if (editor == null) return null; - - return editor.ValueType; - }); - - return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; - } - } -} diff --git a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs index 638306d7ca..6f626938bc 100644 --- a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs +++ b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs @@ -1,4 +1,6 @@ -namespace Umbraco.Web.PropertyEditors +using Umbraco.Core.PropertyEditors; + +namespace Umbraco.Web.PropertyEditors { /// /// Disables all compression for all properties diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index a52fde2f0a..0b3003d7ec 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -129,7 +129,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) + { yield return CreateContentNodeKit(row); + } } public IEnumerable GetBranchContentSources(IScope scope, int id) @@ -145,7 +147,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + { yield return CreateContentNodeKit(row); + } } public IEnumerable GetTypeContentSources(IScope scope, IEnumerable ids) @@ -161,7 +165,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + { yield return CreateContentNodeKit(row); + } } private Sql MediaSourcesSelect(IScope scope, Func, Sql> joins = null) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 0ea2b96fbe..4965935fbf 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using Umbraco.Core.PropertyEditors; using Umbraco.Web.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index c5109db027..baa96a4a2e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -3,6 +3,7 @@ using System.Configuration; using System.Linq; using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Core.PropertyEditors; using Umbraco.Web.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -24,7 +25,7 @@ namespace Umbraco.Web.PublishedCache.NuCache else { composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index e670cb75f6..41dfdd7a64 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -387,10 +387,9 @@ namespace Umbraco.Web.PublishedCache.NuCache // contentStore is wlocked (1 thread) // content (and types) are read-locked - var contentTypes = _serviceContext.ContentTypeService.GetAll() - .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + var contentTypes = _serviceContext.ContentTypeService.GetAll().ToList(); - _contentStore.SetAllContentTypesLocked(contentTypes); + _contentStore.SetAllContentTypesLocked(contentTypes.Select(x => _publishedContentTypeFactory.CreateContentType(x))); using (_logger.TraceDuration("Loading content from database")) { diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 51f4d838af..25478bf626 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -262,7 +262,6 @@ - @@ -271,7 +270,6 @@ - From 780b2e573b991b7a3b59f7f8b56b7cfa19a6d4ed Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 25 Sep 2020 00:27:46 +1000 Subject: [PATCH 040/147] Fixes a startup issue that doesn't bubble the underlying boot failed exception if the container fails. --- src/Umbraco.Core/Composing/Current.cs | 13 ++++++++++++- src/Umbraco.Core/Runtime/CoreRuntime.cs | 8 +++++++- src/Umbraco.Web/Composing/ModuleInjector.cs | 2 +- src/Umbraco.Web/Runtime/WebRuntime.cs | 16 ++++++++++------ 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Core/Composing/Current.cs b/src/Umbraco.Core/Composing/Current.cs index a06f09baf6..a4ccd59f9d 100644 --- a/src/Umbraco.Core/Composing/Current.cs +++ b/src/Umbraco.Core/Composing/Current.cs @@ -30,6 +30,7 @@ namespace Umbraco.Core.Composing public static class Current { private static IFactory _factory; + private static IRuntimeState _state; // TODO: get rid of these oddities // we don't want Umbraco tests to die because the container has not been properly initialized, @@ -125,7 +126,17 @@ namespace Umbraco.Core.Composing ?? new ProfilingLogger(Logger, Profiler); public static IRuntimeState RuntimeState - => Factory.GetInstance(); + { + get + { + return _state ?? Factory.GetInstance(); + } + internal set + { + // this is only used when the boot entirely fails, we need to manually set this so we can report + _state = value; + } + } public static TypeLoader TypeLoader => Factory.GetInstance(); diff --git a/src/Umbraco.Core/Runtime/CoreRuntime.cs b/src/Umbraco.Core/Runtime/CoreRuntime.cs index b852aff2ff..75cbaa6913 100644 --- a/src/Umbraco.Core/Runtime/CoreRuntime.cs +++ b/src/Umbraco.Core/Runtime/CoreRuntime.cs @@ -210,7 +210,13 @@ namespace Umbraco.Core.Runtime { _factory = Current.Factory = composition?.CreateFactory(); } - catch { /* yea */ } + catch + { + // In this case we are basically dead, we do not have a factory but we need + // to report on the state so we need to manually set that, this is the only time + // we ever do this. + Current.RuntimeState = _state; + } } Debugger.Break(); diff --git a/src/Umbraco.Web/Composing/ModuleInjector.cs b/src/Umbraco.Web/Composing/ModuleInjector.cs index 57ef766dea..ccc8b87719 100644 --- a/src/Umbraco.Web/Composing/ModuleInjector.cs +++ b/src/Umbraco.Web/Composing/ModuleInjector.cs @@ -30,7 +30,7 @@ namespace Umbraco.Web.Composing try { - runtimeState = Current.Factory.GetInstance(); + runtimeState = Current.RuntimeState; } catch { /* don't make it worse */ } diff --git a/src/Umbraco.Web/Runtime/WebRuntime.cs b/src/Umbraco.Web/Runtime/WebRuntime.cs index ffcd2343ed..e32f2f824f 100644 --- a/src/Umbraco.Web/Runtime/WebRuntime.cs +++ b/src/Umbraco.Web/Runtime/WebRuntime.cs @@ -54,12 +54,16 @@ namespace Umbraco.Web.Runtime var factory = base.Boot(register); - // now (and only now) is the time to switch over to perWebRequest scopes. - // up until that point we may not have a request, and scoped services would - // fail to resolve - but we run Initialize within a factory scope - and then, - // here, we switch the factory to bind scopes to requests - factory.EnablePerWebRequestScope(); - + // factory can be null if part of the boot process fails + if (factory != null) + { + // now (and only now) is the time to switch over to perWebRequest scopes. + // up until that point we may not have a request, and scoped services would + // fail to resolve - but we run Initialize within a factory scope - and then, + // here, we switch the factory to bind scopes to requests + factory.EnablePerWebRequestScope(); + } + return factory; } From 67a9b5bb9770908f431760f88783b00d64839fae Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 25 Sep 2020 00:32:11 +1000 Subject: [PATCH 041/147] Refactors the serialization of the content data that is stored in the nucache table. This had to change because we need to resolve content type data in order to check if the property should be compressed and we cannot do that data lookup while the data is being processed since we get an open data reader exception. This is fixed now by using a serializer factory instead so the Create method can do any initialization needed prior to running any serialization operation. Renames a few things so we dont have ContentNested (whatever that meant ) --- ...StoragePropertyEditorCompressionOptions.cs | 45 +++++----- .../IPropertyCompressionOptions.cs | 8 +- .../ContentSerializationTests.cs | 20 ++--- .../PublishedContent/NuCacheChildrenTests.cs | 6 +- .../PublishedContent/NuCacheTests.cs | 6 +- .../Scoping/ScopedNuCacheTests.cs | 6 +- .../ContentTypeServiceVariantsTests.cs | 6 +- .../Editors/NuCacheStatusController.cs | 5 ++ .../BlockListPropertyEditor.cs | 1 + .../PropertyEditors/GridPropertyEditor.cs | 1 + .../PropertyEditors/MarkdownPropertyEditor.cs | 1 + .../NestedContentPropertyEditor.cs | 1 + .../NoopPropertyCompressionOptions.cs | 12 --- .../PropertyEditors/RichTextPropertyEditor.cs | 1 + .../PropertyEditors/TextAreaPropertyEditor.cs | 1 + .../PublishedCache/NuCache/ContentNodeKit.cs | 1 + ...NestedData.cs => ContentCacheDataModel.cs} | 4 +- .../ContentCacheDataSerializationResult.cs | 47 +++++++++++ .../ContentCacheDataSerializerEntityType.cs | 13 +++ .../NuCache/DataSource/DatabaseDataSource.cs | 82 +++++++++++-------- .../DataSource/IContentCacheDataSerializer.cs | 18 ++++ .../IContentCacheDataSerializerFactory.cs | 16 ++++ .../IContentNestedDataSerializer.cs | 22 ----- .../JsonContentNestedDataSerializer.cs | 21 ++--- .../JsonContentNestedDataSerializerFactory.cs | 10 +++ .../MsgPackContentNestedDataSerializer.cs | 77 +++++++++-------- ...gPackContentNestedDataSerializerFactory.cs | 62 ++++++++++++++ .../PublishedCache/NuCache/NuCacheComposer.cs | 11 +-- .../NuCache/PublishedSnapshotService.cs | 68 ++++++++------- src/Umbraco.Web/Umbraco.Web.csproj | 10 ++- 30 files changed, 377 insertions(+), 205 deletions(-) delete mode 100644 src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs rename src/Umbraco.Web/PublishedCache/NuCache/DataSource/{ContentNestedData.cs => ContentCacheDataModel.cs} (93%) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs index 15795bb61c..a99452a5b1 100644 --- a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs @@ -1,44 +1,47 @@ using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; -using Umbraco.Core.Services; +using Umbraco.Core.Models; namespace Umbraco.Core.PropertyEditors { /// - /// Ensures all property types that have an editor storing a complex value are compressed + /// Ensures all property types that have a property editor attributed with use data compression /// - public class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions + internal class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions { - private readonly IContentTypeService _contentTypeService; + private readonly IReadOnlyDictionary _contentTypes; private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); + private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache; - public CompressedStoragePropertyEditorCompressionOptions(PropertyEditorCollection propertyEditors) + public CompressedStoragePropertyEditorCompressionOptions( + IReadOnlyDictionary contentTypes, + PropertyEditorCollection propertyEditors, + ConcurrentDictionary<(int, string), CompressedStorageAttribute> compressedStoragePropertyEditorCache) { - _propertyEditors = propertyEditors; + _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); + _propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors)); + _compressedStoragePropertyEditorCache = compressedStoragePropertyEditorCache; } public bool IsCompressed(int contentTypeId, string alias) { - return false; - //var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => - //{ - // var ct = _contentTypeService.Get(contentTypeId); - // if (ct == null) return null; + var compressedStorage = _compressedStoragePropertyEditorCache.GetOrAdd((contentTypeId, alias), x => + { + if (!_contentTypes.TryGetValue(contentTypeId, out var ct)) + return null; - // var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); - // if (propertyType == null) return null; + var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + if (propertyType == null) return null; - // if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; + if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; - // var editor = propertyEditor.GetValueEditor(); - // if (editor == null) return null; + var attribute = propertyEditor.GetType().GetCustomAttribute(true); + return attribute; + }); - // return editor.ValueType; - //}); - - //return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; + return compressedStorage?.IsCompressed ?? false; } } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs index f5aaf3dc57..d1add38f19 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs @@ -1,10 +1,12 @@ -namespace Umbraco.Core.PropertyEditors +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors { /// /// Determines if a property type's value should be compressed /// public interface IPropertyCompressionOptions - { - bool IsCompressed(int contentTypeId, string alias); + { + bool IsCompressed(int contentTypeId, string propertyTypeAlias); } } diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index c85973f4b0..4be80083b8 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -1,10 +1,8 @@ -using NUnit.Framework; +using Moq; +using NUnit.Framework; using System; -using System.Collections; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Umbraco.Core.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Tests.PublishedContent @@ -16,10 +14,10 @@ namespace Umbraco.Tests.PublishedContent public void Ensure_Same_Results() { var jsonSerializer = new JsonContentNestedDataSerializer(); - var msgPackSerializer = new MsgPackContentNestedDataSerializer(); + var msgPackSerializer = new MsgPackContentNestedDataSerializer(Mock.Of()); var now = DateTime.Now; - var content = new ContentNestedData + var content = new ContentCacheDataModel { PropertyData = new Dictionary { @@ -55,14 +53,14 @@ namespace Umbraco.Tests.PublishedContent UrlSegment = "home" }; - var json = jsonSerializer.Serialize(1, content); - var msgPack = msgPackSerializer.Serialize(1, content); + var json = jsonSerializer.Serialize(1, content).StringData; + var msgPack = msgPackSerializer.Serialize(1, content).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(1, json); - var msgPackContent = msgPackSerializer.Deserialize(1, msgPack); + var jsonContent = jsonSerializer.Deserialize(1, json, null); + var msgPackContent = msgPackSerializer.Deserialize(1, null, msgPack); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index fef096498c..afba2dcc4f 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -37,7 +37,7 @@ namespace Umbraco.Tests.PublishedContent private ContentType _contentTypeInvariant; private ContentType _contentTypeVariant; private TestDataSource _source; - private IContentNestedDataSerializer _contentNestedDataSerializer; + private IContentCacheDataSerializerFactory _contentNestedDataSerializerFactory; [TearDown] public void Teardown() @@ -135,7 +135,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache _source = new TestDataSource(kits()); - _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); + _contentNestedDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); // at last, create the complete NuCache snapshot service! var options = new PublishedSnapshotServiceOptions { IgnoreLocalDb = true }; @@ -158,7 +158,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - _contentNestedDataSerializer); + _contentNestedDataSerializerFactory); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 792ccc8529..eee3500495 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -33,7 +33,7 @@ namespace Umbraco.Tests.PublishedContent { private IPublishedSnapshotService _snapshotService; private IVariationContextAccessor _variationAccesor; - private IContentNestedDataSerializer _contentNestedDataSerializer; + private IContentCacheDataSerializerFactory _contentNestedDataSerializerFactory; private ContentType _contentType; private PropertyType _propertyType; @@ -115,7 +115,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache var dataSource = new TestDataSource(kit); - _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); + _contentNestedDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); var runtime = Mock.Of(); Mock.Get(runtime).Setup(x => x.Level).Returns(RuntimeLevel.Run); @@ -204,7 +204,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - _contentNestedDataSerializer); + _contentNestedDataSerializerFactory); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index 5f72947382..be10db3a9d 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -82,7 +82,7 @@ namespace Umbraco.Tests.Scoping var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); + var nestedContentDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); return new PublishedSnapshotService( options, null, @@ -96,12 +96,12 @@ namespace Umbraco.Tests.Scoping ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(nestedContentDataSerializer), + new DatabaseDataSource(nestedContentDataSerializerFactory), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - nestedContentDataSerializer); + nestedContentDataSerializerFactory); } protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable urlProviders = null) diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index 938b14c3a9..aaad60f7e9 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -53,7 +53,7 @@ namespace Umbraco.Tests.Services var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); + var nestedContentDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); return new PublishedSnapshotService( options, @@ -68,12 +68,12 @@ namespace Umbraco.Tests.Services ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(nestedContentDataSerializer), + new DatabaseDataSource(nestedContentDataSerializerFactory), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - nestedContentDataSerializer); + nestedContentDataSerializerFactory); } public class LocalServerMessenger : ServerMessengerBase diff --git a/src/Umbraco.Web/Editors/NuCacheStatusController.cs b/src/Umbraco.Web/Editors/NuCacheStatusController.cs index 86dbdd4e01..589c763363 100644 --- a/src/Umbraco.Web/Editors/NuCacheStatusController.cs +++ b/src/Umbraco.Web/Editors/NuCacheStatusController.cs @@ -35,6 +35,11 @@ namespace Umbraco.Web.Editors service.RebuildContentDbCache(); service.RebuildMediaDbCache(); service.RebuildMemberDbCache(); + + // TODO: Shouldn't this just be ?? + // service.Rebuild(); + + return service.GetStatus(); } diff --git a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs index 42023382f1..2c1221e99e 100644 --- a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs @@ -13,6 +13,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a block list property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.BlockList, "Block List", diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 862837381a..7ce312c516 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -17,6 +17,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a grid property and parameter editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.Grid, "Grid layout", diff --git a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs index 2d66da5461..6ce66aaa00 100644 --- a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs @@ -7,6 +7,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a markdown editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.MarkdownEditor, "Markdown editor", diff --git a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs index 8f25449f99..ffe0051607 100644 --- a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs @@ -22,6 +22,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a nested content property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.NestedContent, "Nested Content", diff --git a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs deleted file mode 100644 index 6f626938bc..0000000000 --- a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Umbraco.Core.PropertyEditors; - -namespace Umbraco.Web.PropertyEditors -{ - /// - /// Disables all compression for all properties - /// - internal class NoopPropertyCompressionOptions : IPropertyCompressionOptions - { - public bool IsCompressed(int contentTypeId, string alias) => false; - } -} diff --git a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs index 42777f11ad..7c7a358bf3 100644 --- a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs @@ -17,6 +17,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a rich text property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TinyMce, "Rich Text Editor", diff --git a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs index c7bc2efbda..878330820a 100644 --- a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs @@ -7,6 +7,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a textarea property and parameter editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TextArea, EditorType.PropertyValue | EditorType.MacroParameter, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs index 61fb5c12a3..43aa9a14d7 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs @@ -15,6 +15,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public bool IsNull => ContentTypeId < 0; + public static ContentNodeKit Empty { get; } = new ContentNodeKit(); public static ContentNodeKit Null { get; } = new ContentNodeKit { ContentTypeId = -1 }; public void Build( diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs similarity index 93% rename from src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs rename to src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs index 1a49aaaf62..40acdfdb55 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs @@ -7,10 +7,10 @@ using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// - /// The content item 1:M data that is serialized to JSON + /// The content model stored in the content cache database table serialized as JSON /// [DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys - public class ContentNestedData + public class ContentCacheDataModel { // TODO: We don't want to allocate empty arrays //dont serialize empty properties diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs new file mode 100644 index 0000000000..7cd388d712 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public struct ContentCacheDataSerializationResult : IEquatable + { + public ContentCacheDataSerializationResult(string stringData, byte[] byteData) + { + StringData = stringData; + ByteData = byteData; + } + + public string StringData { get; } + public byte[] ByteData { get; } + + public override bool Equals(object obj) + { + return obj is ContentCacheDataSerializationResult result && Equals(result); + } + + public bool Equals(ContentCacheDataSerializationResult other) + { + return StringData == other.StringData && + EqualityComparer.Default.Equals(ByteData, other.ByteData); + } + + public override int GetHashCode() + { + var hashCode = 1910544615; + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(StringData); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(ByteData); + return hashCode; + } + + public static bool operator ==(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right) + { + return left.Equals(right); + } + + public static bool operator !=(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right) + { + return !(left == right); + } + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs new file mode 100644 index 0000000000..e5b15f8dce --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs @@ -0,0 +1,13 @@ +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + [Flags] + public enum ContentCacheDataSerializerEntityType + { + Document = 1, + Media = 2, + Member = 4 + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 0b3003d7ec..e60fd6623e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -2,14 +2,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using Newtonsoft.Json; using NPoco; using Umbraco.Core; using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Dtos; using Umbraco.Core.Scoping; -using Umbraco.Core.Serialization; using Umbraco.Web.Composing; using static Umbraco.Core.Persistence.NPocoSqlExtensions.Statics; @@ -21,11 +19,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class DatabaseDataSource : IDataSource { private const int PageSize = 500; - private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory; - public DatabaseDataSource(IContentNestedDataSerializer contentNestedDataSerializer) + public DatabaseDataSource(IContentCacheDataSerializerFactory contentCacheDataSerializerFactory) { - _contentNestedDataSerializer = contentNestedDataSerializer; + _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; } // we want arrays, we want them all loaded, not an enumerable @@ -110,7 +108,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); - return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto); + + if (dto == null) return ContentNodeKit.Empty; + + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + return CreateContentNodeKit(dto, serializer); } public IEnumerable GetAllContentSources(IScope scope) @@ -125,12 +127,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -143,12 +147,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -161,12 +167,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -201,7 +209,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); - return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto); + + if (dto == null) return ContentNodeKit.Empty; + + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + return CreateMediaNodeKit(dto, serializer); } public IEnumerable GetAllMediaSources(IScope scope) @@ -210,11 +222,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } public IEnumerable GetBranchMediaSources(IScope scope, int id) @@ -226,11 +242,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } public IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids) @@ -242,14 +262,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } - private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer) { ContentData d = null; ContentData p = null; @@ -264,9 +288,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); + var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); d = new ContentData { @@ -276,9 +298,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.EditWriterId, - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData, - UrlSegment = nested.UrlSegment + Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedContent.CultureData, + UrlSegment = deserializedContent.UrlSegment }; } } @@ -293,21 +315,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.PubDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.PubData); + var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.PubData, dto.PubDataRaw); p = new ContentData { Name = dto.PubName, - UrlSegment = nested.UrlSegment, + UrlSegment = deserializedContent.UrlSegment, Published = true, TemplateId = dto.PubTemplateId, VersionId = dto.VersionId, VersionDate = dto.PubVersionDate, WriterId = dto.PubWriterId, - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData + Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedContent.CultureData }; } } @@ -326,14 +346,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return s; } - private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer) { if (dto.EditData == null && dto.EditDataRaw == null) throw new InvalidOperationException("No data for media " + dto.Id); - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); + var deserializedMedia = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); var p = new ContentData { @@ -343,8 +361,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.CreatorId, // what-else? - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData + Properties = deserializedMedia.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedMedia.CultureData }; var n = new ContentNode(dto.Id, dto.Uid, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs new file mode 100644 index 0000000000..87ac5af91e --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -0,0 +1,18 @@ +using Umbraco.Core.PropertyEditors; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + + /// + /// Serializes/Deserializes document to the SQL Database as a string + /// + /// + /// Resolved from the . This cannot be resolved from DI. + /// + public interface IContentCacheDataSerializer + { + ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData); + ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model); + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs new file mode 100644 index 0000000000..14dfd7dc5b --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs @@ -0,0 +1,16 @@ +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public interface IContentCacheDataSerializerFactory + { + /// + /// Gets or creates a new instance of + /// + /// + /// + /// This method may return the same instance, however this depends on the state of the application and if any underlying data has changed. + /// This method may also be used to initialize anything before a serialization/deserialization session occurs. + /// + IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types); + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs deleted file mode 100644 index 09933d735d..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - // TODO: We need better names if possible, not sure why the class is called ContentNested in the first place - - /// - /// Serializes/Deserializes document to the SQL Database as bytes - /// - public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer - { - ContentNestedData DeserializeBytes(int contentTypeId, byte[] data); - byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData); - } - - /// - /// Serializes/Deserializes document to the SQL Database as a string - /// - public interface IContentNestedDataSerializer - { - ContentNestedData Deserialize(int contentTypeId, string data); - string Serialize(int contentTypeId, ContentNestedData nestedData); - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index d4f11591c1..2fa892a5e6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,18 +1,18 @@ using Newtonsoft.Json; +using System; using System.Collections.Generic; -using System.Linq; -using System.Reflection.Emit; -using System.Text; -using System.Threading.Tasks; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer + public class JsonContentNestedDataSerializer : IContentCacheDataSerializer { - public ContentNestedData Deserialize(int contentTypeId, string data) + public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { + if (byteData != null) + throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); + // by default JsonConvert will deserialize our numeric values as Int64 // which is bad, because they were Int32 in the database - take care @@ -24,18 +24,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateParseHandling = DateParseHandling.DateTime, DateFormatHandling = DateFormatHandling.IsoDateFormat, DateTimeZoneHandling = DateTimeZoneHandling.Utc, - DateFormatString = "o" + DateFormatString = "o" }; - return JsonConvert.DeserializeObject(data, settings); + return JsonConvert.DeserializeObject(stringData, settings); } - public string Serialize(int contentTypeId, ContentNestedData nestedData) + public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 - return JsonConvert.SerializeObject(nestedData); + var json = JsonConvert.SerializeObject(model); + return new ContentCacheDataSerializationResult(json, null); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs new file mode 100644 index 0000000000..e857eb8bf5 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs @@ -0,0 +1,10 @@ +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class JsonContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory + { + private Lazy _serializer = new Lazy(); + public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) => _serializer.Value; + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 4965935fbf..aad337b236 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,29 +1,27 @@ using K4os.Compression.LZ4; using MessagePack; -using MessagePack.Formatters; using MessagePack.Resolvers; -using NPoco.FluentMappings; using System; -using System.Collections.Generic; using System.Linq; using System.Text; using Umbraco.Core.PropertyEditors; -using Umbraco.Web.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// - /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack + /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack /// - internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer + public class MsgPackContentNestedDataSerializer : IContentCacheDataSerializer { - private MessagePackSerializerOptions _options; + private readonly MessagePackSerializerOptions _options; private readonly IPropertyCompressionOptions _propertyOptions; - public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions = null) + public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions) { - var defaultOptions = ContractlessStandardResolver.Options; + _propertyOptions = propertyOptions ?? throw new ArgumentNullException(nameof(propertyOptions)); + var defaultOptions = ContractlessStandardResolver.Options; var resolver = CompositeResolver.Create( // TODO: We want to be able to intern the strings for aliases when deserializing like we do for Newtonsoft but I'm unsure exactly how @@ -39,52 +37,51 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) - .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptions ?? new NoopPropertyCompressionOptions(); + .WithCompression(MessagePackCompression.Lz4BlockArray); } - public string ToJson(string serialized) + public string ToJson(byte[] bin) { - var bin = Convert.FromBase64String(serialized); var json = MessagePackSerializer.ConvertToJson(bin, _options); return json; } - public ContentNestedData Deserialize(int contentTypeId, string data) + public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { - var bin = Convert.FromBase64String(data); - var nestedData = MessagePackSerializer.Deserialize(bin, _options); - Expand(contentTypeId, nestedData); - return nestedData; + if (stringData != null) + { + // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) + var bin = Convert.FromBase64String(stringData); + var content = MessagePackSerializer.Deserialize(bin, _options); + Expand(contentTypeId, content); + return content; + } + else if (byteData != null) + { + var content = MessagePackSerializer.Deserialize(byteData, _options); + Expand(contentTypeId, content); + return content; + } + else + { + return null; + } } - public string Serialize(int contentTypeId, ContentNestedData nestedData) + public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) { - Compress(contentTypeId, nestedData); - var bin = MessagePackSerializer.Serialize(nestedData, _options); - return Convert.ToBase64String(bin); - } - - public ContentNestedData DeserializeBytes(int contentTypeId, byte[] data) - { - var nestedData = MessagePackSerializer.Deserialize(data, _options); - Expand(contentTypeId, nestedData); - return nestedData; - } - - public byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData) - { - Compress(contentTypeId, nestedData); - return MessagePackSerializer.Serialize(nestedData, _options); + Compress(contentTypeId, model); + var bytes = MessagePackSerializer.Serialize(model, _options); + return new ContentCacheDataSerializationResult(null, bytes); } /// /// Used during serialization to compress properties /// - /// - private void Compress(int contentTypeId, ContentNestedData nestedData) + /// + private void Compress(int contentTypeId, ContentCacheDataModel model) { - foreach(var propertyAliasToData in nestedData.PropertyData) + foreach(var propertyAliasToData in model.PropertyData) { if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) { @@ -100,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(int contentTypeId, ContentNestedData nestedData) + private void Expand(int contentTypeId, ContentCacheDataModel nestedData) { foreach (var propertyAliasToData in nestedData.PropertyData) { @@ -117,6 +114,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } + + //private class ContentNestedDataResolver : IFormatterResolver //{ // // GetFormatter's get cost should be minimized so use type cache. diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs new file mode 100644 index 0000000000..b509334604 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -0,0 +1,62 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using Umbraco.Core.Models; +using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Services; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class MsgPackContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory + { + private readonly IContentTypeService _contentTypeService; + private readonly IMediaTypeService _mediaTypeService; + private readonly IMemberTypeService _memberTypeService; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache = new ConcurrentDictionary<(int, string), CompressedStorageAttribute>(); + + public MsgPackContentNestedDataSerializerFactory(IContentTypeService contentTypeService, IMediaTypeService mediaTypeService, IMemberTypeService memberTypeService, PropertyEditorCollection propertyEditors) + { + _contentTypeService = contentTypeService; + _mediaTypeService = mediaTypeService; + _memberTypeService = memberTypeService; + _propertyEditors = propertyEditors; + } + + public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) + { + // Depending on which entity types are being requested, we need to look up those content types + // to initialize the compression options. + // We need to initialize these options now so that any data lookups required are completed and are not done while the content cache + // is performing DB queries which will result in errors since we'll be trying to query with open readers. + // NOTE: The calls to GetAll() below should be cached if the data has not been changed. + + var contentTypes = new Dictionary(); + if ((types & ContentCacheDataSerializerEntityType.Document) == ContentCacheDataSerializerEntityType.Document) + { + foreach(var ct in _contentTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + if ((types & ContentCacheDataSerializerEntityType.Media) == ContentCacheDataSerializerEntityType.Media) + { + foreach (var ct in _mediaTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + if ((types & ContentCacheDataSerializerEntityType.Member) == ContentCacheDataSerializerEntityType.Member) + { + foreach (var ct in _memberTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + + var options = new CompressedStoragePropertyEditorCompressionOptions(contentTypes, _propertyEditors, _compressedStoragePropertyEditorCache); + var serializer = new MsgPackContentNestedDataSerializer(options); + + return serializer; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index baa96a4a2e..faeb4f90b4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,10 +1,7 @@ -using System.Collections.Generic; -using System.Configuration; -using System.Linq; +using System.Configuration; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.PropertyEditors; -using Umbraco.Web.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Web.PublishedCache.NuCache @@ -19,13 +16,11 @@ namespace Umbraco.Web.PublishedCache.NuCache if (serializer != "MsgPack") { // TODO: This allows people to revert to the legacy serializer, by default it will be MessagePack - composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } else { - composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 41dfdd7a64..ad8705ef47 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1,14 +1,10 @@ using System; using System.Collections.Generic; using System.Configuration; -using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CSharpTest.Net.Collections; -using Newtonsoft.Json; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.Configuration; @@ -49,7 +45,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IPublishedModelFactory _publishedModelFactory; private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; - private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory; private readonly ContentDataSerializer _contentDataSerializer; // volatile because we read it with no lock @@ -84,7 +80,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer, ContentDataSerializer contentDataSerializer = null) + UrlSegmentProviderCollection urlSegmentProviders, IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -101,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _defaultCultureAccessor = defaultCultureAccessor; _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; - _contentNestedDataSerializer = contentNestedDataSerializer; + _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; _contentDataSerializer = contentDataSerializer; // we need an Xml serializer here so that the member cache can support XPath, @@ -1286,8 +1282,10 @@ namespace Umbraco.Web.PublishedCache.NuCache var db = args.Scope.Database; var content = (Content)args.Entity; + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // always refresh the edited data - OnRepositoryRefreshed(db, content, false); + OnRepositoryRefreshed(serializer, db, content, false); // if unpublishing, remove published data from table if (content.PublishedState == PublishedState.Unpublishing) @@ -1295,33 +1293,37 @@ namespace Umbraco.Web.PublishedCache.NuCache // if publishing, refresh the published data else if (content.PublishedState == PublishedState.Publishing) - OnRepositoryRefreshed(db, content, true); + OnRepositoryRefreshed(serializer, db, content, true); } private void OnMediaRefreshedEntity(MediaRepository sender, MediaRepository.ScopedEntityEventArgs args) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + var db = args.Scope.Database; var media = args.Entity; // refresh the edited data - OnRepositoryRefreshed(db, media, false); + OnRepositoryRefreshed(serializer, db, media, false); } private void OnMemberRefreshedEntity(MemberRepository sender, MemberRepository.ScopedEntityEventArgs args) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member); + var db = args.Scope.Database; var member = args.Entity; // refresh the edited data - OnRepositoryRefreshed(db, member, false); + OnRepositoryRefreshed(serializer, db, member, false); } - private void OnRepositoryRefreshed(IUmbracoDatabase db, IContentBase content, bool published) + private void OnRepositoryRefreshed(IContentCacheDataSerializer serializer, IUmbracoDatabase db, IContentBase content, bool published) { // use a custom SQL to update row version on each update //db.InsertOrUpdate(dto); - var dto = GetDto(content, published); + var dto = GetDto(content, published, serializer); db.InsertOrUpdate(dto, "SET data=@data, dataRaw=@dataRaw, rv=rv+1 WHERE nodeId=@id AND published=@published", new @@ -1375,7 +1377,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private ContentNuDto GetDto(IContentBase content, bool published) + private ContentNuDto GetDto(IContentBase content, bool published, IContentCacheDataSerializer serializer) { // should inject these in ctor // BUT for the time being we decide not to support ConvertDbToXml/String @@ -1447,19 +1449,21 @@ namespace Umbraco.Web.PublishedCache.NuCache } //the dictionary that will be serialized - var nestedData = new ContentNestedData + var contentCacheData = new ContentCacheDataModel { PropertyData = propertyData, CultureData = cultureData, UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; + var serialized = serializer.Serialize(content.ContentTypeId, contentCacheData); + var dto = new ContentNuDto { NodeId = content.Id, Published = published, - Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(content.ContentTypeId, nestedData) : null, - RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(content.ContentTypeId, nestedData) : null + Data = serialized.StringData, + RawData = serialized.ByteData }; //Core.Composing.Current.Logger.Debug(dto.Data); @@ -1482,30 +1486,32 @@ namespace Umbraco.Web.PublishedCache.NuCache public override void Rebuild() { _logger.Debug("Rebuilding..."); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document | ContentCacheDataSerializerEntityType.Media | ContentCacheDataSerializerEntityType.Member); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.ContentTree); scope.ReadLock(Constants.Locks.MediaTree); scope.ReadLock(Constants.Locks.MemberTree); - RebuildContentDbCacheLocked(scope, GetSqlPagingSize(), null); - RebuildMediaDbCacheLocked(scope, GetSqlPagingSize(), null); - RebuildMemberDbCacheLocked(scope, GetSqlPagingSize(), null); + RebuildContentDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); + RebuildMediaDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); + RebuildMemberDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); scope.Complete(); } } public void RebuildContentDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.ContentTree); - RebuildContentDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildContentDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes content tree lock - private void RebuildContentDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + private void RebuildContentDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var contentObjectType = Constants.ObjectTypes.Document; @@ -1552,11 +1558,11 @@ WHERE cmsContentNu.nodeId IN ( foreach (var c in descendants) { // always the edited version - items.Add(GetDto(c, false)); + items.Add(GetDto(c, false, serializer)); // and also the published version if it makes any sense if (c.Published) - items.Add(GetDto(c, true)); + items.Add(GetDto(c, true, serializer)); count++; } @@ -1568,16 +1574,17 @@ WHERE cmsContentNu.nodeId IN ( public void RebuildMediaDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.MediaTree); - RebuildMediaDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildMediaDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes media tree lock - public void RebuildMediaDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + public void RebuildMediaDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var mediaObjectType = Constants.ObjectTypes.Media; @@ -1619,7 +1626,7 @@ WHERE cmsContentNu.nodeId IN ( { // the tree is locked, counting and comparing to total is safe var descendants = _mediaRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path")); - var items = descendants.Select(m => GetDto(m, false)).ToList(); + var items = descendants.Select(m => GetDto(m, false, serializer)).ToList(); db.BulkInsertRecords(items); processed += items.Count; } while (processed < total); @@ -1627,16 +1634,17 @@ WHERE cmsContentNu.nodeId IN ( public void RebuildMemberDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.MemberTree); - RebuildMemberDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildMemberDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes member tree lock - public void RebuildMemberDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + public void RebuildMemberDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var memberObjectType = Constants.ObjectTypes.Member; @@ -1677,7 +1685,7 @@ WHERE cmsContentNu.nodeId IN ( do { var descendants = _memberRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path")); - var items = descendants.Select(m => GetDto(m, false)).ToArray(); + var items = descendants.Select(m => GetDto(m, false, serializer)).ToArray(); db.BulkInsertRecords(items); processed += items.Length; } while (processed < total); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 25478bf626..e93caaac66 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -262,12 +262,15 @@ - - + + + + + @@ -277,6 +280,7 @@ + @@ -588,7 +592,7 @@ - + From e3fd9b72c5c29798f52dbb34880fa59c5602ddb9 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sat, 24 Oct 2020 17:06:28 +1300 Subject: [PATCH 042/147] Optimize content and media count queries when paging --- .../NuCache/DataSource/DatabaseDataSource.cs | 66 ++++++++++++++++--- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index e60fd6623e..4c7cb57f19 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -80,7 +80,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// /// - private Sql ContentSourcesCount(IScope scope) + private Sql ContentSourcesCount(IScope scope, Func, Sql> joins = null) { var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesCount, tsql => tsql.Select(x => Alias(x.NodeId, "Id")) @@ -90,6 +90,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var sql = sqlTemplate.Sql(); + if (joins != null) + sql = joins(sql); + // TODO: We can't use a template with this one because of the 'right.Current' and 'right.Published' ends up being a parameter so not sure how we can do that sql = sql .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current) @@ -147,12 +150,20 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + // create a more efficient COUNT query without the join on the cmsContentNu table + var sqlCountQuery = ContentSourcesCount(scope, + s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")) + .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) + .Where(x => x.NodeId == id, "x"); + var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateContentNodeKit(row, serializer); } @@ -167,12 +178,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sqlCountQuery = ContentSourcesCount(scope) + .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) + .WhereIn(x => x.ContentTypeId, ids); + var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateContentNodeKit(row, serializer); } @@ -182,9 +198,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = scope.SqlContext.Sql() - .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), - x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), - x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) + .Select(x => Alias(x.NodeId, "Id")) .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) .AndSelect(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId")) .AndSelect("nuEdit", x => Alias(x.Data, "EditData")) @@ -201,6 +215,22 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } + private Sql MediaSourcesCount(IScope scope, Func, Sql> joins = null) + { + var sql = scope.SqlContext.Sql() + + .Select(x => Alias(x.NodeId, "Id")) + .From(); + + if (joins != null) + sql = joins(sql); + + sql = sql + .InnerJoin().On((left, right) => left.NodeId == right.NodeId) + .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current); + + return sql; + } public ContentNodeKit GetMediaSource(IScope scope, int id) { @@ -222,12 +252,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sqlCountQuery = MediaSourcesCount(scope) + .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed); + + var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateMediaNodeKit(row, serializer); } @@ -242,12 +277,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sqlCountQuery = MediaSourcesCount(scope, + s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")) + .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) + .Where(x => x.NodeId == id, "x"); + + var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateMediaNodeKit(row, serializer); } @@ -262,12 +304,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sqlCountQuery = MediaSourcesCount(scope) + .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) + .WhereIn(x => x.ContentTypeId, ids); + + var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateMediaNodeKit(row, serializer); } From da3d5093337068a5406115c3818c3de937167f17 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Wed, 18 Nov 2020 14:41:03 +1300 Subject: [PATCH 043/147] bug fix. --- .../PublishedCache/NuCache/DataSource/DatabaseDataSource.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 4c7cb57f19..98dbf1f85c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -198,7 +198,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = scope.SqlContext.Sql() - .Select(x => Alias(x.NodeId, "Id")) + .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), + x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), + x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) .AndSelect(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId")) .AndSelect("nuEdit", x => Alias(x.Data, "EditData")) From 95fe8bea23b9a1f4979623788b078b0f3f42aa7e Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 24 Nov 2020 10:09:27 +1300 Subject: [PATCH 044/147] Serialize more primitives --- .../NuCache/DataSource/SerializerBase.cs | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 8b02946fc2..e6ae789702 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -4,7 +4,7 @@ using CSharpTest.Net.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal abstract class SerializerBase + public abstract class SerializerBase { private const char PrefixNull = 'N'; private const char PrefixString = 'S'; @@ -18,6 +18,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private const char PrefixByte = 'O'; private const char PrefixByteArray = 'A'; private const char PrefixCompressedStringByteArray = 'C'; + private const char PrefixSignedByte = 'E'; + private const char PrefixBool = 'M'; + private const char PrefixGuid = 'G'; + private const char PrefixTimeSpan = 'T'; + private const char PrefixInt16 = 'Q'; + private const char PrefixChar = 'R'; protected string ReadString(Stream stream) => PrimitiveSerializer.String.ReadFrom(stream); protected int ReadInt(Stream stream) => PrimitiveSerializer.Int32.ReadFrom(stream); @@ -86,6 +92,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return PrimitiveSerializer.DateTime.ReadFrom(stream); case PrefixByteArray: return PrimitiveSerializer.Bytes.ReadFrom(stream); + case PrefixSignedByte: + return PrimitiveSerializer.SByte.ReadFrom(stream); + case PrefixBool: + return PrimitiveSerializer.Boolean.ReadFrom(stream); + case PrefixGuid: + return PrimitiveSerializer.Guid.ReadFrom(stream); + case PrefixTimeSpan: + return PrimitiveSerializer.TimeSpan.ReadFrom(stream); + case PrefixInt16: + return PrimitiveSerializer.Int16.ReadFrom(stream); + case PrefixChar: + return PrimitiveSerializer.Char.ReadFrom(stream); case PrefixCompressedStringByteArray: return new LazyCompressedString(PrimitiveSerializer.Bytes.ReadFrom(stream)); default: @@ -157,6 +175,36 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream); PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); } + else if (value is sbyte signedByteValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixSignedByte, stream); + PrimitiveSerializer.SByte.WriteTo(signedByteValue, stream); + } + else if (value is bool boolValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixBool, stream); + PrimitiveSerializer.Boolean.WriteTo(boolValue, stream); + } + else if (value is Guid guidValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixGuid, stream); + PrimitiveSerializer.Guid.WriteTo(guidValue, stream); + } + else if (value is TimeSpan timespanValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixTimeSpan, stream); + PrimitiveSerializer.TimeSpan.WriteTo(timespanValue, stream); + } + else if (value is short int16Value) + { + PrimitiveSerializer.Char.WriteTo(PrefixInt16, stream); + PrimitiveSerializer.Int16.WriteTo(int16Value, stream); + } + else if (value is char charValue) + { + PrimitiveSerializer.Char.WriteTo(PrefixChar, stream); + PrimitiveSerializer.Char.WriteTo(charValue, stream); + } else throw new NotSupportedException("Value type " + value.GetType().FullName + " cannot be serialized."); } From e8195bfff05af636d15e350ec39a9b49091d5b6f Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Tue, 24 Nov 2020 10:10:38 +1300 Subject: [PATCH 045/147] fix deserialization order --- .../MsgPackContentNestedDataSerializer.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index aad337b236..42468ad930 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -48,7 +48,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { - if (stringData != null) + if (byteData != null) + { + var content = MessagePackSerializer.Deserialize(byteData, _options); + Expand(contentTypeId, content); + return content; + } + else if (stringData != null) { // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); @@ -56,12 +62,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource Expand(contentTypeId, content); return content; } - else if (byteData != null) - { - var content = MessagePackSerializer.Deserialize(byteData, _options); - Expand(contentTypeId, content); - return content; - } else { return null; From f09fcfbdd4ed4ef9584ab7f4982a09c3fa15da74 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 27 Nov 2020 14:45:44 +1300 Subject: [PATCH 046/147] Fix serialization of decompressed lazy strings --- .../NuCache/DataSource/LazyCompressedString.cs | 9 ++++++++- .../NuCache/DataSource/SerializerBase.cs | 12 ++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 3e0e796d36..99e6f44af7 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -13,6 +13,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private byte[] _bytes; private string _str; private readonly object _locker; + private bool _isDecompressed; /// /// Constructor @@ -22,7 +23,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { _locker = new object(); _bytes = bytes; - _str = null; + _str = null; + _isDecompressed = false; } public byte[] GetBytes() @@ -31,6 +33,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource throw new InvalidOperationException("The bytes have already been expanded"); return _bytes; } + public bool IsDecompressed() + { + return _isDecompressed; + } public override string ToString() { @@ -41,6 +47,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (_bytes == null) throw new PanicException("Bytes have already been cleared"); _str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); _bytes = null; + _isDecompressed = true; } return _str; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index e6ae789702..f7e2f32f5d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -172,8 +172,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else if (value is LazyCompressedString lazyCompressedString) { - PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream); - PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); + if (lazyCompressedString.IsDecompressed()) + { + PrimitiveSerializer.Char.WriteTo(PrefixString, stream); + PrimitiveSerializer.String.WriteTo(lazyCompressedString, stream); + } + else + { + PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream); + PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); + } } else if (value is sbyte signedByteValue) { From c62b3f4497b579d27f3c41db67b51c88647f941b Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 16 Dec 2020 15:44:00 +1100 Subject: [PATCH 047/147] re-updates versions for db changes/migrations --- src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs | 7 ++++--- .../{V_8_9_0 => V_8_11_0}/AddCmsContentNuByteColumn.cs | 2 +- .../{V_8_9_0 => V_8_11_0}/UpgradedIncludeIndexes.cs | 8 ++------ src/Umbraco.Core/Umbraco.Core.csproj | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) rename src/Umbraco.Core/Migrations/Upgrade/{V_8_9_0 => V_8_11_0}/AddCmsContentNuByteColumn.cs (90%) rename src/Umbraco.Core/Migrations/Upgrade/{V_8_9_0 => V_8_11_0}/UpgradedIncludeIndexes.cs (95%) diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index 911cb612d4..a60b046212 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Migrations.Upgrade.V_8_1_0; using Umbraco.Core.Migrations.Upgrade.V_8_6_0; using Umbraco.Core.Migrations.Upgrade.V_8_9_0; using Umbraco.Core.Migrations.Upgrade.V_8_10_0; +using Umbraco.Core.Migrations.Upgrade.V_8_11_0; namespace Umbraco.Core.Migrations.Upgrade { @@ -199,12 +200,12 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.9.0 To("{B5838FF5-1D22-4F6C-BCEB-F83ACB14B575}"); - // to 8.9.0... - To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); - To("{4695D0C9-0729-4976-985B-048D503665D8}"); // to 8.10.0 To("{D6A8D863-38EC-44FB-91EC-ACD6A668BD18}"); + // to 8.11.0... + To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); + To("{4695D0C9-0729-4976-985B-048D503665D8}"); //FINAL } diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/AddCmsContentNuByteColumn.cs similarity index 90% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/AddCmsContentNuByteColumn.cs index c295e13051..dfde1f0577 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/AddCmsContentNuByteColumn.cs @@ -1,7 +1,7 @@ using System.Linq; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_9_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_11_0 { public class AddCmsContentNuByteColumn : MigrationBase { diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/UpgradedIncludeIndexes.cs similarity index 95% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/UpgradedIncludeIndexes.cs index 083294d1a5..6919558fc3 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/UpgradedIncludeIndexes.cs @@ -3,7 +3,7 @@ using Umbraco.Core.Migrations.Expressions.Execute.Expressions; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_9_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_11_0 { public class UpgradedIncludeIndexes : MigrationBase { @@ -44,13 +44,9 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_9_0 var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); foreach (var i in toDelete) - { if (IndexExists(i)) - { Delete.Index(i).OnTable(tableDef.Name).Do(); - } - } - + } private void CreateIndexes(params string[] toCreate) diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index c2cc3a0783..1958ea63b5 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -132,8 +132,8 @@ - - + + From 6b19dfe191e0da43ad189c7dc7878d9235b8eb27 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 16 Dec 2020 15:45:10 +1100 Subject: [PATCH 048/147] Fixing build --- src/Umbraco.Core/Umbraco.Core.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 1958ea63b5..a15fc69bc3 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -134,7 +134,6 @@ - From 78b563c79b47e6b9d5a54980f50881f209ff65b1 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 16 Dec 2020 16:12:02 +1100 Subject: [PATCH 049/147] Adds notes --- .../PublishedCache/IPublishedContentCache.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/IPublishedContentCache.cs b/src/Umbraco.Web/PublishedCache/IPublishedContentCache.cs index 8175285c3a..a31bdf34c5 100644 --- a/src/Umbraco.Web/PublishedCache/IPublishedContentCache.cs +++ b/src/Umbraco.Web/PublishedCache/IPublishedContentCache.cs @@ -43,16 +43,26 @@ namespace Umbraco.Web.PublishedCache /// /// A value indicating whether to consider unpublished content. /// The content unique identifier. - /// The route. - /// The value of overrides defaults. + /// A special string formatted route path. + /// + /// + /// The resulting string is a special encoded route string that may contain the domain ID + /// for the current route. If a domain is present the string will be prefixed with the domain ID integer, example: {domainId}/route-path-of-item + /// + /// The value of overrides defaults. + /// string GetRouteById(bool preview, int contentId, string culture = null); /// /// Gets the route for a content identified by its unique identifier. /// /// The content unique identifier. - /// The route. + /// A special string formatted route path. /// Considers published or unpublished content depending on defaults. + /// + /// The resulting string is a special encoded route string that may contain the domain ID + /// for the current route. If a domain is present the string will be prefixed with the domain ID integer, example: {domainId}/route-path-of-item + /// string GetRouteById(int contentId, string culture = null); } } From c3656d3e58d16a41d0405ed8b99828d1e687fc03 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 16 Dec 2020 16:52:43 +1100 Subject: [PATCH 050/147] Adds notes fixes tests --- .../ContentCacheDataSerializationResult.cs | 4 ++++ .../DataSource/IContentCacheDataSerializer.cs | 14 ++++++++++++++ .../DataSource/JsonContentNestedDataSerializer.cs | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs index 7cd388d712..cde39eaa3c 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs @@ -3,6 +3,10 @@ using System.Collections.Generic; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// + /// The serialization result from for which the serialized value + /// will be either a string or a byte[] + /// public struct ContentCacheDataSerializationResult : IEquatable { public ContentCacheDataSerializationResult(string stringData, byte[] byteData) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index 87ac5af91e..f628c8981b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -11,7 +11,21 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// public interface IContentCacheDataSerializer { + /// + /// Deserialize the data into a + /// + /// + /// + /// + /// ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData); + + /// + /// Serializes the + /// + /// + /// + /// ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model); } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 2fa892a5e6..47f07b8b1d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -10,7 +10,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { - if (byteData != null) + if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); // by default JsonConvert will deserialize our numeric values as Int64 From f8bb53ac0338d41241659a9771d7e076bd2891c3 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Wed, 16 Dec 2020 19:20:24 +1300 Subject: [PATCH 051/147] changed isdecompressed to a property. Changed serializerbase to internal --- .../NuCache/DataSource/LazyCompressedString.cs | 9 +++++---- .../PublishedCache/NuCache/DataSource/SerializerBase.cs | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 99e6f44af7..ea904b0cf4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -33,10 +33,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource throw new InvalidOperationException("The bytes have already been expanded"); return _bytes; } - public bool IsDecompressed() - { - return _isDecompressed; - } + /// + /// Whether the bytes have been decompressed to a string. If true calling GetBytes() will throw InvalidOperationException. + /// + public bool IsDecompressed => _isDecompressed; + public override string ToString() { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index f7e2f32f5d..9e6baed4fe 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -4,7 +4,7 @@ using CSharpTest.Net.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - public abstract class SerializerBase + internal abstract class SerializerBase { private const char PrefixNull = 'N'; private const char PrefixString = 'S'; From 697206188f4933c64a4a66a11c85b5eecad54b43 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Wed, 16 Dec 2020 19:23:14 +1300 Subject: [PATCH 052/147] use property --- .../PublishedCache/NuCache/DataSource/SerializerBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 9e6baed4fe..cafb40657d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -172,7 +172,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else if (value is LazyCompressedString lazyCompressedString) { - if (lazyCompressedString.IsDecompressed()) + if (lazyCompressedString.IsDecompressed) { PrimitiveSerializer.Char.WriteTo(PrefixString, stream); PrimitiveSerializer.String.WriteTo(lazyCompressedString, stream); From 025bcf2f4cc00681cad4d80392ea1f33199e2bd7 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 16 Dec 2020 17:50:34 +1100 Subject: [PATCH 053/147] notes --- .../NuCache/DataSource/BTree.cs | 1 + .../NuCache/DataSource/SerializerBase.cs | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs index 80d8488495..7ddfc8e6ea 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.cs @@ -30,6 +30,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource //btree. return tree; + } private static int GetBlockSize() diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 8b02946fc2..0cd095d77f 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -57,11 +57,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource protected object ReadObject(Stream stream) => ReadObject(PrimitiveSerializer.Char.ReadFrom(stream), stream); + /// + /// Reads in a value based on its char type + /// + /// + /// + /// + /// + /// This will incur boxing because the result is an object but in most cases the value will be a struct. + /// When the type is known use the specific methods like instead + /// protected object ReadObject(char type, Stream stream) { - // NOTE: This method is only called when reading property data, some boxing may occur but all other reads for structs are - // done with ReadStruct to reduce all boxing. - switch (type) { case PrefixNull: @@ -93,11 +100,17 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } + /// + /// Writes a value to the stream ensuring it's char type is prefixed to the value for reading later + /// + /// + /// + /// + /// This method will incur boxing if the value is a struct. When the type is known use the + /// to write the value directly. + /// protected void WriteObject(object value, Stream stream) { - // NOTE: This method is only currently used to write 'string' information, all other writes are done directly with the PrimitiveSerializer - // so no boxing occurs. Though potentially we should write everything via this class just like we do for reads. - if (value == null) { PrimitiveSerializer.Char.WriteTo(PrefixNull, stream); From dd111226090642cd6777630ff45e432ce8349656 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sun, 17 Jan 2021 21:33:35 +1300 Subject: [PATCH 054/147] Huge improvement in speed and memory allocations for localized text service. --- .../Compose/RelateOnTrashComponent.cs | 5 +- .../Services/ILocalizedTextService.cs | 14 + .../Implement/LocalizedTextService.cs | 241 ++++---- .../LocalizedTextServiceExtensions.cs | 67 ++- ...TextServiceGetAllStoredValuesBenchmarks.cs | 533 ++++++++++++++++++ .../Umbraco.Tests.Benchmarks.csproj | 1 + .../Mapping/ContentWebModelMappingTests.cs | 4 +- .../Compose/NotificationsComponent.cs | 8 +- .../Editors/AuthenticationController.cs | 2 +- .../Editors/BackOfficeController.cs | 2 +- src/Umbraco.Web/Editors/CodeFileController.cs | 10 +- src/Umbraco.Web/Editors/ContentController.cs | 134 ++--- .../Editors/ContentTypeController.cs | 6 +- .../Editors/ContentTypeControllerBase.cs | 6 +- .../Editors/CurrentUserController.cs | 2 +- src/Umbraco.Web/Editors/DataTypeController.cs | 4 +- src/Umbraco.Web/Editors/MediaController.cs | 26 +- .../Editors/MediaTypeController.cs | 2 +- src/Umbraco.Web/Editors/MemberController.cs | 2 +- .../Editors/MemberGroupController.cs | 2 +- .../Editors/MemberTypeController.cs | 2 +- .../Editors/PackageInstallController.cs | 10 +- .../Editors/TemplateQueryController.cs | 40 +- .../Editors/UserGroupsController.cs | 6 +- src/Umbraco.Web/Editors/UsersController.cs | 24 +- .../Checks/Config/AbstractConfigCheck.cs | 16 +- .../Checks/Config/CompilationDebugCheck.cs | 6 +- .../Checks/Config/ConfigurationService.cs | 12 +- .../Checks/Config/CustomErrorsCheck.cs | 6 +- .../Checks/Config/MacroErrorsCheck.cs | 6 +- .../Checks/Config/NotificationEmailCheck.cs | 4 +- .../HealthCheck/Checks/Config/TraceCheck.cs | 6 +- .../Config/TrySkipIisCustomErrorsCheck.cs | 6 +- .../FolderAndFilePermissionsCheck.cs | 20 +- .../Checks/Security/BaseHttpHeaderCheck.cs | 14 +- .../Checks/Security/ExcessiveHeadersCheck.cs | 6 +- .../HealthCheck/Checks/Security/HttpsCheck.cs | 28 +- .../HealthCheck/Checks/Services/SmtpCheck.cs | 8 +- .../EmailNotificationMethod.cs | 4 +- src/Umbraco.Web/Macros/MacroRenderer.cs | 4 +- .../Models/Mapping/CommonMapper.cs | 2 +- .../Models/Mapping/ContentVariantMapper.cs | 2 +- .../Mapping/MemberTabsAndPropertiesMapper.cs | 16 +- .../Models/Mapping/SectionMapDefinition.cs | 2 +- .../Models/Mapping/TabsAndPropertiesMapper.cs | 2 +- .../Models/Mapping/UserMapDefinition.cs | 26 +- src/Umbraco.Web/Models/Trees/MenuItem.cs | 7 +- src/Umbraco.Web/Models/Trees/MenuItemList.cs | 6 +- .../BlockEditorPropertyEditor.cs | 4 +- .../DropDownFlexibleConfigurationEditor.cs | 2 +- .../UploadFileTypeValidator.cs | 2 +- .../ValueListConfigurationEditor.cs | 2 +- .../Routing/UrlProviderExtensions.cs | 14 +- .../Trees/ApplicationTreeController.cs | 6 +- .../Trees/ContentTreeControllerBase.cs | 2 +- .../Trees/DataTypeTreeController.cs | 2 +- .../Trees/MediaTypeTreeController.cs | 2 +- src/Umbraco.Web/Trees/MemberTreeController.cs | 2 +- src/Umbraco.Web/Trees/Tree.cs | 2 +- 59 files changed, 1002 insertions(+), 400 deletions(-) create mode 100644 src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs diff --git a/src/Umbraco.Core/Compose/RelateOnTrashComponent.cs b/src/Umbraco.Core/Compose/RelateOnTrashComponent.cs index a216a584ba..27eb559462 100644 --- a/src/Umbraco.Core/Compose/RelateOnTrashComponent.cs +++ b/src/Umbraco.Core/Compose/RelateOnTrashComponent.cs @@ -93,7 +93,8 @@ namespace Umbraco.Core.Compose item.Entity.Id, ObjectTypes.GetName(UmbracoObjectTypes.Document), string.Format(textService.Localize( - "recycleBin/contentTrashed"), + "recycleBin","contentTrashed"), + item.Entity.Id, originalParentId)); } } @@ -132,7 +133,7 @@ namespace Umbraco.Core.Compose item.Entity.Id, ObjectTypes.GetName(UmbracoObjectTypes.Media), string.Format(textService.Localize( - "recycleBin/mediaTrashed"), + "recycleBin", "mediaTrashed"), item.Entity.Id, originalParentId)); } } diff --git a/src/Umbraco.Core/Services/ILocalizedTextService.cs b/src/Umbraco.Core/Services/ILocalizedTextService.cs index f167c64e0f..a4e5141a42 100644 --- a/src/Umbraco.Core/Services/ILocalizedTextService.cs +++ b/src/Umbraco.Core/Services/ILocalizedTextService.cs @@ -4,6 +4,20 @@ using System.Globalization; namespace Umbraco.Core.Services { + + public interface ILocalizedTextService2 : ILocalizedTextService + { + /// + /// Localize a key with variables + /// + /// + /// + /// + /// This can be null + /// + string Localize(string area, string alias, CultureInfo culture, IDictionary tokens = null); + } + /// /// The entry point to localize any key in the text storage source for a given culture /// diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index 4f5121def7..c099960397 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -10,12 +10,13 @@ namespace Umbraco.Core.Services.Implement { // TODO: Convert all of this over to Niels K's localization framework one day - public class LocalizedTextService : ILocalizedTextService + public class LocalizedTextService : ILocalizedTextService2 { private readonly ILogger _logger; private readonly Lazy _fileSources; private readonly IDictionary>> _dictionarySource; - private readonly IDictionary> _xmlSource; + private readonly IDictionary> _noAreaDictionarySource; + private readonly char[] _splitter = new[] { '/' }; /// /// Initializes with a file sources instance @@ -27,9 +28,40 @@ namespace Umbraco.Core.Services.Implement if (logger == null) throw new ArgumentNullException("logger"); _logger = logger; if (fileSources == null) throw new ArgumentNullException("fileSources"); + var dictionaries = FileSourcesToDictionarySources(fileSources.Value); + _dictionarySource = dictionaries.WithArea; + _noAreaDictionarySource = dictionaries.WithoutArea; _fileSources = fileSources; } + private (IDictionary>> WithArea, IDictionary> WithoutArea) FileSourcesToDictionarySources(LocalizedTextServiceFileSources fileSources) + { + var xmlSources = fileSources.GetXmlSources(); + return XmlSourcesToDictionarySources(xmlSources); + } + private (IDictionary>> WithArea, IDictionary> WithoutArea) XmlSourcesToDictionarySources(IDictionary> source) + { + var cultureDictionary = new Dictionary>>(); + var cultureNoAreaDictionary = new Dictionary>(); + foreach (var xmlSource in source) + { + var areaAliaValue = GetAreaStoredTranslations(source, xmlSource.Key); + cultureDictionary.Add(xmlSource.Key, areaAliaValue); + var aliasValue = new Dictionary(); + foreach (var area in areaAliaValue) + { + foreach (var alias in area.Value) + { + if (!aliasValue.ContainsKey(alias.Key)) + { + aliasValue.Add(alias.Key, alias.Value); + } + } + } + cultureNoAreaDictionary.Add(xmlSource.Key, aliasValue); + } + return (cultureDictionary, cultureNoAreaDictionary); + } /// /// Initializes with an XML source /// @@ -39,10 +71,13 @@ namespace Umbraco.Core.Services.Implement { if (source == null) throw new ArgumentNullException("source"); if (logger == null) throw new ArgumentNullException("logger"); - _xmlSource = source; _logger = logger; + var dictionaries = XmlSourcesToDictionarySources(source); + _dictionarySource = dictionaries.WithArea; + _noAreaDictionarySource = dictionaries.WithoutArea; } + /// /// Initializes with a source of a dictionary of culture -> areas -> sub dictionary of keys/values /// @@ -52,35 +87,50 @@ namespace Umbraco.Core.Services.Implement { _dictionarySource = source ?? throw new ArgumentNullException(nameof(source)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + var cultureNoAreaDictionary = new Dictionary>(); + foreach (var cultureDictionary in _dictionarySource) + { + var areaAliaValue = GetAreaStoredTranslations(source, cultureDictionary.Key); + var aliasValue = new Dictionary(); + foreach (var area in areaAliaValue) + { + foreach (var alias in area.Value) + { + if (!aliasValue.ContainsKey(alias.Key)) + { + aliasValue.Add(alias.Key, alias.Value); + } + } + } + cultureNoAreaDictionary.Add(cultureDictionary.Key, aliasValue); + } + _noAreaDictionarySource = cultureNoAreaDictionary; } public string Localize(string key, CultureInfo culture, IDictionary tokens = null) { if (culture == null) throw new ArgumentNullException(nameof(culture)); - - // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode - culture = ConvertToSupportedCultureWithRegionCode(culture); - - //This is what the legacy ui service did - if (string.IsNullOrEmpty(key)) - return string.Empty; - - var keyParts = key.Split(new[] {'/'}, StringSplitOptions.RemoveEmptyEntries); + var keyParts = key.Split(_splitter, StringSplitOptions.RemoveEmptyEntries); var area = keyParts.Length > 1 ? keyParts[0] : null; var alias = keyParts.Length > 1 ? keyParts[1] : keyParts[0]; - - var xmlSource = _xmlSource ?? (_fileSources != null - ? _fileSources.Value.GetXmlSources() - : null); - - if (xmlSource != null) - { - return GetFromXmlSource(xmlSource, culture, area, alias, tokens); - } - else + return Localize(area, alias, culture, tokens); + } + public string Localize(string area, string alias, CultureInfo culture, IDictionary tokens = null) + { + if (culture == null) throw new ArgumentNullException(nameof(culture)); + var sw = System.Diagnostics.Stopwatch.StartNew(); + try { + // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode + culture = ConvertToSupportedCultureWithRegionCode(culture); + return GetFromDictionarySource(culture, area, alias, tokens); } + finally + { + sw.Stop(); + System.Diagnostics.Debug.WriteLine($"Localize {area}/{alias} ({tokens?.Count}) ({sw.ElapsedTicks})"); + } } /// @@ -96,49 +146,23 @@ namespace Umbraco.Core.Services.Implement var result = new Dictionary(); - var xmlSource = _xmlSource ?? (_fileSources != null - ? _fileSources.Value.GetXmlSources() - : null); - if (xmlSource != null) + if (_dictionarySource.ContainsKey(culture) == false) { - if (xmlSource.ContainsKey(culture) == false) - { - _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); - return result; - } - - //convert all areas + keys to a single key with a '/' - result = GetStoredTranslations(xmlSource, culture); - - //merge with the English file in case there's keys in there that don't exist in the local file - var englishCulture = new CultureInfo("en-US"); - if (culture.Equals(englishCulture) == false) - { - var englishResults = GetStoredTranslations(xmlSource, englishCulture); - foreach (var englishResult in englishResults.Where(englishResult => result.ContainsKey(englishResult.Key) == false)) - result.Add(englishResult.Key, englishResult.Value); - } + _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); + return result; } - else - { - if (_dictionarySource.ContainsKey(culture) == false) - { - _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); - return result; - } - //convert all areas + keys to a single key with a '/' - foreach (var area in _dictionarySource[culture]) + //convert all areas + keys to a single key with a '/' + foreach (var area in _dictionarySource[culture]) + { + foreach (var key in area.Value) { - foreach (var key in area.Value) + var dictionaryKey = string.Format("{0}/{1}", area.Key, key.Key); + //i don't think it's possible to have duplicates because we're dealing with a dictionary in the first place, but we'll double check here just in case. + if (result.ContainsKey(dictionaryKey) == false) { - var dictionaryKey = string.Format("{0}/{1}", area.Key, key.Key); - //i don't think it's possible to have duplicates because we're dealing with a dictionary in the first place, but we'll double check here just in case. - if (result.ContainsKey(dictionaryKey) == false) - { - result.Add(dictionaryKey, key.Value); - } + result.Add(dictionaryKey, key.Value); } } } @@ -146,23 +170,44 @@ namespace Umbraco.Core.Services.Implement return result; } - private Dictionary GetStoredTranslations(IDictionary> xmlSource, CultureInfo cult) + private Dictionary> GetAreaStoredTranslations(IDictionary> xmlSource, CultureInfo cult) { - var result = new Dictionary(); + var overallResult = new Dictionary>(); var areas = xmlSource[cult].Value.XPathSelectElements("//area"); foreach (var area in areas) { + var result = new Dictionary(); var keys = area.XPathSelectElements("./key"); foreach (var key in keys) { - var dictionaryKey = string.Format("{0}/{1}", (string)area.Attribute("alias"), - (string)key.Attribute("alias")); + var dictionaryKey = + (string)key.Attribute("alias"); //there could be duplicates if the language file isn't formatted nicely - which is probably the case for quite a few lang files if (result.ContainsKey(dictionaryKey) == false) result.Add(dictionaryKey, key.Value); } + overallResult.Add(area.Attribute("alias").Value, result); } - return result; + return overallResult; + } + private Dictionary> GetAreaStoredTranslations(IDictionary>> dictionarySource, CultureInfo cult) + { + var overallResult = new Dictionary>(); + var areaDict = dictionarySource[cult]; + + foreach (var area in areaDict) + { + var result = new Dictionary(); + var keys = area.Value.Keys; + foreach (var key in keys) + { + //there could be duplicates if the language file isn't formatted nicely - which is probably the case for quite a few lang files + if (result.ContainsKey(key) == false) + result.Add(key, area.Value[key]); + } + overallResult.Add(area.Key, result); + } + return overallResult; } /// @@ -171,11 +216,7 @@ namespace Umbraco.Core.Services.Implement /// public IEnumerable GetSupportedCultures() { - var xmlSource = _xmlSource ?? (_fileSources != null - ? _fileSources.Value.GetXmlSources() - : null); - - return xmlSource != null ? xmlSource.Keys : _dictionarySource.Keys; + return _dictionarySource.Keys; } /// @@ -211,27 +252,21 @@ namespace Umbraco.Core.Services.Implement return "[" + key + "]"; } - var cultureSource = _dictionarySource[culture]; - string found; - if (area.IsNullOrWhiteSpace()) + string found = null; + if (string.IsNullOrWhiteSpace(area)) { - found = cultureSource - .SelectMany(x => x.Value) - .Where(keyvals => keyvals.Key.InvariantEquals(key)) - .Select(x => x.Value) - .FirstOrDefault(); + _noAreaDictionarySource[culture].TryGetValue(key, out found); } else { - found = cultureSource - .Where(areas => areas.Key.InvariantEquals(area)) - .SelectMany(a => a.Value) - .Where(keyvals => keyvals.Key.InvariantEquals(key)) - .Select(x => x.Value) - .FirstOrDefault(); + if (_dictionarySource[culture].TryGetValue(area, out var areaDictionary)) + { + areaDictionary.TryGetValue(key, out found); + } } + if (found != null) { return ParseTokens(found, tokens); @@ -240,44 +275,6 @@ namespace Umbraco.Core.Services.Implement //NOTE: Based on how legacy works, the default text does not contain the area, just the key return "[" + key + "]"; } - - private string GetFromXmlSource(IDictionary> xmlSource, CultureInfo culture, string area, string key, IDictionary tokens) - { - if (xmlSource.ContainsKey(culture) == false) - { - _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); - return "[" + key + "]"; - } - - var found = FindTranslation(xmlSource, culture, area, key); - - if (found != null) - { - return ParseTokens(found.Value, tokens); - } - - // Fall back to English by default if we can't find the key - found = FindTranslation(xmlSource, new CultureInfo("en-US"), area, key); - if (found != null) - return ParseTokens(found.Value, tokens); - - // If it can't be found in either file, fall back to the default, showing just the key in square brackets - // NOTE: Based on how legacy works, the default text does not contain the area, just the key - return "[" + key + "]"; - } - - private XElement FindTranslation(IDictionary> xmlSource, CultureInfo culture, string area, string key) - { - var cultureSource = xmlSource[culture].Value; - - var xpath = area.IsNullOrWhiteSpace() - ? string.Format("//key [@alias = '{0}']", key) - : string.Format("//area [@alias = '{0}']/key [@alias = '{1}']", area, key); - - var found = cultureSource.XPathSelectElement(xpath); - return found; - } - /// /// Parses the tokens in the value /// @@ -301,7 +298,7 @@ namespace Umbraco.Core.Services.Implement foreach (var token in tokens) { - value = value.Replace(string.Format("{0}{1}{0}", "%", token.Key), token.Value); + value = value.Replace(string.Concat("%", token.Key, "%"), token.Value); } return value; diff --git a/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs b/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs index ce5b3ef8c4..c6254f7baa 100644 --- a/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs +++ b/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs @@ -12,12 +12,75 @@ namespace Umbraco.Core.Services /// public static class LocalizedTextServiceExtensions { - public static string Localize(this ILocalizedTextService manager, string area, string key) + public static string Localize(this ILocalizedTextService manager, string area, string alias, CultureInfo culture) { - var fullKey = string.Join("/", area, key); + if(manager is ILocalizedTextService2 manager2) + { + return manager2.Localize(area, alias, culture); + } + var fullKey = string.Concat(area, "/", alias); + return manager.Localize(fullKey, culture); + } + public static string Localize(this ILocalizedTextService manager, string area, string alias) + { + if (manager is ILocalizedTextService2 manager2) + { + return manager2.Localize(area, alias, Thread.CurrentThread.CurrentUICulture); + } + var fullKey = string.Concat(area, "/", alias); return manager.Localize(fullKey, Thread.CurrentThread.CurrentUICulture); } + /// + /// Localize using the current thread culture + /// + /// + /// + /// + /// + /// + public static string Localize(this ILocalizedTextService manager, string area, string alias, string[] tokens) + { + if (manager is ILocalizedTextService2 manager2) + { + return manager2.Localize(area, alias, Thread.CurrentThread.CurrentUICulture, ConvertToDictionaryVars(tokens)); + } + return manager.Localize(string.Join("/",area, alias), Thread.CurrentThread.CurrentUICulture, tokens); + } + /// + /// Localize using the current thread culture + /// + /// + /// + /// + /// + /// + public static string Localize(this ILocalizedTextService manager, string area, string alias, IDictionary tokens = null) + { + if (manager is ILocalizedTextService2 manager2) + { + return manager2.Localize(area, alias, Thread.CurrentThread.CurrentUICulture, tokens); + } + return manager.Localize(string.Join("/", area, alias), Thread.CurrentThread.CurrentUICulture, tokens); + } + + /// + /// Localize a key without any variables + /// + /// + /// + /// + /// + /// + /// + public static string Localize(this ILocalizedTextService manager, string area, string alias, CultureInfo culture, string[] tokens) + { + if (manager is ILocalizedTextService2 manager2) + { + return manager2.Localize(area, alias, Thread.CurrentThread.CurrentUICulture, tokens); + } + return manager.Localize(string.Join("/", area, alias), culture, ConvertToDictionaryVars(tokens)); + } /// /// Localize using the current thread culture /// diff --git a/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs b/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs new file mode 100644 index 0000000000..e084931d9e --- /dev/null +++ b/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs @@ -0,0 +1,533 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Loggers; +using Moq; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Services.Implement; +using System.Xml.Linq; +using Umbraco.Core.Logging; +using Umbraco.Tests.Benchmarks.Config; +using Umbraco.Core.Services; +using Umbraco.Core; +using System.Xml.XPath; +using ILogger = Umbraco.Core.Logging.ILogger; + +namespace Umbraco.Tests.Benchmarks +{ + [QuickRunWithMemoryDiagnoserConfig] + public class LocalizedTextServiceGetAllStoredValuesBenchmarks + { + private CultureInfo culture; + private OldLocalizedTextService _dictionaryService; + private OldLocalizedTextService _xmlService; + + private LocalizedTextService _optimized; + private LocalizedTextService _optimizedDict; + [GlobalSetup] + public void Setup() + { + culture = CultureInfo.GetCultureInfo("en-US"); + _dictionaryService = GetDictionaryLocalizedTextService(culture); + _xmlService = GetXmlService(culture); + _optimized = GetOptimizedService(culture); + _optimizedDict = GetOptimizedServiceDict(culture); + var result1 = _dictionaryService.Localize("language", culture); + var result2 = _xmlService.Localize("language", culture); + var result3 = _dictionaryService.GetAllStoredValues(culture); + var result4 = _xmlService.GetAllStoredValues(culture); + var result5 = _optimized.GetAllStoredValues(culture); + var result6 = _xmlService.GetAllStoredValues(culture); + } + + [Benchmark] + public void OriginalDictionaryGetAll() + { + for (int i = 0; i < 10000; i++) + { + var result = _dictionaryService.GetAllStoredValues(culture); + } + + } + + [Benchmark] + public void OriginalXmlGetAll() + { + for (int i = 0; i < 10000; i++) + { + var result = _xmlService.GetAllStoredValues(culture); + } + + } + + [Benchmark] + public void OriginalDictionaryLocalize() + { + for (int i = 0; i < 10000; i++) + { + var result = _dictionaryService.Localize("language", culture); + } + + } + + + [Benchmark(Baseline = true)] + public void OriginalXmlLocalize() + { + for (int i = 0; i < 10000; i++) + { + var result = _xmlService.Localize("language", culture); + } + } + [Benchmark()] + public void OptimizedXmlLocalize() + { + for (int i = 0; i < 10000; i++) + { + var result = _optimized.Localize(null, "language", culture); + } + } + [Benchmark()] + public void OptimizedDictLocalize() + { + for (int i = 0; i < 10000; i++) + { + var result = _optimizedDict.Localize(null, "language", culture); + } + } + + private static LocalizedTextService GetOptimizedServiceDict(CultureInfo culture) + { + return new LocalizedTextService( + new Dictionary>> + { + { + culture, new Dictionary> + { + { + "testArea1", new Dictionary + { + {"testKey1", "testValue1"}, + {"testKey2", "testValue2"} + } + }, + { + "testArea2", new Dictionary + { + {"blah1", "blahValue1"}, + {"blah2", "blahValue2"} + } + }, + } + } + }, Mock.Of()); + } + private static LocalizedTextService GetOptimizedService(CultureInfo culture) + { + var txtService = new LocalizedTextService(new Dictionary> + { + { + culture, new Lazy(() => new XDocument( + new XElement("language", + new XElement("area", new XAttribute("alias", "testArea1"), + new XElement("key", new XAttribute("alias", "testKey1"), "testValue1"), + new XElement("key", new XAttribute("alias", "testKey2"), "testValue2")), + new XElement("area", new XAttribute("alias", "testArea2"), + new XElement("key", new XAttribute("alias", "blah1"), "blahValue1"), + new XElement("key", new XAttribute("alias", "blah2"), "blahValue2"))))) + } + }, Mock.Of()); + return txtService; + } + + private static OldLocalizedTextService GetXmlService(CultureInfo culture) + { + var txtService = new OldLocalizedTextService(new Dictionary> + { + { + culture, new Lazy(() => new XDocument( + new XElement("language", + new XElement("area", new XAttribute("alias", "testArea1"), + new XElement("key", new XAttribute("alias", "testKey1"), "testValue1"), + new XElement("key", new XAttribute("alias", "testKey2"), "testValue2")), + new XElement("area", new XAttribute("alias", "testArea2"), + new XElement("key", new XAttribute("alias", "blah1"), "blahValue1"), + new XElement("key", new XAttribute("alias", "blah2"), "blahValue2"))))) + } + }, Mock.Of()); + return txtService; + } + + private static OldLocalizedTextService GetDictionaryLocalizedTextService(CultureInfo culture) + { + return new OldLocalizedTextService( + new Dictionary>> + { + { + culture, new Dictionary> + { + { + "testArea1", new Dictionary + { + {"testKey1", "testValue1"}, + {"testKey2", "testValue2"} + } + }, + { + "testArea2", new Dictionary + { + {"blah1", "blahValue1"}, + {"blah2", "blahValue2"} + } + }, + } + } + }, Mock.Of()); + } + } + + //Original + public class OldLocalizedTextService : ILocalizedTextService + { + private readonly ILogger _logger; + private readonly Lazy _fileSources; + private readonly IDictionary>> _dictionarySource; + private readonly IDictionary> _xmlSource; + + /// + /// Initializes with a file sources instance + /// + /// + /// + public OldLocalizedTextService(Lazy fileSources, ILogger logger) + { + if (logger == null) throw new ArgumentNullException("logger"); + _logger = logger; + if (fileSources == null) throw new ArgumentNullException("fileSources"); + _fileSources = fileSources; + } + + /// + /// Initializes with an XML source + /// + /// + /// + public OldLocalizedTextService(IDictionary> source, ILogger logger) + { + if (source == null) throw new ArgumentNullException("source"); + if (logger == null) throw new ArgumentNullException("logger"); + _xmlSource = source; + _logger = logger; + } + + /// + /// Initializes with a source of a dictionary of culture -> areas -> sub dictionary of keys/values + /// + /// + /// + public OldLocalizedTextService(IDictionary>> source, ILogger logger) + { + _dictionarySource = source ?? throw new ArgumentNullException(nameof(source)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Localize(string key, CultureInfo culture, IDictionary tokens = null) + { + if (culture == null) throw new ArgumentNullException(nameof(culture)); + + // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode + culture = ConvertToSupportedCultureWithRegionCode(culture); + + //This is what the legacy ui service did + if (string.IsNullOrEmpty(key)) + return string.Empty; + + var keyParts = key.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + var area = keyParts.Length > 1 ? keyParts[0] : null; + var alias = keyParts.Length > 1 ? keyParts[1] : keyParts[0]; + + var xmlSource = _xmlSource ?? (_fileSources != null + ? _fileSources.Value.GetXmlSources() + : null); + + if (xmlSource != null) + { + return GetFromXmlSource(xmlSource, culture, area, alias, tokens); + } + else + { + return GetFromDictionarySource(culture, area, alias, tokens); + } + } + + /// + /// Returns all key/values in storage for the given culture + /// + /// + public IDictionary GetAllStoredValues(CultureInfo culture) + { + if (culture == null) throw new ArgumentNullException("culture"); + + // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode + culture = ConvertToSupportedCultureWithRegionCode(culture); + + var result = new Dictionary(); + + var xmlSource = _xmlSource ?? (_fileSources != null + ? _fileSources.Value.GetXmlSources() + : null); + + if (xmlSource != null) + { + if (xmlSource.ContainsKey(culture) == false) + { + _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); + return result; + } + + //convert all areas + keys to a single key with a '/' + result = GetStoredTranslations(xmlSource, culture); + + //merge with the English file in case there's keys in there that don't exist in the local file + var englishCulture = new CultureInfo("en-US"); + if (culture.Equals(englishCulture) == false) + { + var englishResults = GetStoredTranslations(xmlSource, englishCulture); + foreach (var englishResult in englishResults.Where(englishResult => result.ContainsKey(englishResult.Key) == false)) + result.Add(englishResult.Key, englishResult.Value); + } + } + else + { + if (_dictionarySource.ContainsKey(culture) == false) + { + _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); + return result; + } + + //convert all areas + keys to a single key with a '/' + foreach (var area in _dictionarySource[culture]) + { + foreach (var key in area.Value) + { + var dictionaryKey = string.Format("{0}/{1}", area.Key, key.Key); + //i don't think it's possible to have duplicates because we're dealing with a dictionary in the first place, but we'll double check here just in case. + if (result.ContainsKey(dictionaryKey) == false) + { + result.Add(dictionaryKey, key.Value); + } + } + } + } + + return result; + } + + private Dictionary GetStoredTranslations(IDictionary> xmlSource, CultureInfo cult) + { + var result = new Dictionary(); + var areas = xmlSource[cult].Value.XPathSelectElements("//area"); + foreach (var area in areas) + { + var keys = area.XPathSelectElements("./key"); + foreach (var key in keys) + { + var dictionaryKey = string.Format("{0}/{1}", (string)area.Attribute("alias"), + (string)key.Attribute("alias")); + //there could be duplicates if the language file isn't formatted nicely - which is probably the case for quite a few lang files + if (result.ContainsKey(dictionaryKey) == false) + result.Add(dictionaryKey, key.Value); + } + } + return result; + } + + /// + /// Returns a list of all currently supported cultures + /// + /// + public IEnumerable GetSupportedCultures() + { + var xmlSource = _xmlSource ?? (_fileSources != null + ? _fileSources.Value.GetXmlSources() + : null); + + return xmlSource != null ? xmlSource.Keys : _dictionarySource.Keys; + } + + /// + /// Tries to resolve a full 4 letter culture from a 2 letter culture name + /// + /// + /// The culture to determine if it is only a 2 letter culture, if so we'll try to convert it, otherwise it will just be returned + /// + /// + /// + /// TODO: This is just a hack due to the way we store the language files, they should be stored with 4 letters since that + /// is what they reference but they are stored with 2, further more our user's languages are stored with 2. So this attempts + /// to resolve the full culture if possible. + /// + /// This only works when this service is constructed with the LocalizedTextServiceFileSources + /// + public CultureInfo ConvertToSupportedCultureWithRegionCode(CultureInfo currentCulture) + { + if (currentCulture == null) throw new ArgumentNullException("currentCulture"); + + if (_fileSources == null) return currentCulture; + if (currentCulture.Name.Length > 2) return currentCulture; + + var attempt = _fileSources.Value.TryConvert2LetterCultureTo4Letter(currentCulture.TwoLetterISOLanguageName); + return attempt ? attempt.Result : currentCulture; + } + + private string GetFromDictionarySource(CultureInfo culture, string area, string key, IDictionary tokens) + { + if (_dictionarySource.ContainsKey(culture) == false) + { + _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); + return "[" + key + "]"; + } + + var cultureSource = _dictionarySource[culture]; + + string found; + if (area.IsNullOrWhiteSpace()) + { + found = cultureSource + .SelectMany(x => x.Value) + .Where(keyvals => keyvals.Key.InvariantEquals(key)) + .Select(x => x.Value) + .FirstOrDefault(); + } + else + { + found = cultureSource + .Where(areas => areas.Key.InvariantEquals(area)) + .SelectMany(a => a.Value) + .Where(keyvals => keyvals.Key.InvariantEquals(key)) + .Select(x => x.Value) + .FirstOrDefault(); + } + + if (found != null) + { + return ParseTokens(found, tokens); + } + + //NOTE: Based on how legacy works, the default text does not contain the area, just the key + return "[" + key + "]"; + } + + private string GetFromXmlSource(IDictionary> xmlSource, CultureInfo culture, string area, string key, IDictionary tokens) + { + if (xmlSource.ContainsKey(culture) == false) + { + _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); + return "[" + key + "]"; + } + + var found = FindTranslation(xmlSource, culture, area, key); + + if (found != null) + { + return ParseTokens(found.Value, tokens); + } + + // Fall back to English by default if we can't find the key + found = FindTranslation(xmlSource, new CultureInfo("en-US"), area, key); + if (found != null) + return ParseTokens(found.Value, tokens); + + // If it can't be found in either file, fall back to the default, showing just the key in square brackets + // NOTE: Based on how legacy works, the default text does not contain the area, just the key + return "[" + key + "]"; + } + + private XElement FindTranslation(IDictionary> xmlSource, CultureInfo culture, string area, string key) + { + var cultureSource = xmlSource[culture].Value; + + var xpath = area.IsNullOrWhiteSpace() + ? string.Format("//key [@alias = '{0}']", key) + : string.Format("//area [@alias = '{0}']/key [@alias = '{1}']", area, key); + + var found = cultureSource.XPathSelectElement(xpath); + return found; + } + + /// + /// Parses the tokens in the value + /// + /// + /// + /// + /// + /// This is based on how the legacy ui localized text worked, each token was just a sequential value delimited with a % symbol. + /// For example: hello %0%, you are %1% ! + /// + /// Since we're going to continue using the same language files for now, the token system needs to remain the same. With our new service + /// we support a dictionary which means in the future we can really have any sort of token system. + /// Currently though, the token key's will need to be an integer and sequential - though we aren't going to throw exceptions if that is not the case. + /// + internal static string ParseTokens(string value, IDictionary tokens) + { + if (tokens == null || tokens.Any() == false) + { + return value; + } + + foreach (var token in tokens) + { + value = value.Replace(string.Format("{0}{1}{0}", "%", token.Key), token.Value); + } + + return value; + } + + } + +// // * Summary * + +// BenchmarkDotNet=v0.11.3, OS=Windows 10.0.18362 +//Intel Core i5-8265U CPU 1.60GHz(Kaby Lake R), 1 CPU, 8 logical and 4 physical cores +// [Host] : .NET Framework 4.7.2 (CLR 4.0.30319.42000), 32bit LegacyJIT-v4.8.4250.0 +// Job-JIATTD : .NET Framework 4.7.2 (CLR 4.0.30319.42000), 32bit LegacyJIT-v4.8.4250.0 + +//IterationCount=3 IterationTime=100.0000 ms LaunchCount = 1 +//WarmupCount=3 + +// Method | Mean | Error | StdDev | Ratio | Gen 0/1k Op | Gen 1/1k Op | Gen 2/1k Op | Allocated Memory/Op | +//---------------------- |----------:|-----------:|----------:|------:|------------:|------------:|------------:|--------------------:| +// DictionaryGetAll | 11.199 ms | 1.8170 ms | 0.0996 ms | 0.14 | 1888.8889 | - | - | 5868.59 KB | +// XmlGetAll | 62.963 ms | 24.0615 ms | 1.3189 ms | 0.81 | 13500.0000 | - | - | 42448.71 KB | +// DictionaryLocalize | 9.757 ms | 1.6966 ms | 0.0930 ms | 0.13 | 1100.0000 | - | - | 3677.65 KB | +// XmlLocalize | 77.725 ms | 14.6069 ms | 0.8007 ms | 1.00 | 14000.0000 | - | - | 43032.8 KB | +// OptimizedXmlLocalize | 2.402 ms | 0.4256 ms | 0.0233 ms | 0.03 | 187.5000 | - | - | 626.01 KB | +// OptimizedDictLocalize | 2.345 ms | 0.2411 ms | 0.0132 ms | 0.03 | 187.5000 | - | - | 626.01 KB | + +//// * Warnings * +//MinIterationTime +// LocalizedTextServiceGetAllStoredValuesBenchmarks.DictionaryGetAll: IterationCount= 3, IterationTime= 100.0000 ms, LaunchCount= 1, WarmupCount= 3->MinIterationTime = 99.7816 ms which is very small. It's recommended to increase it. +// LocalizedTextServiceGetAllStoredValuesBenchmarks.DictionaryLocalize: IterationCount= 3, IterationTime= 100.0000 ms, LaunchCount= 1, WarmupCount= 3->MinIterationTime = 96.7415 ms which is very small. It's recommended to increase it. +// LocalizedTextServiceGetAllStoredValuesBenchmarks.XmlLocalize: IterationCount= 3, IterationTime= 100.0000 ms, LaunchCount= 1, WarmupCount= 3->MinIterationTime = 76.8151 ms which is very small. It's recommended to increase it. + +//// * Legends * +// Mean : Arithmetic mean of all measurements +// Error : Half of 99.9% confidence interval +// StdDev : Standard deviation of all measurements +// Ratio : Mean of the ratio distribution ([Current]/[Baseline]) +// Gen 0/1k Op : GC Generation 0 collects per 1k Operations +// Gen 1/1k Op : GC Generation 1 collects per 1k Operations +// Gen 2/1k Op : GC Generation 2 collects per 1k Operations +// Allocated Memory/Op : Allocated memory per single operation(managed only, inclusive, 1KB = 1024B) +// 1 ms : 1 Millisecond(0.001 sec) + +//// * Diagnostic Output - MemoryDiagnoser * + + +// // ***** BenchmarkRunner: End ***** +// Run time: 00:00:09 (9.15 sec), executed benchmarks: 6 +} diff --git a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj index 48d69cf757..4bb7faa22d 100644 --- a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj +++ b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj @@ -54,6 +54,7 @@ + diff --git a/src/Umbraco.Tests/Models/Mapping/ContentWebModelMappingTests.cs b/src/Umbraco.Tests/Models/Mapping/ContentWebModelMappingTests.cs index 6a4054d5ae..eebc3a0e8b 100644 --- a/src/Umbraco.Tests/Models/Mapping/ContentWebModelMappingTests.cs +++ b/src/Umbraco.Tests/Models/Mapping/ContentWebModelMappingTests.cs @@ -255,8 +255,8 @@ namespace Umbraco.Tests.Models.Mapping } Assert.AreEqual(contentType.CompositionPropertyGroups.Count(), invariantContent.Tabs.Count() - 1); - Assert.IsTrue(invariantContent.Tabs.Any(x => x.Label == Current.Services.TextService.Localize("general/properties"))); - Assert.AreEqual(2, invariantContent.Tabs.Where(x => x.Label == Current.Services.TextService.Localize("general/properties")).SelectMany(x => x.Properties.Where(p => p.Alias.StartsWith("_umb_") == false)).Count()); + Assert.IsTrue(invariantContent.Tabs.Any(x => x.Label == Current.Services.TextService.Localize("general", "properties"))); + Assert.AreEqual(2, invariantContent.Tabs.Where(x => x.Label == Current.Services.TextService.Localize("general", "properties")).SelectMany(x => x.Properties.Where(p => p.Alias.StartsWith("_umb_") == false)).Count()); } #region Assertions diff --git a/src/Umbraco.Web/Compose/NotificationsComponent.cs b/src/Umbraco.Web/Compose/NotificationsComponent.cs index ea5df65f74..42984cef16 100644 --- a/src/Umbraco.Web/Compose/NotificationsComponent.cs +++ b/src/Umbraco.Web/Compose/NotificationsComponent.cs @@ -257,12 +257,12 @@ namespace Umbraco.Web.Compose siteUri, ((IUser user, NotificationEmailSubjectParams subject) x) => _textService.Localize( - "notifications/mailSubject", + "notifications", "mailSubject", x.user.GetUserCulture(_textService, _globalSettings), new[] { x.subject.SiteUrl, x.subject.Action, x.subject.ItemName }), ((IUser user, NotificationEmailBodyParams body, bool isHtml) x) - => _textService.Localize( - x.isHtml ? "notifications/mailBodyHtml" : "notifications/mailBody", + => _textService.Localize("notifications", + x.isHtml ? "mailBodyHtml" : "mailBody", x.user.GetUserCulture(_textService, _globalSettings), new[] { @@ -274,7 +274,7 @@ namespace Umbraco.Web.Compose x.body.ItemId, //format the summary depending on if it's variant or not contentVariantGroup.Key == ContentVariation.Culture - ? (x.isHtml ? _textService.Localize("notifications/mailBodyVariantHtmlSummary", new[]{ x.body.Summary }) : _textService.Localize("notifications/mailBodyVariantSummary", new []{ x.body.Summary })) + ? (x.isHtml ? _textService.Localize("notifications","mailBodyVariantHtmlSummary", new[]{ x.body.Summary }) : _textService.Localize("notifications","mailBodyVariantSummary", new []{ x.body.Summary })) : x.body.Summary, x.body.ItemUrl })); diff --git a/src/Umbraco.Web/Editors/AuthenticationController.cs b/src/Umbraco.Web/Editors/AuthenticationController.cs index e2ab829427..d5f4ef9fe8 100644 --- a/src/Umbraco.Web/Editors/AuthenticationController.cs +++ b/src/Umbraco.Web/Editors/AuthenticationController.cs @@ -337,7 +337,7 @@ namespace Umbraco.Web.Editors new[] { identityUser.UserName, callbackUrl }); await UserManager.SendEmailAsync(identityUser.Id, - Services.TextService.Localize("login/resetPasswordEmailCopySubject", + Services.TextService.Localize("login", "resetPasswordEmailCopySubject", // Ensure the culture of the found user is used for the email! UserExtensions.GetUserCulture(identityUser.Culture, Services.TextService, GlobalSettings)), message); diff --git a/src/Umbraco.Web/Editors/BackOfficeController.cs b/src/Umbraco.Web/Editors/BackOfficeController.cs index 18740d41fc..2959a983bd 100644 --- a/src/Umbraco.Web/Editors/BackOfficeController.cs +++ b/src/Umbraco.Web/Editors/BackOfficeController.cs @@ -347,7 +347,7 @@ namespace Umbraco.Web.Editors } //Add error and redirect for it to be displayed - TempData[ViewDataExtensions.TokenPasswordResetCode] = new[] { Services.TextService.Localize("login/resetCodeExpired") }; + TempData[ViewDataExtensions.TokenPasswordResetCode] = new[] { Services.TextService.Localize("login", "resetCodeExpired") }; return RedirectToLocal(Url.Action("Default", "BackOffice")); } diff --git a/src/Umbraco.Web/Editors/CodeFileController.cs b/src/Umbraco.Web/Editors/CodeFileController.cs index 0e1c4b3e60..967d0d14c7 100644 --- a/src/Umbraco.Web/Editors/CodeFileController.cs +++ b/src/Umbraco.Web/Editors/CodeFileController.cs @@ -88,7 +88,7 @@ namespace Umbraco.Web.Editors if (string.IsNullOrWhiteSpace(parentId)) throw new ArgumentException("Value cannot be null or whitespace.", "parentId"); if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Value cannot be null or whitespace.", "name"); if (name.ContainsAny(Path.GetInvalidPathChars())) { - return Request.CreateNotificationValidationErrorResponse(Services.TextService.Localize("codefile/createFolderIllegalChars")); + return Request.CreateNotificationValidationErrorResponse(Services.TextService.Localize("codefile", "createFolderIllegalChars")); } // if the parentId is root (-1) then we just need an empty string as we are @@ -388,8 +388,8 @@ namespace Umbraco.Web.Editors } display.AddErrorNotification( - Services.TextService.Localize("speechBubbles/partialViewErrorHeader"), - Services.TextService.Localize("speechBubbles/partialViewErrorText")); + Services.TextService.Localize("speechBubbles", "partialViewErrorHeader"), + Services.TextService.Localize("speechBubbles", "partialViewErrorText")); break; case Core.Constants.Trees.PartialViewMacros: @@ -403,8 +403,8 @@ namespace Umbraco.Web.Editors } display.AddErrorNotification( - Services.TextService.Localize("speechBubbles/partialViewErrorHeader"), - Services.TextService.Localize("speechBubbles/partialViewErrorText")); + Services.TextService.Localize("speechBubbles", "partialViewErrorHeader"), + Services.TextService.Localize("speechBubbles", "partialViewErrorText")); break; case Core.Constants.Trees.Scripts: diff --git a/src/Umbraco.Web/Editors/ContentController.cs b/src/Umbraco.Web/Editors/ContentController.cs index 6f85a08751..ed629d746a 100644 --- a/src/Umbraco.Web/Editors/ContentController.cs +++ b/src/Umbraco.Web/Editors/ContentController.cs @@ -242,7 +242,7 @@ namespace Umbraco.Web.Editors new ContentVariantDisplay { CreateDate = DateTime.Now, - Name = Services.TextService.Localize("general/recycleBin") + Name = Services.TextService.Localize("general","recycleBin") } }, ContentApps = apps @@ -567,8 +567,8 @@ namespace Umbraco.Web.Editors var notificationModel = new SimpleNotificationModel(); notificationModel.AddSuccessNotification( - Services.TextService.Localize("blueprints/createdBlueprintHeading"), - Services.TextService.Localize("blueprints/createdBlueprintMessage", new[] { content.Name }) + Services.TextService.Localize("blueprints", "createdBlueprintHeading"), + Services.TextService.Localize("blueprints", "createdBlueprintMessage", new[] { content.Name }) ); return notificationModel; @@ -579,7 +579,7 @@ namespace Umbraco.Web.Editors var existing = Services.ContentService.GetBlueprintsForContentTypes(content.ContentTypeId); if (existing.Any(x => x.Name == name && x.Id != content.Id)) { - ModelState.AddModelError(modelName, Services.TextService.Localize("blueprints/duplicateBlueprintMessage")); + ModelState.AddModelError(modelName, Services.TextService.Localize("blueprints", "duplicateBlueprintMessage")); throw new HttpResponseException(Request.CreateValidationErrorResponse(ModelState)); } } @@ -741,15 +741,15 @@ namespace Umbraco.Web.Editors var variantName = GetVariantName(culture, segment); AddSuccessNotification(notifications, culture, segment, - Services.TextService.Localize("speechBubbles/editContentSendToPublish"), - Services.TextService.Localize("speechBubbles/editVariantSendToPublishText", new[] { variantName })); + Services.TextService.Localize("speechBubbles", "editContentSendToPublish"), + Services.TextService.Localize("speechBubbles", "editVariantSendToPublishText", new[] { variantName })); } } else if (ModelState.IsValid) { globalNotifications.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/editContentSendToPublish"), - Services.TextService.Localize("speechBubbles/editContentSendToPublishText")); + Services.TextService.Localize("speechBubbles", "editContentSendToPublish"), + Services.TextService.Localize("speechBubbles", "editContentSendToPublishText")); } } break; @@ -766,8 +766,8 @@ namespace Umbraco.Web.Editors if (!ValidatePublishBranchPermissions(contentItem, out var noAccess)) { globalNotifications.AddErrorNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/invalidPublishBranchPermissions")); + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "invalidPublishBranchPermissions")); wasCancelled = false; break; } @@ -782,8 +782,8 @@ namespace Umbraco.Web.Editors if (!ValidatePublishBranchPermissions(contentItem, out var noAccess)) { globalNotifications.AddErrorNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/invalidPublishBranchPermissions")); + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "invalidPublishBranchPermissions")); wasCancelled = false; break; } @@ -873,7 +873,7 @@ namespace Umbraco.Web.Editors //if there's more than 1 variant, then we need to add the culture specific error //messages based on the variants in error so that the messages show in the publish/save dialog if (variants.Count > 1) - AddVariantValidationError(variant.Culture, variant.Segment, "publish/contentPublishedFailedByMissingName"); + AddVariantValidationError(variant.Culture, variant.Segment, "publish","contentPublishedFailedByMissingName"); else return false; //It's invariant and is missing critical data, it cannot be saved } @@ -911,14 +911,14 @@ namespace Umbraco.Web.Editors /// /// /// - /// + /// /// /// /// Method is used for normal Saving and Scheduled Publishing /// private void SaveAndNotify(ContentItemSave contentItem, Func saveMethod, int variantCount, Dictionary notifications, SimpleNotificationModel globalNotifications, - string invariantSavedLocalizationKey, string variantSavedLocalizationKey, string cultureForInvariantErrors, + string invariantSavedLocalizationKey, string variantSavedLocalizationAlias, string cultureForInvariantErrors, out bool wasCancelled) { var saveResult = saveMethod(contentItem.PersistedContent); @@ -938,14 +938,14 @@ namespace Umbraco.Web.Editors var variantName = GetVariantName(culture, segment); AddSuccessNotification(notifications, culture, segment, - Services.TextService.Localize("speechBubbles/editContentSavedHeader"), - Services.TextService.Localize(variantSavedLocalizationKey, new[] { variantName })); + Services.TextService.Localize("speechBubbles", "editContentSavedHeader"), + Services.TextService.Localize(null,variantSavedLocalizationAlias, new[] { variantName })); } } else if (ModelState.IsValid) { globalNotifications.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/editContentSavedHeader"), + Services.TextService.Localize("speechBubbles", "editContentSavedHeader"), Services.TextService.Localize(invariantSavedLocalizationKey)); } } @@ -1073,7 +1073,7 @@ namespace Umbraco.Web.Editors { //can't continue, a mandatory variant is not published and not scheduled for publishing // TODO: Add segment - AddVariantValidationError(culture, null, "speechBubbles/scheduleErrReleaseDate2"); + AddVariantValidationError(culture, null, "speechBubbles", "scheduleErrReleaseDate2"); isValid = false; continue; } @@ -1081,7 +1081,7 @@ namespace Umbraco.Web.Editors { //can't continue, a mandatory variant is not published and it's scheduled for publishing after a non-mandatory // TODO: Add segment - AddVariantValidationError(culture, null, "speechBubbles/scheduleErrReleaseDate3"); + AddVariantValidationError(culture, null, "speechBubbles", "scheduleErrReleaseDate3"); isValid = false; continue; } @@ -1095,7 +1095,7 @@ namespace Umbraco.Web.Editors //1) release date cannot be less than now if (variant.ReleaseDate.HasValue && variant.ReleaseDate < DateTime.Now) { - AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles/scheduleErrReleaseDate1"); + AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles", "scheduleErrReleaseDate1"); isValid = false; continue; } @@ -1103,7 +1103,7 @@ namespace Umbraco.Web.Editors //2) expire date cannot be less than now if (variant.ExpireDate.HasValue && variant.ExpireDate < DateTime.Now) { - AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles/scheduleErrExpireDate1"); + AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles", "scheduleErrExpireDate1"); isValid = false; continue; } @@ -1111,7 +1111,7 @@ namespace Umbraco.Web.Editors //3) expire date cannot be less than release date if (variant.ExpireDate.HasValue && variant.ReleaseDate.HasValue && variant.ExpireDate <= variant.ReleaseDate) { - AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles/scheduleErrExpireDate2"); + AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles", "scheduleErrExpireDate2"); isValid = false; continue; } @@ -1360,19 +1360,19 @@ namespace Umbraco.Web.Editors if (r.publishing && !r.isValid) { //flagged for publishing but the mandatory culture is invalid - AddVariantValidationError(r.model.Culture, r.model.Segment, "publish/contentPublishedFailedReqCultureValidationError"); + AddVariantValidationError(r.model.Culture, r.model.Segment, "publish", "contentPublishedFailedReqCultureValidationError"); canPublish = false; } else if (r.publishing && r.isValid && firstInvalidMandatoryCulture != null) { //in this case this culture also cannot be published because another mandatory culture is invalid - AddVariantValidationError(r.model.Culture, r.model.Segment, "publish/contentPublishedFailedReqCultureValidationError", firstInvalidMandatoryCulture); + AddVariantValidationError(r.model.Culture, r.model.Segment, "publish", "contentPublishedFailedReqCultureValidationError", firstInvalidMandatoryCulture); canPublish = false; } else if (!r.publishing) { //cannot continue publishing since a required culture that is not currently being published isn't published - AddVariantValidationError(r.model.Culture, r.model.Segment, "speechBubbles/contentReqCulturePublishError"); + AddVariantValidationError(r.model.Culture, r.model.Segment, "speechBubbles", "contentReqCulturePublishError"); canPublish = false; } } @@ -1397,7 +1397,7 @@ namespace Umbraco.Web.Editors var valid = persistentContent.PublishCulture(CultureImpact.Explicit(variant.Culture, defaultCulture.InvariantEquals(variant.Culture))); if (!valid) { - AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles/contentCultureValidationError"); + AddVariantValidationError(variant.Culture, variant.Segment, "speechBubbles", "contentCultureValidationError"); return false; } } @@ -1414,12 +1414,12 @@ namespace Umbraco.Web.Editors /// /// The culture used in the localization message, null by default which means will be used. /// - private void AddVariantValidationError(string culture, string segment, string localizationKey, string cultureToken = null) + private void AddVariantValidationError(string culture, string segment, string localizationArea,string localizationAlias, string cultureToken = null) { var cultureToUse = cultureToken ?? culture; var variantName = GetVariantName(cultureToUse, segment); - var errMsg = Services.TextService.Localize(localizationKey, new[] { variantName }); + var errMsg = Services.TextService.Localize(localizationArea, localizationAlias, new[] { variantName }); ModelState.AddVariantValidationError(culture, segment, errMsg); } @@ -1554,7 +1554,7 @@ namespace Umbraco.Web.Editors { Services.ContentService.EmptyRecycleBin(Security.GetUserId().ResultOr(Constants.Security.SuperUserId)); - return Request.CreateNotificationSuccessResponse(Services.TextService.Localize("defaultdialogs/recycleBinIsEmpty")); + return Request.CreateNotificationSuccessResponse(Services.TextService.Localize("defaultdialogs", "recycleBinIsEmpty")); } /// @@ -1662,8 +1662,8 @@ namespace Umbraco.Web.Editors else { content.AddSuccessNotification( - Services.TextService.Localize("content/unpublish"), - Services.TextService.Localize("speechBubbles/contentUnpublished")); + Services.TextService.Localize("content", "unpublish"), + Services.TextService.Localize("speechBubbles", "contentUnpublished")); return content; } } @@ -1688,8 +1688,8 @@ namespace Umbraco.Web.Editors if (results.Any(x => x.Value.Result == PublishResultType.SuccessUnpublishMandatoryCulture)) { content.AddSuccessNotification( - Services.TextService.Localize("content/unpublish"), - Services.TextService.Localize("speechBubbles/contentMandatoryCultureUnpublished")); + Services.TextService.Localize("content", "unpublish"), + Services.TextService.Localize("speechBubbles", "contentMandatoryCultureUnpublished")); return content; } @@ -1697,8 +1697,8 @@ namespace Umbraco.Web.Editors foreach (var r in results) { content.AddSuccessNotification( - Services.TextService.Localize("content/unpublish"), - Services.TextService.Localize("speechBubbles/contentCultureUnpublished", new[] { _allLangs.Value[r.Key].CultureName })); + Services.TextService.Localize("conten", "unpublish"), + Services.TextService.Localize("speechBubbles", "contentCultureUnpublished", new[] { _allLangs.Value[r.Key].CultureName })); } return content; @@ -1729,7 +1729,7 @@ namespace Umbraco.Web.Editors } catch (UriFormatException) { - var response = Request.CreateValidationErrorResponse(Services.TextService.Localize("assignDomain/invalidDomain")); + var response = Request.CreateValidationErrorResponse(Services.TextService.Localize("assignDomain", "invalidDomain")); throw new HttpResponseException(response); } } @@ -1887,7 +1887,7 @@ namespace Umbraco.Web.Editors foreach (var (culture, segment) in variantErrors) { - AddVariantValidationError(culture, segment, "speechBubbles/contentCultureValidationError"); + AddVariantValidationError(culture, segment, "speechBubbles", "contentCultureValidationError"); } } @@ -2012,7 +2012,7 @@ namespace Umbraco.Web.Editors { throw new HttpResponseException( Request.CreateNotificationValidationErrorResponse( - Services.TextService.Localize("moveOrCopy/notAllowedAtRoot"))); + Services.TextService.Localize("moveOrCopy", "notAllowedAtRoot"))); } } else @@ -2030,7 +2030,7 @@ namespace Umbraco.Web.Editors { throw new HttpResponseException( Request.CreateNotificationValidationErrorResponse( - Services.TextService.Localize("moveOrCopy/notAllowedByContentType"))); + Services.TextService.Localize("moveOrCopy", "notAllowedByContentType"))); } // Check on paths @@ -2038,7 +2038,7 @@ namespace Umbraco.Web.Editors { throw new HttpResponseException( Request.CreateNotificationValidationErrorResponse( - Services.TextService.Localize("moveOrCopy/notAllowedByPath"))); + Services.TextService.Localize("moveOrCopy", "notAllowedByPath"))); } } @@ -2107,16 +2107,16 @@ namespace Umbraco.Web.Editors { //either invariant single publish, or bulk publish where all statuses are already published display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/editContentPublishedHeader"), - Services.TextService.Localize("speechBubbles/editContentPublishedText")); + Services.TextService.Localize("speechBubbles", "editContentPublishedHeader"), + Services.TextService.Localize("speechBubbles", "editContentPublishedText")); } else { foreach (var c in successfulCultures) { display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/editContentPublishedHeader"), - Services.TextService.Localize("speechBubbles/editVariantPublishedText", new[] { _allLangs.Value[c].CultureName })); + Services.TextService.Localize("speechBubbles", "editContentPublishedHeader"), + Services.TextService.Localize("speechBubbles", "editVariantPublishedText", new[] { _allLangs.Value[c].CultureName })); } } } @@ -2132,20 +2132,20 @@ namespace Umbraco.Web.Editors if (successfulCultures == null) { display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/editContentPublishedHeader"), + Services.TextService.Localize("speechBubbles", "editContentPublishedHeader"), totalStatusCount > 1 - ? Services.TextService.Localize("speechBubbles/editMultiContentPublishedText", new[] { itemCount.ToInvariantString() }) - : Services.TextService.Localize("speechBubbles/editContentPublishedText")); + ? Services.TextService.Localize("speechBubbles", "editMultiContentPublishedText", new[] { itemCount.ToInvariantString() }) + : Services.TextService.Localize("speechBubbles", "editContentPublishedText")); } else { foreach (var c in successfulCultures) { display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/editContentPublishedHeader"), + Services.TextService.Localize("speechBubbles", "editContentPublishedHeader"), totalStatusCount > 1 - ? Services.TextService.Localize("speechBubbles/editMultiVariantPublishedText", new[] { itemCount.ToInvariantString(), _allLangs.Value[c].CultureName }) - : Services.TextService.Localize("speechBubbles/editVariantPublishedText", new[] { _allLangs.Value[c].CultureName })); + ? Services.TextService.Localize("speechBubbles", "editMultiVariantPublishedText", new[] { itemCount.ToInvariantString(), _allLangs.Value[c].CultureName }) + : Services.TextService.Localize("speechBubbles", "editVariantPublishedText", new[] { _allLangs.Value[c].CultureName })); } } } @@ -2155,8 +2155,8 @@ namespace Umbraco.Web.Editors //TODO: This doesn't take into account variations with the successfulCultures param var names = string.Join(", ", status.Select(x => $"'{x.Content.Name}'")); display.AddWarningNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/contentPublishedFailedByParent", + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "contentPublishedFailedByParent", new[] { names }).Trim()); } break; @@ -2172,8 +2172,8 @@ namespace Umbraco.Web.Editors //TODO: This doesn't take into account variations with the successfulCultures param var names = string.Join(", ", status.Select(x => $"'{x.Content.Name}'")); display.AddWarningNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/contentPublishedFailedAwaitingRelease", + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "contentPublishedFailedAwaitingRelease", new[] { names }).Trim()); } break; @@ -2182,8 +2182,8 @@ namespace Umbraco.Web.Editors //TODO: This doesn't take into account variations with the successfulCultures param var names = string.Join(", ", status.Select(x => $"'{x.Content.Name}'")); display.AddWarningNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/contentPublishedFailedExpired", + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "contentPublishedFailedExpired", new[] { names }).Trim()); } break; @@ -2192,8 +2192,8 @@ namespace Umbraco.Web.Editors //TODO: This doesn't take into account variations with the successfulCultures param var names = string.Join(", ", status.Select(x => $"'{x.Content.Name}'")); display.AddWarningNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/contentPublishedFailedIsTrashed", + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "contentPublishedFailedIsTrashed", new[] { names }).Trim()); } break; @@ -2203,8 +2203,8 @@ namespace Umbraco.Web.Editors { var names = string.Join(", ", status.Select(x => $"'{x.Content.Name}'")); display.AddWarningNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/contentPublishedFailedInvalid", + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "contentPublishedFailedInvalid", new[] { names }).Trim()); } else @@ -2213,8 +2213,8 @@ namespace Umbraco.Web.Editors { var names = string.Join(", ", status.Select(x => $"'{(x.Content.ContentType.VariesByCulture() ? x.Content.GetCultureName(c) : x.Content.Name)}'")); display.AddWarningNotification( - Services.TextService.Localize("publish"), - Services.TextService.Localize("publish/contentPublishedFailedInvalid", + Services.TextService.Localize(null,"publish"), + Services.TextService.Localize("publish", "contentPublishedFailedInvalid", new[] { names }).Trim()); } } @@ -2222,7 +2222,7 @@ namespace Umbraco.Web.Editors break; case PublishResultType.FailedPublishMandatoryCultureMissing: display.AddWarningNotification( - Services.TextService.Localize("publish"), + Services.TextService.Localize(null,"publish"), "publish/contentPublishedFailedByCulture"); break; default: @@ -2355,13 +2355,13 @@ namespace Umbraco.Web.Editors case OperationResultType.NoOperation: default: notificationModel.AddErrorNotification( - Services.TextService.Localize("speechBubbles/operationFailedHeader"), + Services.TextService.Localize("speechBubbles", "operationFailedHeader"), null); // TODO: There is no specific failed to save error message AFAIK break; case OperationResultType.FailedCancelledByEvent: notificationModel.AddErrorNotification( - Services.TextService.Localize("speechBubbles/operationCancelledHeader"), - Services.TextService.Localize("speechBubbles/operationCancelledText")); + Services.TextService.Localize("speechBubbles", "operationCancelledHeader"), + Services.TextService.Localize("speechBubbles", "operationCancelledText")); break; } diff --git a/src/Umbraco.Web/Editors/ContentTypeController.cs b/src/Umbraco.Web/Editors/ContentTypeController.cs index d426cb1f56..89f0b99d7f 100644 --- a/src/Umbraco.Web/Editors/ContentTypeController.cs +++ b/src/Umbraco.Web/Editors/ContentTypeController.cs @@ -384,7 +384,7 @@ namespace Umbraco.Web.Editors var display = Mapper.Map(savedCt); display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/contentTypeSavedHeader"), + Services.TextService.Localize("speechBubbles", "contentTypeSavedHeader"), string.Empty); return display; @@ -666,8 +666,8 @@ namespace Umbraco.Web.Editors else { model.Notifications.Add(new Notification( - Services.TextService.Localize("speechBubbles/operationFailedHeader"), - Services.TextService.Localize("media/disallowedFileType"), + Services.TextService.Localize("speechBubbles", "operationFailedHeader"), + Services.TextService.Localize("media", "disallowedFileType"), NotificationStyle.Warning)); } diff --git a/src/Umbraco.Web/Editors/ContentTypeControllerBase.cs b/src/Umbraco.Web/Editors/ContentTypeControllerBase.cs index d56820a549..8eefbb616b 100644 --- a/src/Umbraco.Web/Editors/ContentTypeControllerBase.cs +++ b/src/Umbraco.Web/Editors/ContentTypeControllerBase.cs @@ -256,7 +256,7 @@ namespace Umbraco.Web.Editors var exists = allAliases.InvariantContains(contentTypeSave.Alias); if (exists && (ctId == 0 || !ct.Alias.InvariantEquals(contentTypeSave.Alias))) { - ModelState.AddModelError("Alias", Services.TextService.Localize("editcontenttype/aliasAlreadyExists")); + ModelState.AddModelError("Alias", Services.TextService.Localize("editcontenttype", "aliasAlreadyExists")); } // execute the external validators @@ -388,7 +388,7 @@ namespace Umbraco.Web.Editors return Request.CreateValidationErrorResponse(new SimpleNotificationModel()); case MoveOperationStatusType.FailedNotAllowedByPath: var notificationModel = new SimpleNotificationModel(); - notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy/notAllowedByPath"), ""); + notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy", "notAllowedByPath"), ""); return Request.CreateValidationErrorResponse(notificationModel); default: throw new ArgumentOutOfRangeException(); @@ -432,7 +432,7 @@ namespace Umbraco.Web.Editors return Request.CreateValidationErrorResponse(new SimpleNotificationModel()); case MoveOperationStatusType.FailedNotAllowedByPath: var notificationModel = new SimpleNotificationModel(); - notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy/notAllowedByPath"), ""); + notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy", "notAllowedByPath"), ""); return Request.CreateValidationErrorResponse(notificationModel); default: throw new ArgumentOutOfRangeException(); diff --git a/src/Umbraco.Web/Editors/CurrentUserController.cs b/src/Umbraco.Web/Editors/CurrentUserController.cs index 46151f2a9e..8da2243255 100644 --- a/src/Umbraco.Web/Editors/CurrentUserController.cs +++ b/src/Umbraco.Web/Editors/CurrentUserController.cs @@ -183,7 +183,7 @@ namespace Umbraco.Web.Editors //even if we weren't resetting this, it is the correct value (null), otherwise if we were resetting then it will contain the new pword var result = new ModelWithNotifications(passwordChangeResult.Result.ResetPassword); - result.AddSuccessNotification(Services.TextService.Localize("user/password"), Services.TextService.Localize("user/passwordChanged")); + result.AddSuccessNotification(Services.TextService.Localize("user", "password"), Services.TextService.Localize("user", "passwordChanged")); return result; } diff --git a/src/Umbraco.Web/Editors/DataTypeController.cs b/src/Umbraco.Web/Editors/DataTypeController.cs index ad92d40ecf..76b8d590f2 100644 --- a/src/Umbraco.Web/Editors/DataTypeController.cs +++ b/src/Umbraco.Web/Editors/DataTypeController.cs @@ -281,7 +281,7 @@ namespace Umbraco.Web.Editors // map back to display model, and return var display = Mapper.Map(dataType.PersistedDataType); - display.AddSuccessNotification(Services.TextService.Localize("speechBubbles/dataTypeSaved"), ""); + display.AddSuccessNotification(Services.TextService.Localize("speechBubbles", "dataTypeSaved"), ""); return display; } @@ -316,7 +316,7 @@ namespace Umbraco.Web.Editors return Request.CreateValidationErrorResponse(new SimpleNotificationModel()); case MoveOperationStatusType.FailedNotAllowedByPath: var notificationModel = new SimpleNotificationModel(); - notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy/notAllowedByPath"), ""); + notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy", "notAllowedByPath"), ""); return Request.CreateValidationErrorResponse(notificationModel); default: throw new ArgumentOutOfRangeException(); diff --git a/src/Umbraco.Web/Editors/MediaController.cs b/src/Umbraco.Web/Editors/MediaController.cs index dfe6939552..758fb85898 100644 --- a/src/Umbraco.Web/Editors/MediaController.cs +++ b/src/Umbraco.Web/Editors/MediaController.cs @@ -106,7 +106,7 @@ namespace Umbraco.Web.Editors Id = Constants.System.RecycleBinMedia, Alias = "recycleBin", ParentId = -1, - Name = Services.TextService.Localize("general/recycleBin"), + Name = Services.TextService.Localize("general", "recycleBin"), ContentTypeAlias = "recycleBin", CreateDate = DateTime.Now, IsContainer = true, @@ -431,7 +431,7 @@ namespace Umbraco.Web.Editors if (sourceParentID == destinationParentID) { - return Request.CreateValidationErrorResponse(new SimpleNotificationModel(new Notification("",Services.TextService.Localize("media/moveToSameFolderFailed"),NotificationStyle.Error))); + return Request.CreateValidationErrorResponse(new SimpleNotificationModel(new Notification("",Services.TextService.Localize("media", "moveToSameFolderFailed"),NotificationStyle.Error))); } if (moveResult == false) { @@ -519,8 +519,8 @@ namespace Umbraco.Web.Editors if (saveStatus.Success) { display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/editMediaSaved"), - Services.TextService.Localize("speechBubbles/editMediaSavedText")); + Services.TextService.Localize("speechBubbles", "editMediaSaved"), + Services.TextService.Localize("speechBubbles", "editMediaSavedText")); } else { @@ -551,7 +551,7 @@ namespace Umbraco.Web.Editors { Services.MediaService.EmptyRecycleBin(Security.GetUserId().ResultOr(Constants.Security.SuperUserId)); - return Request.CreateNotificationSuccessResponse(Services.TextService.Localize("defaultdialogs/recycleBinIsEmpty")); + return Request.CreateNotificationSuccessResponse(Services.TextService.Localize("defaultdialogs", "recycleBinIsEmpty")); } /// @@ -730,7 +730,7 @@ namespace Umbraco.Web.Editors if (saveResult == false) { AddCancelMessage(tempFiles, - message: Services.TextService.Localize("speechBubbles/operationCancelledText") + " -- " + mediaItemName); + message: Services.TextService.Localize("speechBubbles", "operationCancelledText") + " -- " + mediaItemName); } else { @@ -745,8 +745,8 @@ namespace Umbraco.Web.Editors else { tempFiles.Notifications.Add(new Notification( - Services.TextService.Localize("speechBubbles/operationFailedHeader"), - Services.TextService.Localize("media/disallowedFileType"), + Services.TextService.Localize("speechBubbles", "operationFailedHeader"), + Services.TextService.Localize("media", "disallowedFileType"), NotificationStyle.Warning)); } } @@ -840,8 +840,8 @@ namespace Umbraco.Web.Editors throw new HttpResponseException(Request.CreateResponse( HttpStatusCode.Forbidden, new SimpleNotificationModel(new Notification( - Services.TextService.Localize("speechBubbles/operationFailedHeader"), - Services.TextService.Localize("speechBubbles/invalidUserPermissionsText"), + Services.TextService.Localize("speechBubbles", "operationFailedHeader"), + Services.TextService.Localize("speechBubbles", "invalidUserPermissionsText"), NotificationStyle.Warning)))); } @@ -874,7 +874,7 @@ namespace Umbraco.Web.Editors if (toMove.ContentType.AllowedAsRoot == false && mediaTypeService.GetAll().Any(ct => ct.AllowedAsRoot)) { var notificationModel = new SimpleNotificationModel(); - notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy/notAllowedAtRoot"), ""); + notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy", "notAllowedAtRoot"), ""); throw new HttpResponseException(Request.CreateValidationErrorResponse(notificationModel)); } } @@ -892,7 +892,7 @@ namespace Umbraco.Web.Editors .Any(x => x.Value == toMove.ContentType.Id) == false) { var notificationModel = new SimpleNotificationModel(); - notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy/notAllowedByContentType"), ""); + notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy", "notAllowedByContentType"), ""); throw new HttpResponseException(Request.CreateValidationErrorResponse(notificationModel)); } @@ -900,7 +900,7 @@ namespace Umbraco.Web.Editors if ((string.Format(",{0},", parent.Path)).IndexOf(string.Format(",{0},", toMove.Id), StringComparison.Ordinal) > -1) { var notificationModel = new SimpleNotificationModel(); - notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy/notAllowedByPath"), ""); + notificationModel.AddErrorNotification(Services.TextService.Localize("moveOrCopy", "notAllowedByPath"), ""); throw new HttpResponseException(Request.CreateValidationErrorResponse(notificationModel)); } } diff --git a/src/Umbraco.Web/Editors/MediaTypeController.cs b/src/Umbraco.Web/Editors/MediaTypeController.cs index d816f13c92..3b7c3fb4d9 100644 --- a/src/Umbraco.Web/Editors/MediaTypeController.cs +++ b/src/Umbraco.Web/Editors/MediaTypeController.cs @@ -250,7 +250,7 @@ namespace Umbraco.Web.Editors var display = Mapper.Map(savedCt); display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/mediaTypeSavedHeader"), + Services.TextService.Localize("speechBubbles", "mediaTypeSavedHeader"), string.Empty); return display; diff --git a/src/Umbraco.Web/Editors/MemberController.cs b/src/Umbraco.Web/Editors/MemberController.cs index 477dced7d0..2f6fb621b0 100644 --- a/src/Umbraco.Web/Editors/MemberController.cs +++ b/src/Umbraco.Web/Editors/MemberController.cs @@ -377,7 +377,7 @@ namespace Umbraco.Web.Editors { case ContentSaveAction.Save: case ContentSaveAction.SaveNew: - display.AddSuccessNotification(localizedTextService.Localize("speechBubbles/editMemberSaved"), localizedTextService.Localize("speechBubbles/editMemberSaved")); + display.AddSuccessNotification(localizedTextService.Localize("speechBubbles", "editMemberSaved"), localizedTextService.Localize("speechBubbles", "editMemberSaved")); break; } diff --git a/src/Umbraco.Web/Editors/MemberGroupController.cs b/src/Umbraco.Web/Editors/MemberGroupController.cs index ed11139e56..19adf17a2a 100644 --- a/src/Umbraco.Web/Editors/MemberGroupController.cs +++ b/src/Umbraco.Web/Editors/MemberGroupController.cs @@ -152,7 +152,7 @@ namespace Umbraco.Web.Editors var display = Mapper.Map(memberGroup); display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/memberGroupSavedHeader"), + Services.TextService.Localize("speechBubbles", "memberGroupSavedHeader"), string.Empty); return display; diff --git a/src/Umbraco.Web/Editors/MemberTypeController.cs b/src/Umbraco.Web/Editors/MemberTypeController.cs index 4bfea76eda..d55afd5e95 100644 --- a/src/Umbraco.Web/Editors/MemberTypeController.cs +++ b/src/Umbraco.Web/Editors/MemberTypeController.cs @@ -231,7 +231,7 @@ namespace Umbraco.Web.Editors var display = Mapper.Map(savedCt); display.AddSuccessNotification( - Services.TextService.Localize("speechBubbles/memberTypeSavedHeader"), + Services.TextService.Localize("speechBubbles", "memberTypeSavedHeader"), string.Empty); return display; diff --git a/src/Umbraco.Web/Editors/PackageInstallController.cs b/src/Umbraco.Web/Editors/PackageInstallController.cs index 1030498734..d12308aeee 100644 --- a/src/Umbraco.Web/Editors/PackageInstallController.cs +++ b/src/Umbraco.Web/Editors/PackageInstallController.cs @@ -180,7 +180,7 @@ namespace Umbraco.Web.Editors { //this package is already installed throw new HttpResponseException(Request.CreateNotificationValidationErrorResponse( - Services.TextService.Localize("packager/packageAlreadyInstalled"))); + Services.TextService.Localize("packager", "packageAlreadyInstalled"))); } model.OriginalVersion = installType == PackageInstallType.Upgrade ? alreadyInstalled.Version : null; @@ -189,8 +189,8 @@ namespace Umbraco.Web.Editors else { model.Notifications.Add(new Notification( - Services.TextService.Localize("speechBubbles/operationFailedHeader"), - Services.TextService.Localize("media/disallowedFileType"), + Services.TextService.Localize("speechBubbles", "operationFailedHeader"), + Services.TextService.Localize("media", "disallowedFileType"), NotificationStyle.Warning)); } @@ -234,7 +234,7 @@ namespace Umbraco.Web.Editors if (installType == PackageInstallType.AlreadyInstalled) { throw new HttpResponseException(Request.CreateNotificationValidationErrorResponse( - Services.TextService.Localize("packager/packageAlreadyInstalled"))); + Services.TextService.Localize("packager", "packageAlreadyInstalled"))); } model.OriginalVersion = installType == PackageInstallType.Upgrade ? alreadyInstalled.Version : null; @@ -260,7 +260,7 @@ namespace Umbraco.Web.Editors var packageMinVersion = packageInfo.UmbracoVersion; if (UmbracoVersion.Current < packageMinVersion) throw new HttpResponseException(Request.CreateNotificationValidationErrorResponse( - Services.TextService.Localize("packager/targetVersionMismatch", new[] {packageMinVersion.ToString()}))); + Services.TextService.Localize("packager", "targetVersionMismatch", new[] {packageMinVersion.ToString()}))); } var installType = Services.PackagingService.GetPackageInstallType(packageInfo.Name, SemVersion.Parse(packageInfo.Version), out var alreadyInstalled); diff --git a/src/Umbraco.Web/Editors/TemplateQueryController.cs b/src/Umbraco.Web/Editors/TemplateQueryController.cs index 1f07267802..a3b19cc9fe 100644 --- a/src/Umbraco.Web/Editors/TemplateQueryController.cs +++ b/src/Umbraco.Web/Editors/TemplateQueryController.cs @@ -21,28 +21,28 @@ namespace Umbraco.Web.Editors { private IEnumerable Terms => new List { - new OperatorTerm(Services.TextService.Localize("template/is"), Operator.Equals, new [] {"string"}), - new OperatorTerm(Services.TextService.Localize("template/isNot"), Operator.NotEquals, new [] {"string"}), - new OperatorTerm(Services.TextService.Localize("template/before"), Operator.LessThan, new [] {"datetime"}), - new OperatorTerm(Services.TextService.Localize("template/beforeIncDate"), Operator.LessThanEqualTo, new [] {"datetime"}), - new OperatorTerm(Services.TextService.Localize("template/after"), Operator.GreaterThan, new [] {"datetime"}), - new OperatorTerm(Services.TextService.Localize("template/afterIncDate"), Operator.GreaterThanEqualTo, new [] {"datetime"}), - new OperatorTerm(Services.TextService.Localize("template/equals"), Operator.Equals, new [] {"int"}), - new OperatorTerm(Services.TextService.Localize("template/doesNotEqual"), Operator.NotEquals, new [] {"int"}), - new OperatorTerm(Services.TextService.Localize("template/contains"), Operator.Contains, new [] {"string"}), - new OperatorTerm(Services.TextService.Localize("template/doesNotContain"), Operator.NotContains, new [] {"string"}), - new OperatorTerm(Services.TextService.Localize("template/greaterThan"), Operator.GreaterThan, new [] {"int"}), - new OperatorTerm(Services.TextService.Localize("template/greaterThanEqual"), Operator.GreaterThanEqualTo, new [] {"int"}), - new OperatorTerm(Services.TextService.Localize("template/lessThan"), Operator.LessThan, new [] {"int"}), - new OperatorTerm(Services.TextService.Localize("template/lessThanEqual"), Operator.LessThanEqualTo, new [] {"int"}) + new OperatorTerm(Services.TextService.Localize("template","is"), Operator.Equals, new [] {"string"}), + new OperatorTerm(Services.TextService.Localize("template","isNot"), Operator.NotEquals, new [] {"string"}), + new OperatorTerm(Services.TextService.Localize("template","before"), Operator.LessThan, new [] {"datetime"}), + new OperatorTerm(Services.TextService.Localize("template","beforeIncDate"), Operator.LessThanEqualTo, new [] {"datetime"}), + new OperatorTerm(Services.TextService.Localize("template","after"), Operator.GreaterThan, new [] {"datetime"}), + new OperatorTerm(Services.TextService.Localize("template","afterIncDate"), Operator.GreaterThanEqualTo, new [] {"datetime"}), + new OperatorTerm(Services.TextService.Localize("template","equals"), Operator.Equals, new [] {"int"}), + new OperatorTerm(Services.TextService.Localize("template","doesNotEqual"), Operator.NotEquals, new [] {"int"}), + new OperatorTerm(Services.TextService.Localize("template","contains"), Operator.Contains, new [] {"string"}), + new OperatorTerm(Services.TextService.Localize("template","doesNotContain"), Operator.NotContains, new [] {"string"}), + new OperatorTerm(Services.TextService.Localize("template","greaterThan"), Operator.GreaterThan, new [] {"int"}), + new OperatorTerm(Services.TextService.Localize("template","greaterThanEqual"), Operator.GreaterThanEqualTo, new [] {"int"}), + new OperatorTerm(Services.TextService.Localize("template","lessThan"), Operator.LessThan, new [] {"int"}), + new OperatorTerm(Services.TextService.Localize("template","lessThanEqual"), Operator.LessThanEqualTo, new [] {"int"}) }; private IEnumerable Properties => new List { - new PropertyModel { Name = Services.TextService.Localize("template/id"), Alias = "Id", Type = "int" }, - new PropertyModel { Name = Services.TextService.Localize("template/name"), Alias = "Name", Type = "string" }, - new PropertyModel { Name = Services.TextService.Localize("template/createdDate"), Alias = "CreateDate", Type = "datetime" }, - new PropertyModel { Name = Services.TextService.Localize("template/lastUpdatedDate"), Alias = "UpdateDate", Type = "datetime" } + new PropertyModel { Name = Services.TextService.Localize("template","id"), Alias = "Id", Type = "int" }, + new PropertyModel { Name = Services.TextService.Localize("template","name"), Alias = "Name", Type = "string" }, + new PropertyModel { Name = Services.TextService.Localize("template","createdDate"), Alias = "CreateDate", Type = "datetime" }, + new PropertyModel { Name = Services.TextService.Localize("template","lastUpdatedDate"), Alias = "UpdateDate", Type = "datetime" } }; public QueryResultModel PostTemplateQuery(QueryModel model) @@ -206,10 +206,10 @@ namespace Umbraco.Web.Editors public IEnumerable GetContentTypes() { var contentTypes = Services.ContentTypeService.GetAll() - .Select(x => new ContentTypeModel { Alias = x.Alias, Name = Services.TextService.Localize("template/contentOfType", tokens: new string[] { x.Name }) }) + .Select(x => new ContentTypeModel { Alias = x.Alias, Name = Services.TextService.Localize("template", "contentOfType", tokens: new string[] { x.Name }) }) .OrderBy(x => x.Name).ToList(); - contentTypes.Insert(0, new ContentTypeModel { Alias = string.Empty, Name = Services.TextService.Localize("template/allContent") }); + contentTypes.Insert(0, new ContentTypeModel { Alias = string.Empty, Name = Services.TextService.Localize("template", "allContent") }); return contentTypes; } diff --git a/src/Umbraco.Web/Editors/UserGroupsController.cs b/src/Umbraco.Web/Editors/UserGroupsController.cs index b081ca6137..03e6c5b8b7 100644 --- a/src/Umbraco.Web/Editors/UserGroupsController.cs +++ b/src/Umbraco.Web/Editors/UserGroupsController.cs @@ -78,7 +78,7 @@ namespace Umbraco.Web.Editors var display = Mapper.Map(userGroupSave.PersistedUserGroup); - display.AddSuccessNotification(Services.TextService.Localize("speechBubbles/operationSavedHeader"), Services.TextService.Localize("speechBubbles/editUserGroupSaved")); + display.AddSuccessNotification(Services.TextService.Localize("speechBubbles", "operationSavedHeader"), Services.TextService.Localize("speechBubbles", "editUserGroupSaved")); return display; } @@ -163,9 +163,9 @@ namespace Umbraco.Web.Editors } if (userGroups.Length > 1) return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/deleteUserGroupsSuccess", new[] {userGroups.Length.ToString()})); + Services.TextService.Localize("speechBubbles", "deleteUserGroupsSuccess", new[] {userGroups.Length.ToString()})); return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/deleteUserGroupSuccess", new[] {userGroups[0].Name})); + Services.TextService.Localize("speechBubbles", "deleteUserGroupSuccess", new[] {userGroups[0].Name})); } } } diff --git a/src/Umbraco.Web/Editors/UsersController.cs b/src/Umbraco.Web/Editors/UsersController.cs index b022e6f27a..72162dd179 100644 --- a/src/Umbraco.Web/Editors/UsersController.cs +++ b/src/Umbraco.Web/Editors/UsersController.cs @@ -470,7 +470,7 @@ namespace Umbraco.Web.Editors await SendUserInviteEmailAsync(display, Security.CurrentUser.Name, Security.CurrentUser.Email, user, userSave.Message); } - display.AddSuccessNotification(Services.TextService.Localize("speechBubbles/resendInviteHeader"), Services.TextService.Localize("speechBubbles/resendInviteSuccess", new[] { user.Name })); + display.AddSuccessNotification(Services.TextService.Localize("speechBubbles", "resendInviteHeader"), Services.TextService.Localize("speechBubbles", "resendInviteSuccess", new[] { user.Name })); return display; } @@ -529,10 +529,10 @@ namespace Umbraco.Web.Editors var applicationUri = RuntimeState.ApplicationUrl; var inviteUri = new Uri(applicationUri, action); - var emailSubject = Services.TextService.Localize("user/inviteEmailCopySubject", + var emailSubject = Services.TextService.Localize("user", "inviteEmailCopySubject", //Ensure the culture of the found user is used for the email! UserExtensions.GetUserCulture(to.Language, Services.TextService, GlobalSettings)); - var emailBody = Services.TextService.Localize("user/inviteEmailCopyFormat", + var emailBody = Services.TextService.Localize("user", "inviteEmailCopyFormat", //Ensure the culture of the found user is used for the email! UserExtensions.GetUserCulture(to.Language, Services.TextService, GlobalSettings), new[] { userDisplay.Name, from, message, inviteUri.ToString(), fromEmail }); @@ -678,7 +678,7 @@ namespace Umbraco.Web.Editors if (passwordChangeResult.Success) { var result = new ModelWithNotifications(passwordChangeResult.Result.ResetPassword); - result.AddSuccessNotification(Services.TextService.Localize("general/success"), Services.TextService.Localize("user/passwordChangedGeneric")); + result.AddSuccessNotification(Services.TextService.Localize("general", "success"), Services.TextService.Localize("user", "passwordChangedGeneric")); return result; } @@ -716,11 +716,11 @@ namespace Umbraco.Web.Editors if (users.Length > 1) { return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/disableUsersSuccess", new[] {userIds.Length.ToString()})); + Services.TextService.Localize("speechBubbles", "disableUsersSuccess", new[] {userIds.Length.ToString()})); } return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/disableUserSuccess", new[] { users[0].Name })); + Services.TextService.Localize("speechBubbles", "disableUserSuccess", new[] { users[0].Name })); } /// @@ -740,11 +740,11 @@ namespace Umbraco.Web.Editors if (users.Length > 1) { return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/enableUsersSuccess", new[] { userIds.Length.ToString() })); + Services.TextService.Localize("speechBubbles", "enableUsersSuccess", new[] { userIds.Length.ToString() })); } return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/enableUserSuccess", new[] { users[0].Name })); + Services.TextService.Localize("speechBubbles", "enableUserSuccess", new[] { users[0].Name })); } /// @@ -767,7 +767,7 @@ namespace Umbraco.Web.Editors } var user = await UserManager.FindByIdAsync(userIds[0]); return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/unlockUserSuccess", new[] { user.Name })); + Services.TextService.Localize("speechBubbles", "unlockUserSuccess", new[] { user.Name })); } foreach (var u in userIds) @@ -781,7 +781,7 @@ namespace Umbraco.Web.Editors } return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/unlockUsersSuccess", new[] { userIds.Length.ToString() })); + Services.TextService.Localize("speechBubbles", "unlockUsersSuccess", new[] { userIds.Length.ToString() })); } [AdminUsersAuthorize("userIds")] @@ -799,7 +799,7 @@ namespace Umbraco.Web.Editors } Services.UserService.Save(users); return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/setUserGroupOnUsersSuccess")); + Services.TextService.Localize("speechBubbles", "setUserGroupOnUsersSuccess")); } /// @@ -830,7 +830,7 @@ namespace Umbraco.Web.Editors Services.UserService.Delete(user, true); return Request.CreateNotificationSuccessResponse( - Services.TextService.Localize("speechBubbles/deleteUserSuccess", new[] { userName })); + Services.TextService.Localize("speechBubbles", "deleteUserSuccess", new[] { userName })); } public class PagedUserResult : PagedResult diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/AbstractConfigCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/AbstractConfigCheck.cs index 01863dac0c..f9e4c85f31 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/AbstractConfigCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/AbstractConfigCheck.cs @@ -74,7 +74,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/checkSuccessMessage", + return TextService.Localize("healthcheck", "checkSuccessMessage", new[] { CurrentValue, Values.First(v => v.IsRecommended).Value, XPath, AbsoluteFilePath }); } } @@ -87,9 +87,9 @@ namespace Umbraco.Web.HealthCheck.Checks.Config get { return ValueComparisonType == ValueComparisonType.ShouldEqual - ? TextService.Localize("healthcheck/checkErrorMessageDifferentExpectedValue", + ? TextService.Localize("healthcheck", "checkErrorMessageDifferentExpectedValue", new[] { CurrentValue, Values.First(v => v.IsRecommended).Value, XPath, AbsoluteFilePath }) - : TextService.Localize("healthcheck/checkErrorMessageUnexpectedValue", + : TextService.Localize("healthcheck", "checkErrorMessageUnexpectedValue", new[] { CurrentValue, Values.First(v => v.IsRecommended).Value, XPath, AbsoluteFilePath }); } } @@ -105,7 +105,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config var rectifiedValue = recommendedValue != null ? recommendedValue.Value : ProvidedValue; - return TextService.Localize("healthcheck/rectifySuccessMessage", + return TextService.Localize("healthcheck", "rectifySuccessMessage", new[] { CurrentValue, @@ -156,7 +156,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config // Declare the action for rectifying the config value var rectifyAction = new HealthCheckAction("rectify", Id) { - Name = TextService.Localize("healthcheck/rectifyButton"), + Name = TextService.Localize("healthcheck", "rectifyButton"), ValueRequired = CanRectifyWithValue, }; @@ -178,7 +178,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config public virtual HealthCheckStatus Rectify() { if (ValueComparisonType == ValueComparisonType.ShouldNotEqual) - throw new InvalidOperationException(TextService.Localize("healthcheck/cannotRectifyShouldNotEqual")); + throw new InvalidOperationException(TextService.Localize("healthcheck", "cannotRectifyShouldNotEqual")); var recommendedValue = Values.First(v => v.IsRecommended).Value; return UpdateConfigurationValue(recommendedValue); @@ -192,10 +192,10 @@ namespace Umbraco.Web.HealthCheck.Checks.Config public virtual HealthCheckStatus Rectify(string value) { if (ValueComparisonType == ValueComparisonType.ShouldEqual) - throw new InvalidOperationException(TextService.Localize("healthcheck/cannotRectifyShouldEqualWithValue")); + throw new InvalidOperationException(TextService.Localize("healthcheck", "cannotRectifyShouldEqualWithValue")); if (string.IsNullOrWhiteSpace(value)) - throw new InvalidOperationException(TextService.Localize("healthcheck/valueToRectifyNotProvided")); + throw new InvalidOperationException(TextService.Localize("healthcheck", "valueToRectifyNotProvided")); // Need to track provided value in order to correctly put together the rectify message ProvidedValue = value; diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/CompilationDebugCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/CompilationDebugCheck.cs index 951609f91f..959a0cdbae 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/CompilationDebugCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/CompilationDebugCheck.cs @@ -25,10 +25,10 @@ namespace Umbraco.Web.HealthCheck.Checks.Config new AcceptableConfiguration { IsRecommended = true, Value = bool.FalseString.ToLower() } }; - public override string CheckSuccessMessage => TextService.Localize("healthcheck/compilationDebugCheckSuccessMessage"); + public override string CheckSuccessMessage => TextService.Localize("healthcheck", "compilationDebugCheckSuccessMessage"); - public override string CheckErrorMessage => TextService.Localize("healthcheck/compilationDebugCheckErrorMessage"); + public override string CheckErrorMessage => TextService.Localize("healthcheck", "compilationDebugCheckErrorMessage"); - public override string RectifySuccessMessage => TextService.Localize("healthcheck/compilationDebugCheckRectifySuccessMessage"); + public override string RectifySuccessMessage => TextService.Localize("healthcheck", "compilationDebugCheckRectifySuccessMessage"); } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/ConfigurationService.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/ConfigurationService.cs index 95b458b142..8801ab0030 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/ConfigurationService.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/ConfigurationService.cs @@ -36,7 +36,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config return new ConfigurationServiceResult { Success = false, - Result = _textService.Localize("healthcheck/configurationServiceFileNotFound", new[] { _configFilePath }) + Result = _textService.Localize("healthcheck", "configurationServiceFileNotFound", new[] { _configFilePath }) }; var xmlDocument = new XmlDocument(); @@ -47,7 +47,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config return new ConfigurationServiceResult { Success = false, - Result = _textService.Localize("healthcheck/configurationServiceNodeNotFound", new[] { _xPath, _configFilePath }) + Result = _textService.Localize("healthcheck", "configurationServiceNodeNotFound", new[] { _xPath, _configFilePath }) }; return new ConfigurationServiceResult @@ -62,7 +62,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config return new ConfigurationServiceResult { Success = false, - Result = _textService.Localize("healthcheck/configurationServiceError", new[] { ex.Message }) + Result = _textService.Localize("healthcheck", "configurationServiceError", new[] { ex.Message }) }; } } @@ -80,7 +80,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config return new ConfigurationServiceResult { Success = false, - Result = _textService.Localize("healthcheck/configurationServiceFileNotFound", new[] { _configFilePath }) + Result = _textService.Localize("healthcheck", "configurationServiceFileNotFound", new[] { _configFilePath }) }; var xmlDocument = new XmlDocument { PreserveWhitespace = true }; @@ -91,7 +91,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config return new ConfigurationServiceResult { Success = false, - Result = _textService.Localize("healthcheck/configurationServiceNodeNotFound", new[] { _xPath, _configFilePath }) + Result = _textService.Localize("healthcheck", "configurationServiceNodeNotFound", new[] { _xPath, _configFilePath }) }; if (node.NodeType == XmlNodeType.Element) @@ -108,7 +108,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config return new ConfigurationServiceResult { Success = false, - Result = _textService.Localize("healthcheck/configurationServiceError", new[] { ex.Message }) + Result = _textService.Localize("healthcheck", "configurationServiceError", new[] { ex.Message }) }; } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/CustomErrorsCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/CustomErrorsCheck.cs index 63986b6c62..9b52e9b8e9 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/CustomErrorsCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/CustomErrorsCheck.cs @@ -30,7 +30,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/customErrorsCheckSuccessMessage", new[] { CurrentValue }); + return TextService.Localize("healthcheck", "customErrorsCheckSuccessMessage", new[] { CurrentValue }); } } @@ -38,7 +38,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/customErrorsCheckErrorMessage", + return TextService.Localize("healthcheck", "customErrorsCheckErrorMessage", new[] { CurrentValue, Values.First(v => v.IsRecommended).Value }); } } @@ -47,7 +47,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/customErrorsCheckRectifySuccessMessage", + return TextService.Localize("healthcheck", "customErrorsCheckRectifySuccessMessage", new[] { Values.First(v => v.IsRecommended).Value }); } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/MacroErrorsCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/MacroErrorsCheck.cs index 09c041998e..1e0283321f 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/MacroErrorsCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/MacroErrorsCheck.cs @@ -45,7 +45,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/macroErrorModeCheckSuccessMessage", + return TextService.Localize("healthcheck", "macroErrorModeCheckSuccessMessage", new[] { CurrentValue, Values.First(v => v.IsRecommended).Value }); } } @@ -54,7 +54,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/macroErrorModeCheckErrorMessage", + return TextService.Localize("healthcheck", "macroErrorModeCheckErrorMessage", new[] { CurrentValue, Values.First(v => v.IsRecommended).Value }); } } @@ -63,7 +63,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/macroErrorModeCheckRectifySuccessMessage", + return TextService.Localize("healthcheck", "macroErrorModeCheckRectifySuccessMessage", new[] { Values.First(v => v.IsRecommended).Value }); } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/NotificationEmailCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/NotificationEmailCheck.cs index 8c52a5686b..af9494ced7 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/NotificationEmailCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/NotificationEmailCheck.cs @@ -25,8 +25,8 @@ namespace Umbraco.Web.HealthCheck.Checks.Config new AcceptableConfiguration { IsRecommended = false, Value = DefaultFromEmail } }; - public override string CheckSuccessMessage => TextService.Localize("healthcheck/notificationEmailsCheckSuccessMessage", new [] { CurrentValue } ); + public override string CheckSuccessMessage => TextService.Localize("healthcheck", "notificationEmailsCheckSuccessMessage", new [] { CurrentValue } ); - public override string CheckErrorMessage => TextService.Localize("healthcheck/notificationEmailsCheckErrorMessage", new[] { DefaultFromEmail }); + public override string CheckErrorMessage => TextService.Localize("healthcheck", "notificationEmailsCheckErrorMessage", new[] { DefaultFromEmail }); } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/TraceCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/TraceCheck.cs index b8e4e51eb9..47c88bf250 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/TraceCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/TraceCheck.cs @@ -24,10 +24,10 @@ namespace Umbraco.Web.HealthCheck.Checks.Config new AcceptableConfiguration { IsRecommended = true, Value = bool.FalseString.ToLower() } }; - public override string CheckSuccessMessage => TextService.Localize("healthcheck/traceModeCheckSuccessMessage"); + public override string CheckSuccessMessage => TextService.Localize("healthcheck", "traceModeCheckSuccessMessage"); - public override string CheckErrorMessage => TextService.Localize("healthcheck/traceModeCheckErrorMessage"); + public override string CheckErrorMessage => TextService.Localize("healthcheck", "traceModeCheckErrorMessage"); - public override string RectifySuccessMessage => TextService.Localize("healthcheck/traceModeCheckRectifySuccessMessage"); + public override string RectifySuccessMessage => TextService.Localize("healthcheck", "traceModeCheckRectifySuccessMessage"); } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Config/TrySkipIisCustomErrorsCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Config/TrySkipIisCustomErrorsCheck.cs index 4a467d7120..2c4e59091e 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Config/TrySkipIisCustomErrorsCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Config/TrySkipIisCustomErrorsCheck.cs @@ -39,7 +39,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/trySkipIisCustomErrorsCheckSuccessMessage", + return TextService.Localize("healthcheck", "trySkipIisCustomErrorsCheckSuccessMessage", new[] { Values.First(v => v.IsRecommended).Value, _serverVersion.ToString() }); } } @@ -48,7 +48,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/trySkipIisCustomErrorsCheckErrorMessage", + return TextService.Localize("healthcheck", "trySkipIisCustomErrorsCheckErrorMessage", new[] { CurrentValue, Values.First(v => v.IsRecommended).Value, _serverVersion.ToString() }); } } @@ -57,7 +57,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Config { get { - return TextService.Localize("healthcheck/trySkipIisCustomErrorsCheckRectifySuccessMessage", + return TextService.Localize("healthcheck", "trySkipIisCustomErrorsCheckRectifySuccessMessage", new[] { Values.First(v => v.IsRecommended).Value, _serverVersion.ToString() }); } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Permissions/FolderAndFilePermissionsCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Permissions/FolderAndFilePermissionsCheck.cs index ee6f254235..218cb2cc9c 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Permissions/FolderAndFilePermissionsCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Permissions/FolderAndFilePermissionsCheck.cs @@ -135,22 +135,20 @@ namespace Umbraco.Web.HealthCheck.Checks.Permissions { // Return error if any required paths fail the check, or warning if any optional ones do var resultType = StatusResultType.Success; - var messageKey = string.Format("healthcheck/{0}PermissionsCheckMessage", - checkingFor == PermissionCheckFor.Folder ? "folder" : "file"); - var message = _textService.Localize(messageKey); + var messageArea = "healthcheck"; + var messageAlias = string.Concat(checkingFor == PermissionCheckFor.Folder ? "folder" : "file", "PermissionsCheckMessage"); + var message = _textService.Localize(messageArea, messageAlias); if (requiredPathCheckResult == false) { resultType = StatusResultType.Error; - messageKey = string.Format("healthcheck/required{0}PermissionFailed", - checkingFor == PermissionCheckFor.Folder ? "Folder" : "File"); - message = GetMessageForPathCheckFailure(messageKey, requiredFailedPaths); + messageAlias = string.Concat("required", checkingFor == PermissionCheckFor.Folder ? "Folder" : "File", "PermissionFailed"); + message = GetMessageForPathCheckFailure(messageArea, messageAlias, requiredFailedPaths); } else if (optionalPathCheckResult == false) { resultType = StatusResultType.Warning; - messageKey = string.Format("healthcheck/optional{0}PermissionFailed", - checkingFor == PermissionCheckFor.Folder ? "Folder" : "File"); - message = GetMessageForPathCheckFailure(messageKey, optionalFailedPaths); + messageAlias = string.Concat("optional", checkingFor == PermissionCheckFor.Folder ? "Folder" : "File", "PermissionFailed"); + message = GetMessageForPathCheckFailure(messageArea, messageAlias, optionalFailedPaths); } var actions = new List(); @@ -162,12 +160,12 @@ namespace Umbraco.Web.HealthCheck.Checks.Permissions }; } - private string GetMessageForPathCheckFailure(string messageKey, IEnumerable failedPaths) + private string GetMessageForPathCheckFailure(string messageArea,string messageAlias, IEnumerable failedPaths) { var rootFolder = IOHelper.MapPath("/"); var failedFolders = failedPaths .Select(x => ParseFolderFromFullPath(rootFolder, x)); - return _textService.Localize(messageKey, + return _textService.Localize(messageArea, messageAlias, new[] { string.Join(", ", failedFolders) }); } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs index 5e66bc47b1..6bb32d9d74 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/BaseHttpHeaderCheck.cs @@ -87,12 +87,12 @@ namespace Umbraco.Web.HealthCheck.Checks.Security } message = success - ? TextService.Localize($"healthcheck/{_localizedTextPrefix}CheckHeaderFound") - : TextService.Localize($"healthcheck/{_localizedTextPrefix}CheckHeaderNotFound"); + ? TextService.Localize($"healthcheck", "{_localizedTextPrefix}CheckHeaderFound") + : TextService.Localize($"healthcheck", "{_localizedTextPrefix}CheckHeaderNotFound"); } catch (Exception ex) { - message = TextService.Localize("healthcheck/healthCheckInvalidUrl", new[] { url.ToString(), ex.Message }); + message = TextService.Localize("healthcheck", "healthCheckInvalidUrl", new[] { url.ToString(), ex.Message }); } var actions = new List(); @@ -100,8 +100,8 @@ namespace Umbraco.Web.HealthCheck.Checks.Security { actions.Add(new HealthCheckAction(SetHeaderInConfigAction, Id) { - Name = TextService.Localize("healthcheck/setHeaderInConfig"), - Description = TextService.Localize($"healthcheck/{_localizedTextPrefix}SetHeaderInConfigDescription") + Name = TextService.Localize("healthcheck", "setHeaderInConfig"), + Description = TextService.Localize($"healthcheck", "{_localizedTextPrefix}SetHeaderInConfigDescription") }); } @@ -149,14 +149,14 @@ namespace Umbraco.Web.HealthCheck.Checks.Security if (success) { return - new HealthCheckStatus(TextService.Localize(string.Format("healthcheck/{0}SetHeaderInConfigSuccess", _localizedTextPrefix))) + new HealthCheckStatus(TextService.Localize("healthcheck", _localizedTextPrefix + "SetHeaderInConfigSuccess")) { ResultType = StatusResultType.Success }; } return - new HealthCheckStatus(TextService.Localize("healthcheck/setHeaderInConfigError", new [] { errorMessage })) + new HealthCheckStatus(TextService.Localize("healthcheck", "setHeaderInConfigError", new [] { errorMessage })) { ResultType = StatusResultType.Error }; diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs index fd76b9d486..28904cc6bd 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs @@ -64,12 +64,12 @@ namespace Umbraco.Web.HealthCheck.Checks.Security .ToArray(); success = headersFound.Any() == false; message = success - ? _textService.Localize("healthcheck/excessiveHeadersNotFound") - : _textService.Localize("healthcheck/excessiveHeadersFound", new [] { string.Join(", ", headersFound) }); + ? _textService.Localize("healthcheck", "excessiveHeadersNotFound") + : _textService.Localize("healthcheck", "excessiveHeadersFound", new [] { string.Join(", ", headersFound) }); } catch (Exception ex) { - message = _textService.Localize("healthcheck/httpsCheckInvalidUrl", new[] { url.ToString(), ex.Message }); + message = _textService.Localize("healthcheck", "httpsCheckInvalidUrl", new[] { url.ToString(), ex.Message }); } var actions = new List(); diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs index 98f8a83c1d..51b253fe94 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs @@ -84,23 +84,23 @@ namespace Umbraco.Web.HealthCheck.Checks.Security if (daysToExpiry <= 0) { result = StatusResultType.Error; - message = _textService.Localize("healthcheck/httpsCheckExpiredCertificate"); + message = _textService.Localize("healthcheck", "httpsCheckExpiredCertificate"); } else if (daysToExpiry < NumberOfDaysForExpiryWarning) { result = StatusResultType.Warning; - message = _textService.Localize("healthcheck/httpsCheckExpiringCertificate", new[] { daysToExpiry.ToString() }); + message = _textService.Localize("healthcheck", "httpsCheckExpiringCertificate", new[] { daysToExpiry.ToString() }); } else { result = StatusResultType.Success; - message = _textService.Localize("healthcheck/httpsCheckValidCertificate"); + message = _textService.Localize("healthcheck", "httpsCheckValidCertificate"); } } else { result = StatusResultType.Error; - message = _textService.Localize("healthcheck/healthCheckInvalidUrl", new[] { url, response.StatusDescription }); + message = _textService.Localize("healthcheck", "healthCheckInvalidUrl", new[] { url, response.StatusDescription }); } } catch (Exception ex) @@ -109,12 +109,12 @@ namespace Umbraco.Web.HealthCheck.Checks.Security if (exception != null) { message = exception.Status == WebExceptionStatus.TrustFailure - ? _textService.Localize("healthcheck/httpsCheckInvalidCertificate", new [] { exception.Message }) - : _textService.Localize("healthcheck/healthCheckInvalidUrl", new [] { url, exception.Message }); + ? _textService.Localize("healthcheck", "httpsCheckInvalidCertificate", new [] { exception.Message }) + : _textService.Localize("healthcheck", "healthCheckInvalidUrl", new [] { url, exception.Message }); } else { - message = _textService.Localize("healthcheck/healthCheckInvalidUrl", new[] { url, ex.Message }); + message = _textService.Localize("healthcheck", "healthCheckInvalidUrl", new[] { url, ex.Message }); } result = StatusResultType.Error; @@ -138,7 +138,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Security var actions = new List(); return - new HealthCheckStatus(_textService.Localize("healthcheck/httpsCheckIsCurrentSchemeHttps", new[] { success ? string.Empty : "not" })) + new HealthCheckStatus(_textService.Localize("healthcheck", "httpsCheckIsCurrentSchemeHttps", new[] { success ? string.Empty : "not" })) { ResultType = success ? StatusResultType.Success : StatusResultType.Error, Actions = actions @@ -155,7 +155,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Security StatusResultType resultType; if (uri.Scheme != "https") { - resultMessage = _textService.Localize("healthcheck/httpsCheckConfigurationRectifyNotPossible"); + resultMessage = _textService.Localize("healthcheck", "httpsCheckConfigurationRectifyNotPossible"); resultType = StatusResultType.Info; } else @@ -163,11 +163,11 @@ namespace Umbraco.Web.HealthCheck.Checks.Security if (httpsSettingEnabled == false) actions.Add(new HealthCheckAction(FixHttpsSettingAction, Id) { - Name = _textService.Localize("healthcheck/httpsCheckEnableHttpsButton"), - Description = _textService.Localize("healthcheck/httpsCheckEnableHttpsDescription") + Name = _textService.Localize("healthcheck", "httpsCheckEnableHttpsButton"), + Description = _textService.Localize("healthcheck", "httpsCheckEnableHttpsDescription") }); - resultMessage = _textService.Localize("healthcheck/httpsCheckConfigurationCheckResult", + resultMessage = _textService.Localize("healthcheck", "httpsCheckConfigurationCheckResult", new[] {httpsSettingEnabled.ToString(), httpsSettingEnabled ? string.Empty : "not"}); resultType = httpsSettingEnabled ? StatusResultType.Success: StatusResultType.Error; } @@ -190,14 +190,14 @@ namespace Umbraco.Web.HealthCheck.Checks.Security if (updateConfigFile.Success) { return - new HealthCheckStatus(_textService.Localize("healthcheck/httpsCheckEnableHttpsSuccess")) + new HealthCheckStatus(_textService.Localize("healthcheck", "httpsCheckEnableHttpsSuccess")) { ResultType = StatusResultType.Success }; } return - new HealthCheckStatus(_textService.Localize("healthcheck/httpsCheckEnableHttpsError", new [] { updateConfigFile.Result })) + new HealthCheckStatus(_textService.Localize("healthcheck", "httpsCheckEnableHttpsError", new [] { updateConfigFile.Result })) { ResultType = StatusResultType.Error }; diff --git a/src/Umbraco.Web/HealthCheck/Checks/Services/SmtpCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Services/SmtpCheck.cs index d6f7cef497..2627e24fb8 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Services/SmtpCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Services/SmtpCheck.cs @@ -56,7 +56,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Services var settings = (MailSettingsSectionGroup)config.GetSectionGroup("system.net/mailSettings"); if (settings == null) { - message = _textService.Localize("healthcheck/smtpMailSettingsNotFound"); + message = _textService.Localize("healthcheck", "smtpMailSettingsNotFound"); } else { @@ -64,14 +64,14 @@ namespace Umbraco.Web.HealthCheck.Checks.Services var port = settings.Smtp.Network.Port == 0 ? DefaultSmtpPort : settings.Smtp.Network.Port; if (string.IsNullOrEmpty(host)) { - message = _textService.Localize("healthcheck/smtpMailSettingsHostNotConfigured"); + message = _textService.Localize("healthcheck", "smtpMailSettingsHostNotConfigured"); } else { success = CanMakeSmtpConnection(host, port); message = success - ? _textService.Localize("healthcheck/smtpMailSettingsConnectionSuccess") - : _textService.Localize("healthcheck/smtpMailSettingsConnectionFail", new [] { host, port.ToString() }); + ? _textService.Localize("healthcheck", "smtpMailSettingsConnectionSuccess") + : _textService.Localize("healthcheck", "smtpMailSettingsConnectionFail", new [] { host, port.ToString() }); } } diff --git a/src/Umbraco.Web/HealthCheck/NotificationMethods/EmailNotificationMethod.cs b/src/Umbraco.Web/HealthCheck/NotificationMethods/EmailNotificationMethod.cs index 873b356214..54e4f2c52e 100644 --- a/src/Umbraco.Web/HealthCheck/NotificationMethods/EmailNotificationMethod.cs +++ b/src/Umbraco.Web/HealthCheck/NotificationMethods/EmailNotificationMethod.cs @@ -48,7 +48,7 @@ namespace Umbraco.Web.HealthCheck.NotificationMethods return; } - var message = _textService.Localize("healthcheck/scheduledHealthCheckEmailBody", new[] + var message = _textService.Localize("healthcheck", "scheduledHealthCheckEmailBody", new[] { DateTime.Now.ToShortDateString(), DateTime.Now.ToShortTimeString(), @@ -59,7 +59,7 @@ namespace Umbraco.Web.HealthCheck.NotificationMethods // you can identify the site that these results are for. var host = _runtimeState.ApplicationUrl; - var subject = _textService.Localize("healthcheck/scheduledHealthCheckEmailSubject", new[] { host.ToString() }); + var subject = _textService.Localize("healthcheck", "scheduledHealthCheckEmailSubject", new[] { host.ToString() }); var mailSender = new EmailSender(); using (var mailMessage = CreateMailMessage(subject, message)) diff --git a/src/Umbraco.Web/Macros/MacroRenderer.cs b/src/Umbraco.Web/Macros/MacroRenderer.cs index b4fd8c0d86..4538dc03c6 100755 --- a/src/Umbraco.Web/Macros/MacroRenderer.cs +++ b/src/Umbraco.Web/Macros/MacroRenderer.cs @@ -340,14 +340,14 @@ namespace Umbraco.Web.Macros $"Executing PartialView: MacroSource=\"{model.MacroSource}\".", "Executed PartialView.", () => ExecutePartialView(model, content), - () => textService.Localize("errors/macroErrorLoadingPartialView", new[] { model.MacroSource })); + () => textService.Localize("errors", "macroErrorLoadingPartialView", new[] { model.MacroSource })); default: return ExecuteMacroWithErrorWrapper(model, $"Execute macro with unsupported type \"{model.MacroType}\".", "Executed.", () => { throw new Exception("Unsupported macro type."); }, - () => textService.Localize("errors/macroErrorUnsupportedType")); + () => textService.Localize("errors", "macroErrorUnsupportedType")); } } diff --git a/src/Umbraco.Web/Models/Mapping/CommonMapper.cs b/src/Umbraco.Web/Models/Mapping/CommonMapper.cs index f7db17ff74..d04579d5ee 100644 --- a/src/Umbraco.Web/Models/Mapping/CommonMapper.cs +++ b/src/Umbraco.Web/Models/Mapping/CommonMapper.cs @@ -80,7 +80,7 @@ namespace Umbraco.Web.Models.Mapping // localize content app names foreach (var app in apps) { - var localizedAppName = _localizedTextService.Localize($"apps/{app.Alias}"); + var localizedAppName = _localizedTextService.Localize($"apps", "{app.Alias}"); if (localizedAppName.Equals($"[{app.Alias}]", StringComparison.OrdinalIgnoreCase) == false) { app.Name = localizedAppName; diff --git a/src/Umbraco.Web/Models/Mapping/ContentVariantMapper.cs b/src/Umbraco.Web/Models/Mapping/ContentVariantMapper.cs index 0b6be53045..7b240d2981 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentVariantMapper.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentVariantMapper.cs @@ -153,7 +153,7 @@ namespace Umbraco.Web.Models.Mapping if(!isCultureVariant && !isSegmentVariant) { - return _localizedTextService.Localize("general/default"); + return _localizedTextService.Localize("general", "default"); } var parts = new List(); diff --git a/src/Umbraco.Web/Models/Mapping/MemberTabsAndPropertiesMapper.cs b/src/Umbraco.Web/Models/Mapping/MemberTabsAndPropertiesMapper.cs index 8744b068a7..61c6684093 100644 --- a/src/Umbraco.Web/Models/Mapping/MemberTabsAndPropertiesMapper.cs +++ b/src/Umbraco.Web/Models/Mapping/MemberTabsAndPropertiesMapper.cs @@ -61,7 +61,7 @@ namespace Umbraco.Web.Models.Mapping if (isLockedOutProperty?.Value != null && isLockedOutProperty.Value.ToString() != "1") { isLockedOutProperty.View = "readonlyvalue"; - isLockedOutProperty.Value = _localizedTextService.Localize("general/no"); + isLockedOutProperty.Value = _localizedTextService.Localize("general", "no"); } } else @@ -75,7 +75,7 @@ namespace Umbraco.Web.Models.Mapping if (isLockedOutProperty?.Value != null && isLockedOutProperty.Value.ToString() != "1") { isLockedOutProperty.View = "readonlyvalue"; - isLockedOutProperty.Value = _localizedTextService.Localize("general/no"); + isLockedOutProperty.Value = _localizedTextService.Localize("general", "no"); } } @@ -115,14 +115,14 @@ namespace Umbraco.Web.Models.Mapping new ContentPropertyDisplay { Alias = $"{Constants.PropertyEditors.InternalGenericPropertiesPrefix}id", - Label = _localizedTextService.Localize("general/id"), + Label = _localizedTextService.Localize("general","id"), Value = new List {member.Id.ToString(), member.Key.ToString()}, View = "idwithguid" }, new ContentPropertyDisplay { Alias = $"{Constants.PropertyEditors.InternalGenericPropertiesPrefix}doctype", - Label = _localizedTextService.Localize("content/membertype"), + Label = _localizedTextService.Localize("content","membertype"), Value = _localizedTextService.UmbracoDictionaryTranslate(member.ContentType.Name), View = Current.PropertyEditors[Constants.PropertyEditors.Aliases.Label].GetValueEditor().View }, @@ -130,7 +130,7 @@ namespace Umbraco.Web.Models.Mapping new ContentPropertyDisplay { Alias = $"{Constants.PropertyEditors.InternalGenericPropertiesPrefix}email", - Label = _localizedTextService.Localize("general/email"), + Label = _localizedTextService.Localize("general","email"), Value = member.Email, View = "email", Validation = {Mandatory = true} @@ -138,7 +138,7 @@ namespace Umbraco.Web.Models.Mapping new ContentPropertyDisplay { Alias = $"{Constants.PropertyEditors.InternalGenericPropertiesPrefix}password", - Label = _localizedTextService.Localize("password"), + Label = _localizedTextService.Localize(null,"password"), // NOTE: The value here is a json value - but the only property we care about is the generatedPassword one if it exists, the newPassword exists // only when creating a new member and we want to have a generated password pre-filled. Value = new Dictionary @@ -159,7 +159,7 @@ namespace Umbraco.Web.Models.Mapping new ContentPropertyDisplay { Alias = $"{Constants.PropertyEditors.InternalGenericPropertiesPrefix}membergroup", - Label = _localizedTextService.Localize("content/membergroup"), + Label = _localizedTextService.Localize("content","membergroup"), Value = GetMemberGroupValue(member.Username), View = "membergroups", Config = new Dictionary {{"IsRequired", true}} @@ -223,7 +223,7 @@ namespace Umbraco.Web.Models.Mapping var prop = new ContentPropertyDisplay { Alias = $"{Constants.PropertyEditors.InternalGenericPropertiesPrefix}login", - Label = localizedText.Localize("login"), + Label = localizedText.Localize(null,"login"), Value = member.Username }; diff --git a/src/Umbraco.Web/Models/Mapping/SectionMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/SectionMapDefinition.cs index e05e6e5c84..de6a92549a 100644 --- a/src/Umbraco.Web/Models/Mapping/SectionMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/SectionMapDefinition.cs @@ -35,7 +35,7 @@ namespace Umbraco.Web.Models.Mapping private void Map(ISection source, Section target, MapperContext context) { target.Alias = source.Alias; - target.Name = _textService.Localize("sections/" + source.Alias); + target.Name = _textService.Localize("sections", source.Alias); } // Umbraco.Code.MapAll diff --git a/src/Umbraco.Web/Models/Mapping/TabsAndPropertiesMapper.cs b/src/Umbraco.Web/Models/Mapping/TabsAndPropertiesMapper.cs index b8d76572fb..85af1dbe9e 100644 --- a/src/Umbraco.Web/Models/Mapping/TabsAndPropertiesMapper.cs +++ b/src/Umbraco.Web/Models/Mapping/TabsAndPropertiesMapper.cs @@ -58,7 +58,7 @@ namespace Umbraco.Web.Models.Mapping tabs.Add(new Tab { Id = 0, - Label = LocalizedTextService.Localize("general/properties"), + Label = LocalizedTextService.Localize("general", "properties"), Alias = "Generic properties", Properties = genericproperties }); diff --git a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs index aa158799cb..924ffda554 100644 --- a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs @@ -273,8 +273,8 @@ namespace Umbraco.Web.Models.Mapping { target.AvailableCultures = _textService.GetSupportedCultures().ToDictionary(x => x.Name, x => x.DisplayName); target.Avatars = source.GetUserAvatarUrls(_appCaches.RuntimeCache); - target.CalculatedStartContentIds = GetStartNodes(source.CalculateContentStartNodeIds(_entityService), UmbracoObjectTypes.Document, "content/contentRoot", context); - target.CalculatedStartMediaIds = GetStartNodes(source.CalculateMediaStartNodeIds(_entityService), UmbracoObjectTypes.Media, "media/mediaRoot", context); + target.CalculatedStartContentIds = GetStartNodes(source.CalculateContentStartNodeIds(_entityService), UmbracoObjectTypes.Document, "content","contentRoot", context); + target.CalculatedStartMediaIds = GetStartNodes(source.CalculateMediaStartNodeIds(_entityService), UmbracoObjectTypes.Media, "media","mediaRoot", context); target.CreateDate = source.CreateDate; target.Culture = source.GetUserCulture(_textService, _globalSettings).ToString(); target.Email = source.Email; @@ -289,8 +289,8 @@ namespace Umbraco.Web.Models.Mapping target.Navigation = CreateUserEditorNavigation(); target.ParentId = -1; target.Path = "-1," + source.Id; - target.StartContentIds = GetStartNodes(source.StartContentIds.ToArray(), UmbracoObjectTypes.Document, "content/contentRoot", context); - target.StartMediaIds = GetStartNodes(source.StartMediaIds.ToArray(), UmbracoObjectTypes.Media, "media/mediaRoot", context); + target.StartContentIds = GetStartNodes(source.StartContentIds.ToArray(), UmbracoObjectTypes.Document, "content","contentRoot", context); + target.StartMediaIds = GetStartNodes(source.StartMediaIds.ToArray(), UmbracoObjectTypes.Media, "media","mediaRoot", context); target.UpdateDate = source.UpdateDate; target.UserGroups = context.MapEnumerable(source.Groups); target.Username = source.Username; @@ -347,12 +347,12 @@ namespace Umbraco.Web.Models.Mapping if (sourceStartMediaId > 0) target.MediaStartNode = context.Map(_entityService.Get(sourceStartMediaId.Value, UmbracoObjectTypes.Media)); else if (sourceStartMediaId == -1) - target.MediaStartNode = CreateRootNode(_textService.Localize("media/mediaRoot")); + target.MediaStartNode = CreateRootNode(_textService.Localize("media", "mediaRoot")); if (sourceStartContentId > 0) target.ContentStartNode = context.Map(_entityService.Get(sourceStartContentId.Value, UmbracoObjectTypes.Document)); else if (sourceStartContentId == -1) - target.ContentStartNode = CreateRootNode(_textService.Localize("content/contentRoot")); + target.ContentStartNode = CreateRootNode(_textService.Localize("content", "contentRoot")); if (target.Icon.IsNullOrWhiteSpace()) target.Icon = Constants.Icons.UserGroup; @@ -364,10 +364,10 @@ namespace Umbraco.Web.Models.Mapping => new Permission { Category = action.Category.IsNullOrWhiteSpace() - ? _textService.Localize($"actionCategories/{Constants.Conventions.PermissionCategories.OtherCategory}") - : _textService.Localize($"actionCategories/{action.Category}"), - Name = _textService.Localize($"actions/{action.Alias}"), - Description = _textService.Localize($"actionDescriptions/{action.Alias}"), + ? _textService.Localize("actionCategories",Constants.Conventions.PermissionCategories.OtherCategory) + : _textService.Localize("actionCategories", action.Category), + Name = _textService.Localize("actions", action.Alias), + Description = _textService.Localize("actionDescriptions", action.Alias), Icon = action.Icon, Checked = source.Permissions != null && source.Permissions.Contains(action.Letter.ToString(CultureInfo.InvariantCulture)), PermissionCode = action.Letter.ToString(CultureInfo.InvariantCulture) @@ -383,14 +383,14 @@ namespace Umbraco.Web.Models.Mapping private static string MapContentTypeIcon(IEntitySlim entity) => entity is IContentEntitySlim contentEntity ? contentEntity.ContentTypeIcon : null; - private IEnumerable GetStartNodes(int[] startNodeIds, UmbracoObjectTypes objectType, string localizedKey, MapperContext context) + private IEnumerable GetStartNodes(int[] startNodeIds, UmbracoObjectTypes objectType, string localizedArea,string localizedAlias, MapperContext context) { if (startNodeIds.Length <= 0) return Enumerable.Empty(); var startNodes = new List(); if (startNodeIds.Contains(-1)) - startNodes.Add(CreateRootNode(_textService.Localize(localizedKey))); + startNodes.Add(CreateRootNode(_textService.Localize(localizedArea, localizedAlias))); var mediaItems = _entityService.GetAll(objectType, startNodeIds); startNodes.AddRange(context.MapEnumerable(mediaItems)); @@ -406,7 +406,7 @@ namespace Umbraco.Web.Models.Mapping Active = true, Alias = "details", Icon = "icon-umb-users", - Name = _textService.Localize("general/user"), + Name = _textService.Localize("general","user"), View = "views/users/views/user/details.html" } }; diff --git a/src/Umbraco.Web/Models/Trees/MenuItem.cs b/src/Umbraco.Web/Models/Trees/MenuItem.cs index fb4dfb836c..9d4d146596 100644 --- a/src/Umbraco.Web/Models/Trees/MenuItem.cs +++ b/src/Umbraco.Web/Models/Trees/MenuItem.cs @@ -32,12 +32,9 @@ namespace Umbraco.Web.Models.Trees public MenuItem(string alias, ILocalizedTextService textService) : this() { - var values = textService.GetAllStoredValues(Thread.CurrentThread.CurrentUICulture); - values.TryGetValue($"visuallyHiddenTexts/{alias}_description", out var textDescription); - Alias = alias; - Name = textService.Localize($"actions/{Alias}"); - TextDescription = textDescription; + Name = textService.Localize("actions", Alias); + TextDescription = textService.Localize("visuallyHiddenTexts", alias + "_description", Thread.CurrentThread.CurrentUICulture); } /// diff --git a/src/Umbraco.Web/Models/Trees/MenuItemList.cs b/src/Umbraco.Web/Models/Trees/MenuItemList.cs index 4aaf0632ab..edcfd24584 100644 --- a/src/Umbraco.Web/Models/Trees/MenuItemList.cs +++ b/src/Umbraco.Web/Models/Trees/MenuItemList.cs @@ -97,14 +97,12 @@ namespace Umbraco.Web.Models.Trees var item = Current.Actions.GetAction(); if (item == null) return null; - var values = textService.GetAllStoredValues(Thread.CurrentThread.CurrentUICulture); - values.TryGetValue($"visuallyHiddenTexts/{item.Alias}Description", out var textDescription); - var menuItem = new MenuItem(item, textService.Localize($"actions/{item.Alias}")) + var menuItem = new MenuItem(item, textService.Localize("actions",item.Alias)) { SeparatorBefore = hasSeparator, OpensDialog = opensDialog, - TextDescription = textDescription, + TextDescription = textService.Localize("visuallyHiddenTexts", item.Alias+"Description", Thread.CurrentThread.CurrentUICulture), }; return menuItem; diff --git a/src/Umbraco.Web/PropertyEditors/BlockEditorPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/BlockEditorPropertyEditor.cs index 8c4ebf49c3..705d4a706d 100644 --- a/src/Umbraco.Web/PropertyEditors/BlockEditorPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/BlockEditorPropertyEditor.cs @@ -249,7 +249,7 @@ namespace Umbraco.Web.PropertyEditors || (blockEditorData != null && validationLimit.Min.HasValue && blockEditorData.Layout.Count() < validationLimit.Min)) { yield return new ValidationResult( - _textService.Localize("validation/entriesShort", new[] + _textService.Localize("validation", "entriesShort", new[] { validationLimit.Min.ToString(), (validationLimit.Min - blockEditorData.Layout.Count()).ToString() @@ -260,7 +260,7 @@ namespace Umbraco.Web.PropertyEditors if (blockEditorData != null && validationLimit.Max.HasValue && blockEditorData.Layout.Count() > validationLimit.Max) { yield return new ValidationResult( - _textService.Localize("validation/entriesExceed", new[] + _textService.Localize("validation", "entriesExceed", new[] { validationLimit.Max.ToString(), (blockEditorData.Layout.Count() - validationLimit.Max).ToString() diff --git a/src/Umbraco.Web/PropertyEditors/DropDownFlexibleConfigurationEditor.cs b/src/Umbraco.Web/PropertyEditors/DropDownFlexibleConfigurationEditor.cs index dbdedebd0a..cf0aca7ba0 100644 --- a/src/Umbraco.Web/PropertyEditors/DropDownFlexibleConfigurationEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/DropDownFlexibleConfigurationEditor.cs @@ -14,7 +14,7 @@ namespace Umbraco.Web.PropertyEditors var items = Fields.First(x => x.Key == "items"); // customize the items field - items.Name = textService.Localize("editdatatype/addPrevalue"); + items.Name = textService.Localize("editdatatype", "addPrevalue"); items.Validators.Add(new ValueListUniqueValueValidator()); } diff --git a/src/Umbraco.Web/PropertyEditors/UploadFileTypeValidator.cs b/src/Umbraco.Web/PropertyEditors/UploadFileTypeValidator.cs index 8d7bd29889..f691feba52 100644 --- a/src/Umbraco.Web/PropertyEditors/UploadFileTypeValidator.cs +++ b/src/Umbraco.Web/PropertyEditors/UploadFileTypeValidator.cs @@ -40,7 +40,7 @@ namespace Umbraco.Web.PropertyEditors { //we only store a single value for this editor so the 'member' or 'field' // we'll associate this error with will simply be called 'value' - yield return new ValidationResult(Current.Services.TextService.Localize("errors/dissallowedMediaType"), new[] { "value" }); + yield return new ValidationResult(Current.Services.TextService.Localize("errors", "dissallowedMediaType"), new[] { "value" }); } } } diff --git a/src/Umbraco.Web/PropertyEditors/ValueListConfigurationEditor.cs b/src/Umbraco.Web/PropertyEditors/ValueListConfigurationEditor.cs index abfde3646a..b4f600dbf3 100644 --- a/src/Umbraco.Web/PropertyEditors/ValueListConfigurationEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/ValueListConfigurationEditor.cs @@ -19,7 +19,7 @@ namespace Umbraco.Web.PropertyEditors var items = Fields.First(x => x.Key == "items"); // customize the items field - items.Name = textService.Localize("editdatatype/addPrevalue"); + items.Name = textService.Localize("editdatatype", "addPrevalue"); items.Validators.Add(new ValueListUniqueValueValidator()); } diff --git a/src/Umbraco.Web/Routing/UrlProviderExtensions.cs b/src/Umbraco.Web/Routing/UrlProviderExtensions.cs index afa0bc96cc..7d5fba7df5 100644 --- a/src/Umbraco.Web/Routing/UrlProviderExtensions.cs +++ b/src/Umbraco.Web/Routing/UrlProviderExtensions.cs @@ -35,7 +35,7 @@ namespace Umbraco.Web.Routing if (content.Published == false) { - yield return UrlInfo.Message(textService.Localize("content/itemNotPublished")); + yield return UrlInfo.Message(textService.Localize("content", "itemNotPublished")); yield break; } @@ -126,7 +126,7 @@ namespace Umbraco.Web.Routing // deal with exceptions case "#ex": - yield return UrlInfo.Message(textService.Localize("content/getUrlException"), culture); + yield return UrlInfo.Message(textService.Localize("content", "getUrlException"), culture); break; // got a URL, deal with collisions, add URL @@ -152,13 +152,13 @@ namespace Umbraco.Web.Routing while (parent != null && parent.Published && (!parent.ContentType.VariesByCulture() || parent.IsCulturePublished(culture))); if (parent == null) // oops, internal error - return UrlInfo.Message(textService.Localize("content/parentNotPublishedAnomaly"), culture); + return UrlInfo.Message(textService.Localize("content", "parentNotPublishedAnomaly"), culture); else if (!parent.Published) // totally not published - return UrlInfo.Message(textService.Localize("content/parentNotPublished", new[] {parent.Name}), culture); + return UrlInfo.Message(textService.Localize("content", "parentNotPublished", new[] {parent.Name}), culture); else // culture not published - return UrlInfo.Message(textService.Localize("content/parentCultureNotPublished", new[] {parent.Name}), culture); + return UrlInfo.Message(textService.Localize("content", "parentCultureNotPublished", new[] {parent.Name}), culture); } private static bool DetectCollision(IContent content, string url, string culture, UmbracoContext umbracoContext, IPublishedRouter publishedRouter, ILocalizedTextService textService, out UrlInfo urlInfo) @@ -174,7 +174,7 @@ namespace Umbraco.Web.Routing if (pcr.HasPublishedContent == false) { - urlInfo = UrlInfo.Message(textService.Localize("content/routeErrorCannotRoute"), culture); + urlInfo = UrlInfo.Message(textService.Localize("content", "routeErrorCannotRoute"), culture); return true; } @@ -193,7 +193,7 @@ namespace Umbraco.Web.Routing l.Reverse(); var s = "/" + string.Join("/", l) + " (id=" + pcr.PublishedContent.Id + ")"; - urlInfo = UrlInfo.Message(textService.Localize("content/routeError", new[] { s }), culture); + urlInfo = UrlInfo.Message(textService.Localize("content", "routeError", new[] { s }), culture); return true; } diff --git a/src/Umbraco.Web/Trees/ApplicationTreeController.cs b/src/Umbraco.Web/Trees/ApplicationTreeController.cs index ad9e3ca172..82853bace3 100644 --- a/src/Umbraco.Web/Trees/ApplicationTreeController.cs +++ b/src/Umbraco.Web/Trees/ApplicationTreeController.cs @@ -70,7 +70,7 @@ namespace Umbraco.Web.Trees { //if there are no trees defined for this section but the section is defined then we can have a simple //full screen section without trees - var name = Services.TextService.Localize("sections/" + application); + var name = Services.TextService.Localize("sections", application); return TreeRootNode.CreateSingleTreeRoot(Constants.System.RootString, null, null, name, TreeNodeCollection.Empty, true); } @@ -103,7 +103,7 @@ namespace Umbraco.Web.Trees nodes.Add(node); } - var name = Services.TextService.Localize("sections/" + application); + var name = Services.TextService.Localize("sections", application); if (nodes.Count > 0) { @@ -138,7 +138,7 @@ namespace Umbraco.Web.Trees var name = groupName.IsNullOrWhiteSpace() ? "thirdPartyGroup" : groupName; var groupRootNode = TreeRootNode.CreateGroupNode(nodes, application); - groupRootNode.Name = Services.TextService.Localize("treeHeaders/" + name); + groupRootNode.Name = Services.TextService.Localize("treeHeaders", name); treeRootNodes.Add(groupRootNode); } diff --git a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs index 95de72b7bf..66e9fbf50c 100644 --- a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs +++ b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs @@ -331,7 +331,7 @@ namespace Umbraco.Web.Trees RecycleBinId.ToInvariantString(), id, queryStrings, - Services.TextService.Localize("general/recycleBin"), + Services.TextService.Localize("general", "recycleBin"), "icon-trash", RecycleBinSmells, queryStrings.GetRequiredValue("application") + TreeAlias.EnsureStartsWith('/') + "/recyclebin")); diff --git a/src/Umbraco.Web/Trees/DataTypeTreeController.cs b/src/Umbraco.Web/Trees/DataTypeTreeController.cs index 6a7fb7f5ad..c0727d6e78 100644 --- a/src/Umbraco.Web/Trees/DataTypeTreeController.cs +++ b/src/Umbraco.Web/Trees/DataTypeTreeController.cs @@ -135,7 +135,7 @@ namespace Umbraco.Web.Trees menu.Items.Add(Services.TextService, opensDialog: true); - menu.Items.Add(new MenuItem("rename", Services.TextService.Localize("actions/rename")) + menu.Items.Add(new MenuItem("rename", Services.TextService.Localize("actions", "rename")) { Icon = "icon icon-edit" }); diff --git a/src/Umbraco.Web/Trees/MediaTypeTreeController.cs b/src/Umbraco.Web/Trees/MediaTypeTreeController.cs index 7a9a80c8fc..d3989e54bb 100644 --- a/src/Umbraco.Web/Trees/MediaTypeTreeController.cs +++ b/src/Umbraco.Web/Trees/MediaTypeTreeController.cs @@ -99,7 +99,7 @@ namespace Umbraco.Web.Trees menu.Items.Add(Services.TextService, opensDialog: true); - menu.Items.Add(new MenuItem("rename", Services.TextService.Localize("actions/rename")) + menu.Items.Add(new MenuItem("rename", Services.TextService.Localize("actions", "rename")) { Icon = "icon icon-edit" }); diff --git a/src/Umbraco.Web/Trees/MemberTreeController.cs b/src/Umbraco.Web/Trees/MemberTreeController.cs index c0a9d15cfa..0bf1787402 100644 --- a/src/Umbraco.Web/Trees/MemberTreeController.cs +++ b/src/Umbraco.Web/Trees/MemberTreeController.cs @@ -124,7 +124,7 @@ namespace Umbraco.Web.Trees if (id == Constants.System.RootString) { nodes.Add( - CreateTreeNode(Constants.Conventions.MemberTypes.AllMembersListId, id, queryStrings, Services.TextService.Localize("member/allMembers"), Constants.Icons.MemberType, true, + CreateTreeNode(Constants.Conventions.MemberTypes.AllMembersListId, id, queryStrings, Services.TextService.Localize("member", "allMembers"), Constants.Icons.MemberType, true, queryStrings.GetRequiredValue("application") + TreeAlias.EnsureStartsWith('/') + "/list/" + Constants.Conventions.MemberTypes.AllMembersListId)); if (_isUmbracoProvider) diff --git a/src/Umbraco.Web/Trees/Tree.cs b/src/Umbraco.Web/Trees/Tree.cs index 4747d2495b..bc0511fe1e 100644 --- a/src/Umbraco.Web/Trees/Tree.cs +++ b/src/Umbraco.Web/Trees/Tree.cs @@ -50,7 +50,7 @@ namespace Umbraco.Web.Trees var label = $"[{tree.TreeAlias}]"; // try to look up a the localized tree header matching the tree alias - var localizedLabel = textService.Localize("treeHeaders/" + tree.TreeAlias); + var localizedLabel = textService.Localize("treeHeader", tree.TreeAlias); // if the localizedLabel returns [alias] then return the title if it's defined if (localizedLabel != null && localizedLabel.Equals(label, StringComparison.InvariantCultureIgnoreCase)) From 69e8ee087e22150ca2ffc56ed8a242bce91216b4 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sun, 17 Jan 2021 21:38:52 +1300 Subject: [PATCH 055/147] Remove debug code, set comparer StringComparer.InvariantCultureIgnoreCase --- .../Implement/LocalizedTextService.cs | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index c099960397..ad75e8a2d1 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -17,7 +18,6 @@ namespace Umbraco.Core.Services.Implement private readonly IDictionary>> _dictionarySource; private readonly IDictionary> _noAreaDictionarySource; private readonly char[] _splitter = new[] { '/' }; - /// /// Initializes with a file sources instance /// @@ -47,7 +47,7 @@ namespace Umbraco.Core.Services.Implement { var areaAliaValue = GetAreaStoredTranslations(source, xmlSource.Key); cultureDictionary.Add(xmlSource.Key, areaAliaValue); - var aliasValue = new Dictionary(); + var aliasValue = new Dictionary(StringComparer.InvariantCultureIgnoreCase); foreach (var area in areaAliaValue) { foreach (var alias in area.Value) @@ -118,19 +118,10 @@ namespace Umbraco.Core.Services.Implement public string Localize(string area, string alias, CultureInfo culture, IDictionary tokens = null) { if (culture == null) throw new ArgumentNullException(nameof(culture)); - var sw = System.Diagnostics.Stopwatch.StartNew(); - try - { // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode culture = ConvertToSupportedCultureWithRegionCode(culture); return GetFromDictionarySource(culture, area, alias, tokens); - } - finally - { - sw.Stop(); - System.Diagnostics.Debug.WriteLine($"Localize {area}/{alias} ({tokens?.Count}) ({sw.ElapsedTicks})"); - } } /// @@ -172,11 +163,11 @@ namespace Umbraco.Core.Services.Implement private Dictionary> GetAreaStoredTranslations(IDictionary> xmlSource, CultureInfo cult) { - var overallResult = new Dictionary>(); + var overallResult = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); var areas = xmlSource[cult].Value.XPathSelectElements("//area"); foreach (var area in areas) { - var result = new Dictionary(); + var result = new Dictionary(StringComparer.InvariantCultureIgnoreCase); var keys = area.XPathSelectElements("./key"); foreach (var key in keys) { @@ -192,12 +183,12 @@ namespace Umbraco.Core.Services.Implement } private Dictionary> GetAreaStoredTranslations(IDictionary>> dictionarySource, CultureInfo cult) { - var overallResult = new Dictionary>(); + var overallResult = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); var areaDict = dictionarySource[cult]; foreach (var area in areaDict) { - var result = new Dictionary(); + var result = new Dictionary(StringComparer.InvariantCultureIgnoreCase); var keys = area.Value.Keys; foreach (var key in keys) { From befa789d2361ba97733f49279dc3025493ae90fe Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sun, 17 Jan 2021 21:56:40 +1300 Subject: [PATCH 056/147] Additional benchmarks --- ...dTextServiceGetAllStoredValuesBenchmarks.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs b/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs index e084931d9e..be45ffbd8c 100644 --- a/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs +++ b/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs @@ -82,6 +82,24 @@ namespace Umbraco.Tests.Benchmarks var result = _xmlService.Localize("language", culture); } } + [Benchmark] + public void OptimizedXmlGetAll() + { + for (int i = 0; i < 10000; i++) + { + var result = _optimized.GetAllStoredValues(culture); + } + + } + [Benchmark] + public void OptimizedDictGetAll() + { + for (int i = 0; i < 10000; i++) + { + var result = _optimizedDict.GetAllStoredValues(culture); + } + } + [Benchmark()] public void OptimizedXmlLocalize() { From 2c2ffc671cd0112b618347690f50a53c33e73bb8 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sun, 17 Jan 2021 22:02:01 +1300 Subject: [PATCH 057/147] Add back in check for empty alias. --- .../Services/Implement/LocalizedTextService.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index ad75e8a2d1..ff53dd2ec5 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -110,6 +110,11 @@ namespace Umbraco.Core.Services.Implement public string Localize(string key, CultureInfo culture, IDictionary tokens = null) { if (culture == null) throw new ArgumentNullException(nameof(culture)); + + //This is what the legacy ui service did + if (string.IsNullOrEmpty(key)) + return string.Empty; + var keyParts = key.Split(_splitter, StringSplitOptions.RemoveEmptyEntries); var area = keyParts.Length > 1 ? keyParts[0] : null; var alias = keyParts.Length > 1 ? keyParts[1] : keyParts[0]; @@ -118,8 +123,13 @@ namespace Umbraco.Core.Services.Implement public string Localize(string area, string alias, CultureInfo culture, IDictionary tokens = null) { if (culture == null) throw new ArgumentNullException(nameof(culture)); - // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode - culture = ConvertToSupportedCultureWithRegionCode(culture); + + //This is what the legacy ui service did + if (string.IsNullOrEmpty(alias)) + return string.Empty; + + // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode + culture = ConvertToSupportedCultureWithRegionCode(culture); return GetFromDictionarySource(culture, area, alias, tokens); } From 264259164e634de3cfe9b4bb26e80a4e150bc5da Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sun, 17 Jan 2021 22:07:35 +1300 Subject: [PATCH 058/147] corrected comparer. --- .../Services/Implement/LocalizedTextService.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index ff53dd2ec5..246481a48b 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -47,7 +47,7 @@ namespace Umbraco.Core.Services.Implement { var areaAliaValue = GetAreaStoredTranslations(source, xmlSource.Key); cultureDictionary.Add(xmlSource.Key, areaAliaValue); - var aliasValue = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + var aliasValue = new Dictionary(StringComparer.InvariantCulture); foreach (var area in areaAliaValue) { foreach (var alias in area.Value) @@ -173,11 +173,11 @@ namespace Umbraco.Core.Services.Implement private Dictionary> GetAreaStoredTranslations(IDictionary> xmlSource, CultureInfo cult) { - var overallResult = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); + var overallResult = new Dictionary>(StringComparer.InvariantCulture); var areas = xmlSource[cult].Value.XPathSelectElements("//area"); foreach (var area in areas) { - var result = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + var result = new Dictionary(StringComparer.InvariantCulture); var keys = area.XPathSelectElements("./key"); foreach (var key in keys) { @@ -193,12 +193,12 @@ namespace Umbraco.Core.Services.Implement } private Dictionary> GetAreaStoredTranslations(IDictionary>> dictionarySource, CultureInfo cult) { - var overallResult = new Dictionary>(StringComparer.InvariantCultureIgnoreCase); + var overallResult = new Dictionary>(StringComparer.InvariantCulture); var areaDict = dictionarySource[cult]; foreach (var area in areaDict) { - var result = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + var result = new Dictionary(StringComparer.InvariantCulture); var keys = area.Value.Keys; foreach (var key in keys) { From 4a400b125b0b0a8b694f9ff91fc23c8e6cccad8f Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 22 Jan 2021 00:14:37 +1100 Subject: [PATCH 059/147] Fixes merge issues and updates npoco to latest 4.x --- src/Umbraco.Core/Umbraco.Core.csproj | 2 +- src/Umbraco.Examine/Umbraco.Examine.csproj | 2 +- .../Umbraco.Tests.Benchmarks.csproj | 2 +- src/Umbraco.Tests/Umbraco.Tests.csproj | 6 ++---- src/Umbraco.Web/Umbraco.Web.csproj | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index f890faecb3..eaf13eb955 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -91,7 +91,7 @@ - + 2.8.0 diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index 55ed24b3e8..91a2fdda99 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -56,7 +56,7 @@ all - + 3.3.0 runtime; build; native; contentfiles; analyzers diff --git a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj index 48d69cf757..df2af67573 100644 --- a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj +++ b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj @@ -93,4 +93,4 @@ - \ No newline at end of file + diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index c7b9954ca4..a475a6a355 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -103,9 +103,7 @@ 4.14.5 - - - + @@ -648,4 +646,4 @@ - \ No newline at end of file + diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 86c21b7182..35218e5787 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -97,7 +97,7 @@ - + 3.3.0 runtime; build; native; contentfiles; analyzers From eec81a3890bcc974e9a92bb5127631bf216ac44b Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 22 Jan 2021 00:15:07 +1100 Subject: [PATCH 060/147] updates sql usage in DatabaseDataSource --- src/Umbraco.Core/Constants-SqlTemplates.cs | 8 +- .../NuCache/DataSource/DatabaseDataSource.cs | 241 ++++++++++-------- 2 files changed, 145 insertions(+), 104 deletions(-) diff --git a/src/Umbraco.Core/Constants-SqlTemplates.cs b/src/Umbraco.Core/Constants-SqlTemplates.cs index 3def094702..940524cd1a 100644 --- a/src/Umbraco.Core/Constants-SqlTemplates.cs +++ b/src/Umbraco.Core/Constants-SqlTemplates.cs @@ -27,8 +27,14 @@ internal static class NuCacheDatabaseDataSource { - public const string ContentSourcesSelect1 = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelect1"; + public const string ContentSourcesSelect = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelect"; + public const string ContentSourcesSelectUmbracoNodeJoin = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelectUmbracoNodeJoin"; public const string ContentSourcesCount = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesCount"; + public const string MediaSourcesSelect = "Umbraco.Web.PublishedCache.NuCache.DataSource.MediaSourcesSelect"; + public const string MediaSourcesCount = "Umbraco.Web.PublishedCache.NuCache.DataSource.MediaSourcesCount"; + public const string ObjectTypeNotTrashedFilter = "Umbraco.Web.PublishedCache.NuCache.DataSource.ObjectTypeNotTrashedFilter"; + public const string OrderByLevelIdSortOrder = "Umbraco.Web.PublishedCache.NuCache.DataSource.OrderByLevelIdSortOrder"; + } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 98dbf1f85c..a49a97568b 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -28,9 +28,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // we want arrays, we want them all loaded, not an enumerable - private Sql ContentSourcesSelect(IScope scope, Func, Sql> joins = null) + private Sql SqlContentSourcesSelect(IScope scope, Func, Sql> joins = null) { - var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelect1, tsql => + var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelect, tsql => tsql.Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) @@ -75,12 +75,48 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } + private Sql SqlContentSourcesSelectUmbracoNodeJoin(Sql s) + { + var syntax = s.SqlContext.SqlSyntax; + var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelectUmbracoNodeJoin, s => + s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")); + var sql = sqlTemplate.Sql(); + return sql; + } + + private Sql SqlWhereNodeId(Sql s, int id) + { + var syntax = s.SqlContext.SqlSyntax; + var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelectUmbracoNodeJoin, s => + s.Where(x => x.NodeId == SqlTemplate.Arg("id"), "x")); + var sql = sqlTemplate.Sql(id); + return sql; + } + + private Sql SqlOrderByLevelIdSortOrder(Sql s) + { + var syntax = s.SqlContext.SqlSyntax; + var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.OrderByLevelIdSortOrder, s => + s.OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder)); + var sql = sqlTemplate.Sql(); + return sql; + } + + private Sql SqlObjectTypeNotTrashed(Sql s, Guid nodeObjectType) + { + var syntax = s.SqlContext.SqlSyntax; + var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ObjectTypeNotTrashedFilter, s => + s.Where(x => x.NodeObjectType == SqlTemplate.Arg("nodeObjectType") && !x.Trashed)); + var sql = sqlTemplate.Sql(nodeObjectType); + return sql; + } + /// /// Returns a slightly more optimized query to use for the document counting when paging over the content sources /// /// /// - private Sql ContentSourcesCount(IScope scope, Func, Sql> joins = null) + private Sql SqlContentSourcesCount(IScope scope, Func, Sql> joins = null) { var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesCount, tsql => tsql.Select(x => Alias(x.NodeId, "Id")) @@ -104,11 +140,55 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } + private Sql SqlMediaSourcesSelect(IScope scope, Func, Sql> joins = null) + { + var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.MediaSourcesSelect, tsql => + tsql.Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), + x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), + x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) + .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) + .AndSelect(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId")) + .AndSelect("nuEdit", x => Alias(x.Data, "EditData")) + .AndSelect("nuEdit", x => Alias(x.RawData, "EditDataRaw")) + .From()); + + var sql = sqlTemplate.Sql(); + + if (joins != null) + sql = joins(sql); + + // TODO: We can't use a template with this one because of the 'right.Published' ends up being a parameter so not sure how we can do that + sql = sql + .InnerJoin().On((left, right) => left.NodeId == right.NodeId) + .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current) + .LeftJoin("nuEdit").On((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit"); + + return sql; + } + private Sql SqlMediaSourcesCount(IScope scope, Func, Sql> joins = null) + { + var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.MediaSourcesCount, tsql => + tsql.Select(x => Alias(x.NodeId, "Id")).From()); + + var sql = sqlTemplate.Sql(); + + if (joins != null) + sql = joins(sql); + + // TODO: We can't use a template with this one because of the 'right.Current' ends up being a parameter so not sure how we can do that + sql = sql + .InnerJoin().On((left, right) => left.NodeId == right.NodeId) + .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current); + + return sql; + } + public ContentNodeKit GetContentSource(IScope scope, int id) { - var sql = ContentSourcesSelect(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && x.NodeId == id && !x.Trashed) - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sql = SqlContentSourcesSelect(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) + .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); var dto = scope.Database.Fetch(sql).FirstOrDefault(); @@ -120,14 +200,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetAllContentSources(IScope scope) { - // Create a different query for the SQL vs the COUNT Sql since the auto-generated COUNT Sql will be inneficient - var sql = ContentSourcesSelect(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sql = SqlContentSourcesSelect(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + + // Use a more efficient COUNT query + var sqlCountQuery = SqlContentSourcesCount(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)); - // create a more efficient COUNT query without the join on the cmsContentNu table - var sqlCountQuery = ContentSourcesCount(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); @@ -138,26 +218,22 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateContentNodeKit(row, serializer); - } + } } public IEnumerable GetBranchContentSources(IScope scope, int id) { - var syntax = scope.SqlContext.SqlSyntax; - var sql = ContentSourcesSelect(scope, - s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) - .Where(x => x.NodeId == id, "x") - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sql = SqlContentSourcesSelect(scope, SqlContentSourcesSelectUmbracoNodeJoin) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) + .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); - // create a more efficient COUNT query without the join on the cmsContentNu table - var sqlCountQuery = ContentSourcesCount(scope, - s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) - .Where(x => x.NodeId == id, "x"); + // Use a more efficient COUNT query + var sqlCountQuery = SqlContentSourcesCount(scope, SqlContentSourcesSelectUmbracoNodeJoin) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) + .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. @@ -166,20 +242,21 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateContentNodeKit(row, serializer); - } + } } public IEnumerable GetTypeContentSources(IScope scope, IEnumerable ids) { if (!ids.Any()) yield break; - var sql = ContentSourcesSelect(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) + var sql = SqlContentSourcesSelect(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) .WhereIn(x => x.ContentTypeId, ids) - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); - var sqlCountQuery = ContentSourcesCount(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) + // Use a more efficient COUNT query + var sqlCountQuery = SqlContentSourcesCount(scope, SqlContentSourcesSelectUmbracoNodeJoin) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) .WhereIn(x => x.ContentTypeId, ids); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); @@ -191,54 +268,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateContentNodeKit(row, serializer); - } - } - - private Sql MediaSourcesSelect(IScope scope, Func, Sql> joins = null) - { - var sql = scope.SqlContext.Sql() - - .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), - x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), - x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) - .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) - .AndSelect(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId")) - .AndSelect("nuEdit", x => Alias(x.Data, "EditData")) - .AndSelect("nuEdit", x => Alias(x.RawData, "EditDataRaw")) - .From(); - - if (joins != null) - sql = joins(sql); - - sql = sql - .InnerJoin().On((left, right) => left.NodeId == right.NodeId) - .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current) - .LeftJoin("nuEdit").On((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit"); - - return sql; - } - private Sql MediaSourcesCount(IScope scope, Func, Sql> joins = null) - { - var sql = scope.SqlContext.Sql() - - .Select(x => Alias(x.NodeId, "Id")) - .From(); - - if (joins != null) - sql = joins(sql); - - sql = sql - .InnerJoin().On((left, right) => left.NodeId == right.NodeId) - .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current); - - return sql; + } } public ContentNodeKit GetMediaSource(IScope scope, int id) { - var sql = MediaSourcesSelect(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && x.NodeId == id && !x.Trashed) - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var sql = SqlMediaSourcesSelect(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) + .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); var dto = scope.Database.Fetch(sql).FirstOrDefault(); @@ -250,13 +288,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetAllMediaSources(IScope scope) { - var sql = MediaSourcesSelect(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); - - var sqlCountQuery = MediaSourcesCount(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed); + var sql = SqlMediaSourcesSelect(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + // Use a more efficient COUNT query + var sqlCountQuery = SqlMediaSourcesCount(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); @@ -267,23 +305,20 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateMediaNodeKit(row, serializer); - } + } } public IEnumerable GetBranchMediaSources(IScope scope, int id) { - var syntax = scope.SqlContext.SqlSyntax; - var sql = MediaSourcesSelect(scope, - s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) - .Where(x => x.NodeId == id, "x") - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); - - var sqlCountQuery = MediaSourcesCount(scope, - s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) - .Where(x => x.NodeId == id, "x"); + var sql = SqlMediaSourcesSelect(scope, SqlContentSourcesSelectUmbracoNodeJoin) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) + .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + // Use a more efficient COUNT query + var sqlCountQuery = SqlMediaSourcesCount(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) + .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); @@ -294,22 +329,22 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateMediaNodeKit(row, serializer); - } + } } public IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids) { if (!ids.Any()) yield break; - var sql = MediaSourcesSelect(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) + var sql = SqlMediaSourcesSelect(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) .WhereIn(x => x.ContentTypeId, ids) - .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); - - var sqlCountQuery = MediaSourcesCount(scope) - .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) - .WhereIn(x => x.ContentTypeId, ids); + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + // Use a more efficient COUNT query + var sqlCountQuery = SqlMediaSourcesCount(scope) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) + .WhereIn(x => x.ContentTypeId, ids); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); @@ -320,7 +355,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { yield return CreateMediaNodeKit(row, serializer); - } + } } private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer) From e01f8319f6b78feb0a64dd9d96a3c1f232f3bb33 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 22 Jan 2021 01:12:26 +1100 Subject: [PATCH 061/147] Updates the new nodedto index to include 2 cols as the key and update the includes, this is required for more optimal querying for nucache and others. --- src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs | 4 ++-- .../{V_8_11_0 => V_8_12_0}/AddCmsContentNuByteColumn.cs | 2 +- .../Upgrade/{V_8_11_0 => V_8_12_0}/UpgradedIncludeIndexes.cs | 2 +- src/Umbraco.Core/Persistence/Dtos/NodeDto.cs | 2 +- src/Umbraco.Core/Umbraco.Core.csproj | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) rename src/Umbraco.Core/Migrations/Upgrade/{V_8_11_0 => V_8_12_0}/AddCmsContentNuByteColumn.cs (90%) rename src/Umbraco.Core/Migrations/Upgrade/{V_8_11_0 => V_8_12_0}/UpgradedIncludeIndexes.cs (98%) diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index a60b046212..2a24c800b5 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -9,7 +9,7 @@ using Umbraco.Core.Migrations.Upgrade.V_8_1_0; using Umbraco.Core.Migrations.Upgrade.V_8_6_0; using Umbraco.Core.Migrations.Upgrade.V_8_9_0; using Umbraco.Core.Migrations.Upgrade.V_8_10_0; -using Umbraco.Core.Migrations.Upgrade.V_8_11_0; +using Umbraco.Core.Migrations.Upgrade.V_8_12_0; namespace Umbraco.Core.Migrations.Upgrade { @@ -203,7 +203,7 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.10.0 To("{D6A8D863-38EC-44FB-91EC-ACD6A668BD18}"); - // to 8.11.0... + // to 8.12.0... To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); To("{4695D0C9-0729-4976-985B-048D503665D8}"); diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/AddCmsContentNuByteColumn.cs similarity index 90% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/AddCmsContentNuByteColumn.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/AddCmsContentNuByteColumn.cs index dfde1f0577..7c793688ec 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/AddCmsContentNuByteColumn.cs @@ -1,7 +1,7 @@ using System.Linq; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_11_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_12_0 { public class AddCmsContentNuByteColumn : MigrationBase { diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/UpgradedIncludeIndexes.cs similarity index 98% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/UpgradedIncludeIndexes.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/UpgradedIncludeIndexes.cs index 6919558fc3..d88abdef75 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_11_0/UpgradedIncludeIndexes.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/UpgradedIncludeIndexes.cs @@ -3,7 +3,7 @@ using Umbraco.Core.Migrations.Expressions.Execute.Expressions; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_11_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_12_0 { public class UpgradedIncludeIndexes : MigrationBase { diff --git a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs index 62475af833..207195e594 100644 --- a/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/NodeDto.cs @@ -58,7 +58,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("nodeObjectType")] // TODO: db rename to 'objectType' [NullSetting(NullSetting = NullSettings.Null)] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType", IncludeColumns = "parentId,level,path,sortOrder,trashed,nodeUser,text,createDate")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_ObjectType", ForColumns = "nodeObjectType,trashed", IncludeColumns = "uniqueId,parentId,level,path,sortOrder,nodeUser,text,createDate")] public Guid? NodeObjectType { get; set; } [Column("createDate")] diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index eaf13eb955..6c473235a7 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -133,8 +133,8 @@ - - + + From c5bd53770a8289a8d0c54953572d64d701012dbe Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 22 Jan 2021 01:13:03 +1100 Subject: [PATCH 062/147] fixes sql template --- .../PublishedCache/NuCache/DataSource/DatabaseDataSource.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index a49a97568b..88e364634e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -106,8 +106,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var syntax = s.SqlContext.SqlSyntax; var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ObjectTypeNotTrashedFilter, s => - s.Where(x => x.NodeObjectType == SqlTemplate.Arg("nodeObjectType") && !x.Trashed)); - var sql = sqlTemplate.Sql(nodeObjectType); + s.Where(x => x.NodeObjectType == SqlTemplate.Arg("nodeObjectType") && x.Trashed == SqlTemplate.Arg("trashed"))); + var sql = sqlTemplate.Sql(nodeObjectType, false); return sql; } From 808e58b60f97e7b2a7015656bfffd293bed22a6e Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 22 Jan 2021 16:14:25 +1100 Subject: [PATCH 063/147] fixes up inconsistencies --- src/Umbraco.Core/Constants-SqlTemplates.cs | 6 ++-- .../NuCache/DataSource/DatabaseDataSource.cs | 28 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/Umbraco.Core/Constants-SqlTemplates.cs b/src/Umbraco.Core/Constants-SqlTemplates.cs index 940524cd1a..8529e6bfbc 100644 --- a/src/Umbraco.Core/Constants-SqlTemplates.cs +++ b/src/Umbraco.Core/Constants-SqlTemplates.cs @@ -27,8 +27,10 @@ internal static class NuCacheDatabaseDataSource { - public const string ContentSourcesSelect = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelect"; - public const string ContentSourcesSelectUmbracoNodeJoin = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelectUmbracoNodeJoin"; + public const string WhereNodeId = "Umbraco.Web.PublishedCache.NuCache.DataSource.WhereNodeId"; + public const string WhereNodeIdX = "Umbraco.Web.PublishedCache.NuCache.DataSource.WhereNodeIdX"; + public const string SourcesSelectUmbracoNodeJoin = "Umbraco.Web.PublishedCache.NuCache.DataSource.SourcesSelectUmbracoNodeJoin"; + public const string ContentSourcesSelect = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelect"; public const string ContentSourcesCount = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesCount"; public const string MediaSourcesSelect = "Umbraco.Web.PublishedCache.NuCache.DataSource.MediaSourcesSelect"; public const string MediaSourcesCount = "Umbraco.Web.PublishedCache.NuCache.DataSource.MediaSourcesCount"; diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 88e364634e..3bd30d2d73 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -78,8 +78,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private Sql SqlContentSourcesSelectUmbracoNodeJoin(Sql s) { var syntax = s.SqlContext.SqlSyntax; - var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelectUmbracoNodeJoin, s => - s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")); + var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.SourcesSelectUmbracoNodeJoin, s => + s.InnerJoin("x") + .On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")); var sql = sqlTemplate.Sql(); return sql; } @@ -87,7 +88,16 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private Sql SqlWhereNodeId(Sql s, int id) { var syntax = s.SqlContext.SqlSyntax; - var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelectUmbracoNodeJoin, s => + var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.WhereNodeId, s => + s.Where(x => x.NodeId == SqlTemplate.Arg("id"))); + var sql = sqlTemplate.Sql(id); + return sql; + } + + private Sql SqlWhereNodeIdX(Sql s, int id) + { + var syntax = s.SqlContext.SqlSyntax; + var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.WhereNodeIdX, s => s.Where(x => x.NodeId == SqlTemplate.Arg("id"), "x")); var sql = sqlTemplate.Sql(id); return sql; @@ -225,13 +235,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = SqlContentSourcesSelect(scope, SqlContentSourcesSelectUmbracoNodeJoin) .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) - .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) + .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)) .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); // Use a more efficient COUNT query var sqlCountQuery = SqlContentSourcesCount(scope, SqlContentSourcesSelectUmbracoNodeJoin) .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) - .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)); + .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); @@ -255,7 +265,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); // Use a more efficient COUNT query - var sqlCountQuery = SqlContentSourcesCount(scope, SqlContentSourcesSelectUmbracoNodeJoin) + var sqlCountQuery = SqlContentSourcesCount(scope) .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) .WhereIn(x => x.ContentTypeId, ids); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); @@ -312,13 +322,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = SqlMediaSourcesSelect(scope, SqlContentSourcesSelectUmbracoNodeJoin) .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) - .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) + .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)) .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); // Use a more efficient COUNT query - var sqlCountQuery = SqlMediaSourcesCount(scope) + var sqlCountQuery = SqlMediaSourcesCount(scope, SqlContentSourcesSelectUmbracoNodeJoin) .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) - .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)); + .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); From 0cb4952338e0dc1ad82cda3b0a56145626c201c6 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 25 Jan 2021 15:56:01 +1100 Subject: [PATCH 064/147] Don't keep re-creating serializer settings on deserialize --- .../JsonContentNestedDataSerializer.cs | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index 47f07b8b1d..e7bb5656bb 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -8,26 +8,25 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public class JsonContentNestedDataSerializer : IContentCacheDataSerializer { + // by default JsonConvert will deserialize our numeric values as Int64 + // which is bad, because they were Int32 in the database - take care + private readonly JsonSerializerSettings _jsonSerializerSettings = new JsonSerializerSettings + { + Converters = new List { new ForceInt32Converter() }, + + // Explicitly specify date handling so that it's consistent and follows the same date handling as MessagePack + DateParseHandling = DateParseHandling.DateTime, + DateFormatHandling = DateFormatHandling.IsoDateFormat, + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + DateFormatString = "o" + }; + public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); - // by default JsonConvert will deserialize our numeric values as Int64 - // which is bad, because they were Int32 in the database - take care - - var settings = new JsonSerializerSettings - { - Converters = new List { new ForceInt32Converter() }, - - // Explicitly specify date handling so that it's consistent and follows the same date handling as MessagePack - DateParseHandling = DateParseHandling.DateTime, - DateFormatHandling = DateFormatHandling.IsoDateFormat, - DateTimeZoneHandling = DateTimeZoneHandling.Utc, - DateFormatString = "o" - }; - - return JsonConvert.DeserializeObject(stringData, settings); + return JsonConvert.DeserializeObject(stringData, _jsonSerializerSettings); } public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) From 9200c46cd0b33aed42c4e66ee9e38ce6afc4d145 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 27 Jan 2021 15:11:16 +1100 Subject: [PATCH 065/147] Ensure LazyCompressedString doesn't decompress when debugging --- .../DataSource/LazyCompressedString.cs | 68 +++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs index 3e0e796d36..2be2568f7e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/LazyCompressedString.cs @@ -1,5 +1,6 @@ using K4os.Compression.LZ4; using System; +using System.Diagnostics; using System.Text; using Umbraco.Core.Exceptions; @@ -8,6 +9,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Lazily decompresses a LZ4 Pickler compressed UTF8 string /// + [DebuggerDisplay("{Display}")] internal struct LazyCompressedString { private byte[] _bytes; @@ -22,29 +24,85 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { _locker = new object(); _bytes = bytes; - _str = null; + _str = null; } public byte[] GetBytes() { if (_bytes == null) + { throw new InvalidOperationException("The bytes have already been expanded"); + } + return _bytes; } - public override string ToString() + /// + /// Returns the decompressed string from the bytes. This methods can only be called once. + /// + /// + /// Throws if this is called more than once + public string DecompressString() { - if (_str != null) return _str; + if (_str != null) + { + return _str; + } + lock (_locker) { - if (_str != null) return _str; // double check - if (_bytes == null) throw new PanicException("Bytes have already been cleared"); + if (_str != null) + { + // double check + return _str; + } + + if (_bytes == null) + { + throw new InvalidOperationException("Bytes have already been cleared"); + } + _str = Encoding.UTF8.GetString(LZ4Pickler.Unpickle(_bytes)); _bytes = null; } return _str; } + /// + /// Used to display debugging output since ToString() can only be called once + /// + private string Display + { + get + { + if (_str != null) + { + return $"Decompressed: {_str}"; + } + + lock (_locker) + { + if (_str != null) + { + // double check + return $"Decompressed: {_str}"; + } + + if (_bytes == null) + { + // This shouldn't happen + throw new PanicException("Bytes have already been cleared"); + } + else + { + return $"Compressed Bytes: {_bytes.Length}"; + } + } + } + } + + public override string ToString() => DecompressString(); + public static implicit operator string(LazyCompressedString l) => l.ToString(); } From 57708960f0429d0e116fc06754ad45f7e370ec8d Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 27 Jan 2021 15:16:22 +1100 Subject: [PATCH 066/147] reverts change for IsDecompressed --- .../NuCache/DataSource/SerializerBase.cs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs index 6fdb4dd188..7a0f6962b6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/SerializerBase.cs @@ -185,16 +185,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else if (value is LazyCompressedString lazyCompressedString) { - if (lazyCompressedString.IsDecompressed) - { - PrimitiveSerializer.Char.WriteTo(PrefixString, stream); - PrimitiveSerializer.String.WriteTo(lazyCompressedString, stream); - } - else - { - PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream); - PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); - } + PrimitiveSerializer.Char.WriteTo(PrefixCompressedStringByteArray, stream); + PrimitiveSerializer.Bytes.WriteTo(lazyCompressedString.GetBytes(), stream); } else if (value is sbyte signedByteValue) { From 4bfba2eebab1e7f091c5fb891d006191e2337bbe Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 27 Jan 2021 15:31:40 +1100 Subject: [PATCH 067/147] adds some notes --- .../DataSource/MsgPackContentNestedDataSerializer.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 42468ad930..6ffa130da0 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -79,6 +79,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during serialization to compress properties /// /// + /// + /// This will essentially 'double compress' property data. The MsgPack data as a whole will already be compressed + /// but this will go a step further and double compress property data so that it is stored in the nucache file + /// as compressed bytes and therefore will exist in memory as compressed bytes. That is, until the bytes are + /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant + /// memory savings but could also affect performance of first rendering pages while decompression occurs. + /// private void Compress(int contentTypeId, ContentCacheDataModel model) { foreach(var propertyAliasToData in model.PropertyData) From 32f88dba95f31e6cc87ebaabe0404578490d4976 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 28 Jan 2021 13:50:18 +1100 Subject: [PATCH 068/147] removes CompressedStorageAttribute, makes IPropertyCacheCompressionOptions which can be defined in DI to configure what properties can be compressed in memory and new IReadOnlyContentBase to provide some context to the IPropertyCacheCompressionOptions --- src/Umbraco.Core/Models/IContentBase.cs | 1 + .../Models/IReadOnlyContentBase.cs | 72 +++++++++++++++++++ .../Models/ReadOnlyContentBaseAdapter.cs | 42 +++++++++++ .../CompressedStorageAttribute.cs | 21 ------ ...StoragePropertyEditorCompressionOptions.cs | 47 ------------ ...ptions.cs => IPropertyCacheCompression.cs} | 9 ++- .../IPropertyCacheCompressionOptions.cs | 9 +++ .../NoopPropertyCacheCompressionOptions.cs | 12 ++++ .../PropertyCacheCompression.cs | 49 +++++++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 9 ++- .../ContentSerializationTests.cs | 17 +++-- .../BlockListPropertyEditor.cs | 1 - .../PropertyEditors/GridPropertyEditor.cs | 1 - .../PropertyEditors/MarkdownPropertyEditor.cs | 1 - .../NestedContentPropertyEditor.cs | 1 - .../PropertyEditors/RichTextPropertyEditor.cs | 1 - .../PropertyEditors/TextAreaPropertyEditor.cs | 1 - .../NuCache/DataSource/ContentSourceDto.cs | 10 ++- .../NuCache/DataSource/DatabaseDataSource.cs | 10 +-- .../DataSource/IContentCacheDataSerializer.cs | 13 +--- .../JsonContentNestedDataSerializer.cs | 5 +- .../MsgPackContentNestedDataSerializer.cs | 31 ++++---- ...gPackContentNestedDataSerializerFactory.cs | 15 ++-- .../PublishedCache/NuCache/NuCacheComposer.cs | 6 +- .../NuCache/PublishedSnapshotService.cs | 2 +- 25 files changed, 258 insertions(+), 128 deletions(-) create mode 100644 src/Umbraco.Core/Models/IReadOnlyContentBase.cs create mode 100644 src/Umbraco.Core/Models/ReadOnlyContentBaseAdapter.cs delete mode 100644 src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs delete mode 100644 src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs rename src/Umbraco.Core/PropertyEditors/{IPropertyCompressionOptions.cs => IPropertyCacheCompression.cs} (50%) create mode 100644 src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs create mode 100644 src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs create mode 100644 src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs diff --git a/src/Umbraco.Core/Models/IContentBase.cs b/src/Umbraco.Core/Models/IContentBase.cs index 0f660181fb..f4fae44ffc 100644 --- a/src/Umbraco.Core/Models/IContentBase.cs +++ b/src/Umbraco.Core/Models/IContentBase.cs @@ -4,6 +4,7 @@ using Umbraco.Core.Models.Entities; namespace Umbraco.Core.Models { + /// /// Provides a base class for content items. /// diff --git a/src/Umbraco.Core/Models/IReadOnlyContentBase.cs b/src/Umbraco.Core/Models/IReadOnlyContentBase.cs new file mode 100644 index 0000000000..3ed39b63ab --- /dev/null +++ b/src/Umbraco.Core/Models/IReadOnlyContentBase.cs @@ -0,0 +1,72 @@ +using System; + +namespace Umbraco.Core.Models +{ + public interface IReadOnlyContentBase + { + /// + /// Gets the integer identifier of the entity. + /// + int Id { get; } + + /// + /// Gets the Guid unique identifier of the entity. + /// + Guid Key { get; } + + /// + /// Gets the creation date. + /// + DateTime CreateDate { get; } + + /// + /// Gets the last update date. + /// + DateTime UpdateDate { get; } + + /// + /// Gets the name of the entity. + /// + string Name { get; } + + /// + /// Gets the identifier of the user who created this entity. + /// + int CreatorId { get; } + + /// + /// Gets the identifier of the parent entity. + /// + int ParentId { get; } + + /// + /// Gets the level of the entity. + /// + int Level { get; } + + /// + /// Gets the path to the entity. + /// + string Path { get; } + + /// + /// Gets the sort order of the entity. + /// + int SortOrder { get; } + + /// + /// Gets the content type id + /// + int ContentTypeId { get; } + + /// + /// Gets the identifier of the writer. + /// + int WriterId { get; } + + /// + /// Gets the version identifier. + /// + int VersionId { get; } + } +} diff --git a/src/Umbraco.Core/Models/ReadOnlyContentBaseAdapter.cs b/src/Umbraco.Core/Models/ReadOnlyContentBaseAdapter.cs new file mode 100644 index 0000000000..f707d2ab1c --- /dev/null +++ b/src/Umbraco.Core/Models/ReadOnlyContentBaseAdapter.cs @@ -0,0 +1,42 @@ +using System; + +namespace Umbraco.Core.Models +{ + internal struct ReadOnlyContentBaseAdapter : IReadOnlyContentBase + { + private readonly IContentBase _content; + + private ReadOnlyContentBaseAdapter(IContentBase content) + { + _content = content ?? throw new ArgumentNullException(nameof(content)); + } + + public static ReadOnlyContentBaseAdapter Create(IContentBase content) => new ReadOnlyContentBaseAdapter(content); + + public int Id => _content.Id; + + public Guid Key => _content.Key; + + public DateTime CreateDate => _content.CreateDate; + + public DateTime UpdateDate => _content.UpdateDate; + + public string Name => _content.Name; + + public int CreatorId => _content.CreatorId; + + public int ParentId => _content.ParentId; + + public int Level => _content.Level; + + public string Path => _content.Path; + + public int SortOrder => _content.SortOrder; + + public int ContentTypeId => _content.ContentTypeId; + + public int WriterId => _content.WriterId; + + public int VersionId => _content.VersionId; + } +} diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs b/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs deleted file mode 100644 index 31689c4ee9..0000000000 --- a/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; - -namespace Umbraco.Core.PropertyEditors -{ - /// - /// When assigned to a DataEditor it indicates that the values it generates can be compressed - /// - /// - /// Used in conjunction with - /// - [AttributeUsage(AttributeTargets.Class)] - public sealed class CompressedStorageAttribute : Attribute - { - public CompressedStorageAttribute(bool isCompressed = true) - { - IsCompressed = isCompressed; - } - - public bool IsCompressed { get; } - } -} diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs deleted file mode 100644 index a99452a5b1..0000000000 --- a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using Umbraco.Core.Models; - -namespace Umbraco.Core.PropertyEditors -{ - - /// - /// Ensures all property types that have a property editor attributed with use data compression - /// - internal class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions - { - private readonly IReadOnlyDictionary _contentTypes; - private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache; - - public CompressedStoragePropertyEditorCompressionOptions( - IReadOnlyDictionary contentTypes, - PropertyEditorCollection propertyEditors, - ConcurrentDictionary<(int, string), CompressedStorageAttribute> compressedStoragePropertyEditorCache) - { - _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); - _propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors)); - _compressedStoragePropertyEditorCache = compressedStoragePropertyEditorCache; - } - - public bool IsCompressed(int contentTypeId, string alias) - { - var compressedStorage = _compressedStoragePropertyEditorCache.GetOrAdd((contentTypeId, alias), x => - { - if (!_contentTypes.TryGetValue(contentTypeId, out var ct)) - return null; - - var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); - if (propertyType == null) return null; - - if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; - - var attribute = propertyEditor.GetType().GetCustomAttribute(true); - return attribute; - }); - - return compressedStorage?.IsCompressed ?? false; - } - } -} diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs similarity index 50% rename from src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs rename to src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs index d1add38f19..96a559630b 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompression.cs @@ -3,10 +3,13 @@ namespace Umbraco.Core.PropertyEditors { /// - /// Determines if a property type's value should be compressed + /// Determines if a property type's value should be compressed in memory /// - public interface IPropertyCompressionOptions + /// + /// + /// + public interface IPropertyCacheCompression { - bool IsCompressed(int contentTypeId, string propertyTypeAlias); + bool IsCompressed(IReadOnlyContentBase content, string propertyTypeAlias); } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs new file mode 100644 index 0000000000..2fa0153f9e --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCacheCompressionOptions.cs @@ -0,0 +1,9 @@ +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + public interface IPropertyCacheCompressionOptions + { + bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor); + } +} diff --git a/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs new file mode 100644 index 0000000000..1f12d45769 --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/NoopPropertyCacheCompressionOptions.cs @@ -0,0 +1,12 @@ +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// Default implementation for which does not compress any property data + /// + internal class NoopPropertyCacheCompressionOptions : IPropertyCacheCompressionOptions + { + public bool IsCompressed(IReadOnlyContentBase content, PropertyType propertyType, IDataEditor dataEditor) => false; + } +} diff --git a/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs new file mode 100644 index 0000000000..6be21fca7f --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/PropertyCacheCompression.cs @@ -0,0 +1,49 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors +{ + + /// + /// Compresses property data based on config + /// + internal class PropertyCacheCompression : IPropertyCacheCompression + { + private readonly IPropertyCacheCompressionOptions _compressionOptions; + private readonly IReadOnlyDictionary _contentTypes; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int contentTypeId, string propertyAlias), bool> _isCompressedCache; + + public PropertyCacheCompression( + IPropertyCacheCompressionOptions compressionOptions, + IReadOnlyDictionary contentTypes, + PropertyEditorCollection propertyEditors, + ConcurrentDictionary<(int, string), bool> compressedStoragePropertyEditorCache) + { + _compressionOptions = compressionOptions; + _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); + _propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors)); + _isCompressedCache = compressedStoragePropertyEditorCache; + } + + public bool IsCompressed(IReadOnlyContentBase content, string alias) + { + var compressedStorage = _isCompressedCache.GetOrAdd((content.ContentTypeId, alias), x => + { + if (!_contentTypes.TryGetValue(x.contentTypeId, out var ct)) + return false; + + var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + if (propertyType == null) return false; + + if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return false; + + return _compressionOptions.IsCompressed(content, propertyType, propertyEditor); + }); + + return compressedStorage; + } + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 6c473235a7..a146d5cc4a 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -152,14 +152,17 @@ + + - - - + + + + diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index 4be80083b8..b3543dad1a 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -2,6 +2,7 @@ using NUnit.Framework; using System; using System.Collections.Generic; +using Umbraco.Core.Models; using Umbraco.Core.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -11,13 +12,13 @@ namespace Umbraco.Tests.PublishedContent public class ContentSerializationTests { [Test] - public void Ensure_Same_Results() + public void GivenACacheModel_WhenItsSerializedAndDeserializedWithAnySerializer_TheResultsAreTheSame() { var jsonSerializer = new JsonContentNestedDataSerializer(); - var msgPackSerializer = new MsgPackContentNestedDataSerializer(Mock.Of()); + var msgPackSerializer = new MsgPackContentNestedDataSerializer(Mock.Of()); var now = DateTime.Now; - var content = new ContentCacheDataModel + var cacheModel = new ContentCacheDataModel { PropertyData = new Dictionary { @@ -53,14 +54,16 @@ namespace Umbraco.Tests.PublishedContent UrlSegment = "home" }; - var json = jsonSerializer.Serialize(1, content).StringData; - var msgPack = msgPackSerializer.Serialize(1, content).ByteData; + var content = Mock.Of(x => x.ContentTypeId == 1); + + var json = jsonSerializer.Serialize(content, cacheModel).StringData; + var msgPack = msgPackSerializer.Serialize(content, cacheModel).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(1, json, null); - var msgPackContent = msgPackSerializer.Deserialize(1, null, msgPack); + var jsonContent = jsonSerializer.Deserialize(content, json, null); + var msgPackContent = msgPackSerializer.Deserialize(content, null, msgPack); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs index 2c1221e99e..42023382f1 100644 --- a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs @@ -13,7 +13,6 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a block list property editor. /// - [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.BlockList, "Block List", diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 7ce312c516..862837381a 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -17,7 +17,6 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a grid property and parameter editor. /// - [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.Grid, "Grid layout", diff --git a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs index 6ce66aaa00..2d66da5461 100644 --- a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs @@ -7,7 +7,6 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a markdown editor. /// - [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.MarkdownEditor, "Markdown editor", diff --git a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs index 0e1cea3399..ca3c5a2a04 100644 --- a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs @@ -22,7 +22,6 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a nested content property editor. /// - [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.NestedContent, "Nested Content", diff --git a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs index 7c7a358bf3..42777f11ad 100644 --- a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs @@ -17,7 +17,6 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a rich text property editor. /// - [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TinyMce, "Rich Text Editor", diff --git a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs index 878330820a..c7bc2efbda 100644 --- a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs @@ -7,7 +7,6 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a textarea property and parameter editor. /// - [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TextArea, EditorType.PropertyValue | EditorType.MacroParameter, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs index be2f9921d1..343885b037 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentSourceDto.cs @@ -1,12 +1,13 @@ using System; +using Umbraco.Core.Models; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { // read-only dto - internal class ContentSourceDto + internal class ContentSourceDto : IReadOnlyContentBase { public int Id { get; set; } - public Guid Uid { get; set; } + public Guid Key { get; set; } public int ContentTypeId { get; set; } public int Level { get; set; } @@ -37,5 +38,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public int PubTemplateId { get; set; } public string PubData { get; set; } public byte[] PubDataRaw { get; set; } + + // Explicit implementation + DateTime IReadOnlyContentBase.UpdateDate => EditVersionDate; + string IReadOnlyContentBase.Name => EditName; + int IReadOnlyContentBase.WriterId => EditWriterId; } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 3bd30d2d73..4584344891 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -383,7 +383,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); + var deserializedContent = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); d = new ContentData { @@ -410,7 +410,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.PubData, dto.PubDataRaw); + var deserializedContent = serializer.Deserialize(dto, dto.PubData, dto.PubDataRaw); p = new ContentData { @@ -427,7 +427,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - var n = new ContentNode(dto.Id, dto.Uid, + var n = new ContentNode(dto.Id, dto.Key, dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId); var s = new ContentNodeKit @@ -446,7 +446,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (dto.EditData == null && dto.EditDataRaw == null) throw new InvalidOperationException("No data for media " + dto.Id); - var deserializedMedia = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); + var deserializedMedia = serializer.Deserialize(dto, dto.EditData, dto.EditDataRaw); var p = new ContentData { @@ -460,7 +460,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource CultureInfos = deserializedMedia.CultureData }; - var n = new ContentNode(dto.Id, dto.Uid, + var n = new ContentNode(dto.Id, dto.Key, dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId); var s = new ContentNodeKit diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs index f628c8981b..d1a83d8452 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -1,4 +1,4 @@ -using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Models; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { @@ -14,19 +14,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// Deserialize the data into a /// - /// - /// - /// - /// - ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData); + ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData); /// /// Serializes the /// - /// - /// - /// - ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model); + ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index e7bb5656bb..c4d40f721f 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,6 +1,7 @@ using Newtonsoft.Json; using System; using System.Collections.Generic; +using Umbraco.Core.Models; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource @@ -21,7 +22,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateFormatString = "o" }; - public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); @@ -29,7 +30,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return JsonConvert.DeserializeObject(stringData, _jsonSerializerSettings); } - public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 6ffa130da0..944d93107d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -4,6 +4,7 @@ using MessagePack.Resolvers; using System; using System.Linq; using System.Text; +using Umbraco.Core.Models; using Umbraco.Core.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource @@ -15,9 +16,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public class MsgPackContentNestedDataSerializer : IContentCacheDataSerializer { private readonly MessagePackSerializerOptions _options; - private readonly IPropertyCompressionOptions _propertyOptions; + private readonly IPropertyCacheCompression _propertyOptions; - public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions) + public MsgPackContentNestedDataSerializer(IPropertyCacheCompression propertyOptions) { _propertyOptions = propertyOptions ?? throw new ArgumentNullException(nameof(propertyOptions)); @@ -46,21 +47,21 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return json; } - public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) + public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) { if (byteData != null) { - var content = MessagePackSerializer.Deserialize(byteData, _options); - Expand(contentTypeId, content); - return content; + var cacheModel = MessagePackSerializer.Deserialize(byteData, _options); + Expand(content, cacheModel); + return cacheModel; } else if (stringData != null) { // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) var bin = Convert.FromBase64String(stringData); - var content = MessagePackSerializer.Deserialize(bin, _options); - Expand(contentTypeId, content); - return content; + var cacheModel = MessagePackSerializer.Deserialize(bin, _options); + Expand(content, cacheModel); + return cacheModel; } else { @@ -68,9 +69,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) + public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) { - Compress(contentTypeId, model); + Compress(content, model); var bytes = MessagePackSerializer.Serialize(model, _options); return new ContentCacheDataSerializationResult(null, bytes); } @@ -86,11 +87,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// read/decompressed as a string to be displayed on the front-end. This allows for potentially a significant /// memory savings but could also affect performance of first rendering pages while decompression occurs. /// - private void Compress(int contentTypeId, ContentCacheDataModel model) + private void Compress(IReadOnlyContentBase content, ContentCacheDataModel model) { foreach(var propertyAliasToData in model.PropertyData) { - if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) { foreach(var property in propertyAliasToData.Value.Where(x => x.Value != null && x.Value is string)) { @@ -104,11 +105,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(int contentTypeId, ContentCacheDataModel nestedData) + private void Expand(IReadOnlyContentBase content, ContentCacheDataModel nestedData) { foreach (var propertyAliasToData in nestedData.PropertyData) { - if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) + if (_propertyOptions.IsCompressed(content, propertyAliasToData.Key)) { foreach (var property in propertyAliasToData.Value.Where(x => x.Value != null)) { diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs index b509334604..fcc3fa2bb8 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -12,14 +12,21 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private readonly IMediaTypeService _mediaTypeService; private readonly IMemberTypeService _memberTypeService; private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache = new ConcurrentDictionary<(int, string), CompressedStorageAttribute>(); + private readonly IPropertyCacheCompressionOptions _compressionOptions; + private readonly ConcurrentDictionary<(int, string), bool> _isCompressedCache = new ConcurrentDictionary<(int, string), bool>(); - public MsgPackContentNestedDataSerializerFactory(IContentTypeService contentTypeService, IMediaTypeService mediaTypeService, IMemberTypeService memberTypeService, PropertyEditorCollection propertyEditors) + public MsgPackContentNestedDataSerializerFactory( + IContentTypeService contentTypeService, + IMediaTypeService mediaTypeService, + IMemberTypeService memberTypeService, + PropertyEditorCollection propertyEditors, + IPropertyCacheCompressionOptions compressionOptions) { _contentTypeService = contentTypeService; _mediaTypeService = mediaTypeService; _memberTypeService = memberTypeService; _propertyEditors = propertyEditors; + _compressionOptions = compressionOptions; } public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) @@ -53,8 +60,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } - var options = new CompressedStoragePropertyEditorCompressionOptions(contentTypes, _propertyEditors, _compressedStoragePropertyEditorCache); - var serializer = new MsgPackContentNestedDataSerializer(options); + var compression = new PropertyCacheCompression(_compressionOptions, contentTypes, _propertyEditors, _isCompressedCache); + var serializer = new MsgPackContentNestedDataSerializer(compression); return serializer; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index faeb4f90b4..98d8b91386 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -20,9 +20,11 @@ namespace Umbraco.Web.PublishedCache.NuCache } else { - composition.RegisterUnique(); + composition.RegisterUnique(); } - + + composition.RegisterUnique(); + composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); // register the NuCache database data source diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index ad8705ef47..dbbcb4ee9b 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1456,7 +1456,7 @@ namespace Umbraco.Web.PublishedCache.NuCache UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; - var serialized = serializer.Serialize(content.ContentTypeId, contentCacheData); + var serialized = serializer.Serialize(ReadOnlyContentBaseAdapter.Create(content), contentCacheData); var dto = new ContentNuDto { From 5a80b30289ff25b5640145331b7ca0659220c4da Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 28 Jan 2021 14:28:09 +1100 Subject: [PATCH 069/147] Fixes SQL (oops), wasn't appending the joins --- .../NuCache/DataSource/DatabaseDataSource.cs | 116 ++++++++++-------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 4584344891..0ca882b014 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -28,7 +28,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // we want arrays, we want them all loaded, not an enumerable - private Sql SqlContentSourcesSelect(IScope scope, Func, Sql> joins = null) + private Sql SqlContentSourcesSelect(IScope scope, Func> joins = null) { var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelect, tsql => tsql.Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), @@ -56,7 +56,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // TODO: I'm unsure how we can format the below into SQL templates also because right.Current and right.Published end up being parameters if (joins != null) - sql = joins(sql); + sql = sql.Append(joins(sql.SqlContext)); sql = sql .InnerJoin().On((left, right) => left.NodeId == right.NodeId) @@ -75,48 +75,58 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } - private Sql SqlContentSourcesSelectUmbracoNodeJoin(Sql s) + private Sql SqlContentSourcesSelectUmbracoNodeJoin(ISqlContext sqlContext) { - var syntax = s.SqlContext.SqlSyntax; - var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.SourcesSelectUmbracoNodeJoin, s => - s.InnerJoin("x") + var syntax = sqlContext.SqlSyntax; + + var sqlTemplate = sqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.SourcesSelectUmbracoNodeJoin, builder => + builder.InnerJoin("x") .On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x")); + var sql = sqlTemplate.Sql(); return sql; } - private Sql SqlWhereNodeId(Sql s, int id) + private Sql SqlWhereNodeId(ISqlContext sqlContext, int id) { - var syntax = s.SqlContext.SqlSyntax; - var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.WhereNodeId, s => - s.Where(x => x.NodeId == SqlTemplate.Arg("id"))); + var syntax = sqlContext.SqlSyntax; + + var sqlTemplate = sqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.WhereNodeId, builder => + builder.Where(x => x.NodeId == SqlTemplate.Arg("id"))); + var sql = sqlTemplate.Sql(id); return sql; } - private Sql SqlWhereNodeIdX(Sql s, int id) + private Sql SqlWhereNodeIdX(ISqlContext sqlContext, int id) { - var syntax = s.SqlContext.SqlSyntax; - var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.WhereNodeIdX, s => + var syntax = sqlContext.SqlSyntax; + + var sqlTemplate = sqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.WhereNodeIdX, s => s.Where(x => x.NodeId == SqlTemplate.Arg("id"), "x")); + var sql = sqlTemplate.Sql(id); return sql; } - private Sql SqlOrderByLevelIdSortOrder(Sql s) + private Sql SqlOrderByLevelIdSortOrder(ISqlContext sqlContext) { - var syntax = s.SqlContext.SqlSyntax; - var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.OrderByLevelIdSortOrder, s => + var syntax = sqlContext.SqlSyntax; + + var sqlTemplate = sqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.OrderByLevelIdSortOrder, s => s.OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder)); + var sql = sqlTemplate.Sql(); return sql; } - private Sql SqlObjectTypeNotTrashed(Sql s, Guid nodeObjectType) + private Sql SqlObjectTypeNotTrashed(ISqlContext sqlContext, Guid nodeObjectType) { - var syntax = s.SqlContext.SqlSyntax; - var sqlTemplate = s.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ObjectTypeNotTrashedFilter, s => + var syntax = sqlContext.SqlSyntax; + + var sqlTemplate = sqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ObjectTypeNotTrashedFilter, s => s.Where(x => x.NodeObjectType == SqlTemplate.Arg("nodeObjectType") && x.Trashed == SqlTemplate.Arg("trashed"))); + var sql = sqlTemplate.Sql(nodeObjectType, false); return sql; } @@ -126,7 +136,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// /// /// - private Sql SqlContentSourcesCount(IScope scope, Func, Sql> joins = null) + private Sql SqlContentSourcesCount(IScope scope, Func> joins = null) { var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesCount, tsql => tsql.Select(x => Alias(x.NodeId, "Id")) @@ -137,7 +147,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var sql = sqlTemplate.Sql(); if (joins != null) - sql = joins(sql); + sql = sql.Append(joins(sql.SqlContext)); // TODO: We can't use a template with this one because of the 'right.Current' and 'right.Published' ends up being a parameter so not sure how we can do that sql = sql @@ -150,7 +160,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } - private Sql SqlMediaSourcesSelect(IScope scope, Func, Sql> joins = null) + private Sql SqlMediaSourcesSelect(IScope scope, Func> joins = null) { var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.MediaSourcesSelect, tsql => tsql.Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), @@ -165,7 +175,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var sql = sqlTemplate.Sql(); if (joins != null) - sql = joins(sql); + sql = sql.Append(joins(sql.SqlContext)); // TODO: We can't use a template with this one because of the 'right.Published' ends up being a parameter so not sure how we can do that sql = sql @@ -175,7 +185,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } - private Sql SqlMediaSourcesCount(IScope scope, Func, Sql> joins = null) + private Sql SqlMediaSourcesCount(IScope scope, Func> joins = null) { var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.MediaSourcesCount, tsql => tsql.Select(x => Alias(x.NodeId, "Id")).From()); @@ -183,7 +193,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var sql = sqlTemplate.Sql(); if (joins != null) - sql = joins(sql); + sql = sql.Append(joins(sql.SqlContext)); // TODO: We can't use a template with this one because of the 'right.Current' ends up being a parameter so not sure how we can do that sql = sql @@ -196,9 +206,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public ContentNodeKit GetContentSource(IScope scope, int id) { var sql = SqlContentSourcesSelect(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) - .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Document)) + .Append(SqlWhereNodeId(scope.SqlContext, id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); var dto = scope.Database.Fetch(sql).FirstOrDefault(); @@ -211,12 +221,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetAllContentSources(IScope scope) { var sql = SqlContentSourcesSelect(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Document)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); // Use a more efficient COUNT query var sqlCountQuery = SqlContentSourcesCount(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Document)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); @@ -234,14 +244,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetBranchContentSources(IScope scope, int id) { var sql = SqlContentSourcesSelect(scope, SqlContentSourcesSelectUmbracoNodeJoin) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) - .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Document)) + .Append(SqlWhereNodeIdX(scope.SqlContext, id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); // Use a more efficient COUNT query var sqlCountQuery = SqlContentSourcesCount(scope, SqlContentSourcesSelectUmbracoNodeJoin) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) - .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Document)) + .Append(SqlWhereNodeIdX(scope.SqlContext, id)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); @@ -260,13 +270,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (!ids.Any()) yield break; var sql = SqlContentSourcesSelect(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Document)) .WhereIn(x => x.ContentTypeId, ids) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); // Use a more efficient COUNT query var sqlCountQuery = SqlContentSourcesCount(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Document)) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Document)) .WhereIn(x => x.ContentTypeId, ids); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); @@ -284,9 +294,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public ContentNodeKit GetMediaSource(IScope scope, int id) { var sql = SqlMediaSourcesSelect(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) - .Append(SqlWhereNodeId(scope.SqlContext.Sql(), id)) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Media)) + .Append(SqlWhereNodeId(scope.SqlContext, id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); var dto = scope.Database.Fetch(sql).FirstOrDefault(); @@ -299,12 +309,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetAllMediaSources(IScope scope) { var sql = SqlMediaSourcesSelect(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Media)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); // Use a more efficient COUNT query var sqlCountQuery = SqlMediaSourcesCount(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Media)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); @@ -321,14 +331,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetBranchMediaSources(IScope scope, int id) { var sql = SqlMediaSourcesSelect(scope, SqlContentSourcesSelectUmbracoNodeJoin) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) - .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Media)) + .Append(SqlWhereNodeIdX(scope.SqlContext, id)) + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); // Use a more efficient COUNT query var sqlCountQuery = SqlMediaSourcesCount(scope, SqlContentSourcesSelectUmbracoNodeJoin) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) - .Append(SqlWhereNodeIdX(scope.SqlContext.Sql(), id)); + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Media)) + .Append(SqlWhereNodeIdX(scope.SqlContext, id)); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); @@ -347,13 +357,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource if (!ids.Any()) yield break; var sql = SqlMediaSourcesSelect(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Media)) .WhereIn(x => x.ContentTypeId, ids) - .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext.Sql())); + .Append(SqlOrderByLevelIdSortOrder(scope.SqlContext)); // Use a more efficient COUNT query var sqlCountQuery = SqlMediaSourcesCount(scope) - .Append(SqlObjectTypeNotTrashed(scope.SqlContext.Sql(), Constants.ObjectTypes.Media)) + .Append(SqlObjectTypeNotTrashed(scope.SqlContext, Constants.ObjectTypes.Media)) .WhereIn(x => x.ContentTypeId, ids); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); From 91830e55533a8f09e92366aec612ae960f504fb9 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 28 Jan 2021 14:42:02 +1100 Subject: [PATCH 070/147] Adds error checking --- .../PublishedCache/NuCache/PublishedContent.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs index ac5719c015..24747c43ad 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedContent.cs @@ -298,7 +298,18 @@ namespace Umbraco.Web.PublishedCache.NuCache throw new PanicException($"failed to get content with id={id}"); } - id = UnwrapIPublishedContent(content)._contentNode.NextSiblingContentId; + var next = UnwrapIPublishedContent(content)._contentNode.NextSiblingContentId; + +#if DEBUG + // I've seen this happen but I think that may have been due to corrupt DB data due to my own + // bugs, but I'm leaving this here just in case we encounter it again while we're debugging. + if (next == id) + { + throw new PanicException($"The current content id {id} is the same as it's next sibling id {next}"); + } +#endif + + id = next; } } } From 3dd84fa6ff9232892b747a1371a9d3f7fe0c32d0 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sat, 6 Feb 2021 21:57:48 +1300 Subject: [PATCH 071/147] bugfix --- src/Umbraco.Core/Services/Implement/LocalizedTextService.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index 246481a48b..c10ae59907 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -265,6 +265,10 @@ namespace Umbraco.Core.Services.Implement { areaDictionary.TryGetValue(key, out found); } + if(found == null) + { + _noAreaDictionarySource[culture].TryGetValue(key, out found); + } } From e7baa0878878bded2d92a6e48f535e91331244e6 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sat, 6 Feb 2021 23:17:15 +1300 Subject: [PATCH 072/147] Improve getAllStoredValues. --- .../Services/ILocalizedTextService.cs | 7 +++++++ .../Implement/LocalizedTextService.cs | 21 ++++++++++++++++--- .../Editors/BackOfficeController.cs | 7 +++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Core/Services/ILocalizedTextService.cs b/src/Umbraco.Core/Services/ILocalizedTextService.cs index a4e5141a42..73212546a5 100644 --- a/src/Umbraco.Core/Services/ILocalizedTextService.cs +++ b/src/Umbraco.Core/Services/ILocalizedTextService.cs @@ -16,6 +16,13 @@ namespace Umbraco.Core.Services /// This can be null /// string Localize(string area, string alias, CultureInfo culture, IDictionary tokens = null); + + + /// + /// Returns all key/values in storage for the given culture + /// + /// + IDictionary> GetAllStoredValuesByAreaAndAlias(CultureInfo culture); } /// diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index c10ae59907..79c568e374 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -145,15 +145,15 @@ namespace Umbraco.Core.Services.Implement // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode culture = ConvertToSupportedCultureWithRegionCode(culture); - var result = new Dictionary(); + if (_dictionarySource.ContainsKey(culture) == false) { _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); - return result; + return new Dictionary(0); } - + IDictionary result = new Dictionary(); //convert all areas + keys to a single key with a '/' foreach (var area in _dictionarySource[culture]) { @@ -309,5 +309,20 @@ namespace Umbraco.Core.Services.Implement return value; } + public IDictionary> GetAllStoredValuesByAreaAndAlias(CultureInfo culture) + { + if (culture == null) throw new ArgumentNullException("culture"); + + // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode + culture = ConvertToSupportedCultureWithRegionCode(culture); + + if (_dictionarySource.ContainsKey(culture) == false) + { + _logger.Warn("The culture specified {Culture} was not found in any configured sources for this service", culture); + return new Dictionary>(0); + } + + return _dictionarySource[culture]; + } } } diff --git a/src/Umbraco.Web/Editors/BackOfficeController.cs b/src/Umbraco.Web/Editors/BackOfficeController.cs index 46bd52978d..5da0cd0cc3 100644 --- a/src/Umbraco.Web/Editors/BackOfficeController.cs +++ b/src/Umbraco.Web/Editors/BackOfficeController.cs @@ -207,6 +207,13 @@ namespace Umbraco.Web.Editors : CultureInfo.GetCultureInfo(GlobalSettings.DefaultUILanguage) : CultureInfo.GetCultureInfo(culture); + + if(Services.TextService is ILocalizedTextService2 localizedText2) + { + var nestedDictionary2 = localizedText2.GetAllStoredValuesByAreaAndAlias(cultureInfo); + return new JsonNetResult { Data = nestedDictionary2, Formatting = Formatting.None }; + } + var allValues = Services.TextService.GetAllStoredValues(cultureInfo); var pathedValues = allValues.Select(kv => { From 3202fe445be2a7c8003c6d3a259d08151a49f94b Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sat, 6 Feb 2021 23:26:13 +1300 Subject: [PATCH 073/147] Benchmark --- ...LocalizedTextServiceGetAllStoredValuesBenchmarks.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs b/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs index be45ffbd8c..f16df01ef4 100644 --- a/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs +++ b/src/Umbraco.Tests.Benchmarks/LocalizedTextServiceGetAllStoredValuesBenchmarks.cs @@ -41,6 +41,7 @@ namespace Umbraco.Tests.Benchmarks var result4 = _xmlService.GetAllStoredValues(culture); var result5 = _optimized.GetAllStoredValues(culture); var result6 = _xmlService.GetAllStoredValues(culture); + var result7 = _optimized.GetAllStoredValuesByAreaAndAlias(culture); } [Benchmark] @@ -100,6 +101,15 @@ namespace Umbraco.Tests.Benchmarks } } + [Benchmark] + public void OptimizedDictGetAllV2() + { + for (int i = 0; i < 10000; i++) + { + var result = _optimizedDict.GetAllStoredValuesByAreaAndAlias(culture); + } + } + [Benchmark()] public void OptimizedXmlLocalize() { From 094029134d8723cfe7e3996bc82f7f537dd44dd6 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Sun, 7 Feb 2021 00:06:32 +1300 Subject: [PATCH 074/147] Bugfix app key --- src/Umbraco.Web/Models/Mapping/CommonMapper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/Models/Mapping/CommonMapper.cs b/src/Umbraco.Web/Models/Mapping/CommonMapper.cs index d04579d5ee..2440ecc44c 100644 --- a/src/Umbraco.Web/Models/Mapping/CommonMapper.cs +++ b/src/Umbraco.Web/Models/Mapping/CommonMapper.cs @@ -80,7 +80,7 @@ namespace Umbraco.Web.Models.Mapping // localize content app names foreach (var app in apps) { - var localizedAppName = _localizedTextService.Localize($"apps", "{app.Alias}"); + var localizedAppName = _localizedTextService.Localize("apps", app.Alias); if (localizedAppName.Equals($"[{app.Alias}]", StringComparison.OrdinalIgnoreCase) == false) { app.Name = localizedAppName; From 603b38bd6c3b1b7903c0c714d0a79fde9306402d Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Wed, 10 Feb 2021 12:47:35 +1300 Subject: [PATCH 075/147] implement lazy --- .../Implement/LocalizedTextService.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index 79c568e374..f5a85dda7e 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -15,8 +15,10 @@ namespace Umbraco.Core.Services.Implement { private readonly ILogger _logger; private readonly Lazy _fileSources; - private readonly IDictionary>> _dictionarySource; - private readonly IDictionary> _noAreaDictionarySource; + private IDictionary>> _dictionarySource => _dictionarySourceLazy.Value; + private IDictionary> _noAreaDictionarySource => _noAreaDictionarySourceLazy.Value; + private readonly Lazy>>> _dictionarySourceLazy; + private readonly Lazy>> _noAreaDictionarySourceLazy; private readonly char[] _splitter = new[] { '/' }; /// /// Initializes with a file sources instance @@ -28,9 +30,8 @@ namespace Umbraco.Core.Services.Implement if (logger == null) throw new ArgumentNullException("logger"); _logger = logger; if (fileSources == null) throw new ArgumentNullException("fileSources"); - var dictionaries = FileSourcesToDictionarySources(fileSources.Value); - _dictionarySource = dictionaries.WithArea; - _noAreaDictionarySource = dictionaries.WithoutArea; + _dictionarySourceLazy = new Lazy>>>(()=> FileSourcesToDictionarySources(fileSources.Value).WithArea); + _noAreaDictionarySourceLazy = new Lazy>>(() => FileSourcesToDictionarySources(fileSources.Value).WithoutArea); _fileSources = fileSources; } @@ -72,9 +73,9 @@ namespace Umbraco.Core.Services.Implement if (source == null) throw new ArgumentNullException("source"); if (logger == null) throw new ArgumentNullException("logger"); _logger = logger; - var dictionaries = XmlSourcesToDictionarySources(source); - _dictionarySource = dictionaries.WithArea; - _noAreaDictionarySource = dictionaries.WithoutArea; + _dictionarySourceLazy = new Lazy>>>(()=> XmlSourcesToDictionarySources(source).WithArea); + _noAreaDictionarySourceLazy = new Lazy>>(() => XmlSourcesToDictionarySources(source).WithoutArea); + } @@ -85,10 +86,11 @@ namespace Umbraco.Core.Services.Implement /// public LocalizedTextService(IDictionary>> source, ILogger logger) { - _dictionarySource = source ?? throw new ArgumentNullException(nameof(source)); + var dictionarySource = source ?? throw new ArgumentNullException(nameof(source)); + _dictionarySourceLazy = new Lazy>>>(() => dictionarySource); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); var cultureNoAreaDictionary = new Dictionary>(); - foreach (var cultureDictionary in _dictionarySource) + foreach (var cultureDictionary in dictionarySource) { var areaAliaValue = GetAreaStoredTranslations(source, cultureDictionary.Key); var aliasValue = new Dictionary(); @@ -104,7 +106,7 @@ namespace Umbraco.Core.Services.Implement } cultureNoAreaDictionary.Add(cultureDictionary.Key, aliasValue); } - _noAreaDictionarySource = cultureNoAreaDictionary; + _noAreaDictionarySourceLazy = new Lazy>>(() => cultureNoAreaDictionary); } public string Localize(string key, CultureInfo culture, IDictionary tokens = null) From fb93e15c1046597c1675b3e5266a5b94501e739a Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 12 Feb 2021 14:50:21 +1300 Subject: [PATCH 076/147] fix key --- .../PublishedCache/NuCache/DataSource/DatabaseDataSource.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index e39f649eaf..9c6a799312 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -34,7 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = scope.SqlContext.Sql() - .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), + .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Key"), x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) @@ -130,7 +130,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { var sql = scope.SqlContext.Sql() - .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), + .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Key"), x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) From 2a8d043445fe180380f9dbd1ec01c2f29dc1ce81 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 12 Feb 2021 15:01:46 +1300 Subject: [PATCH 077/147] support bulk insert for binary in sql ce. Fix column ordering in contentnudto. --- src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs | 6 ++++-- .../Persistence/NPocoDatabaseExtensions-Bulk.cs | 11 ++++++++++- .../Persistence/SqlSyntax/SqlCeSyntaxProvider.cs | 9 ++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs b/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs index 664d188a10..a2f36584e0 100644 --- a/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/ContentNuDto.cs @@ -28,11 +28,13 @@ namespace Umbraco.Core.Persistence.Dtos [NullSetting(NullSetting = NullSettings.Null)] public string Data { get; set; } + [Column("rv")] + public long Rv { get; set; } + [Column("dataRaw")] [NullSetting(NullSetting = NullSettings.Null)] public byte[] RawData { get; set; } - [Column("rv")] - public long Rv { get; set; } + } } diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs index 10db1ca18e..bff682d095 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Data; using System.Data.SqlClient; using System.Data.SqlServerCe; +using System.Data.SqlTypes; using System.Linq; using NPoco; using Umbraco.Core.Persistence.SqlSyntax; @@ -210,7 +211,15 @@ namespace Umbraco.Core.Persistence if (IncludeColumn(pocoData, columns[i])) { var val = columns[i].Value.GetValue(record); - updatableRecord.SetValue(i, val); + if (val is byte[]) + { + var bytes = val as byte[]; + updatableRecord.SetSqlBinary(i, new SqlBinary(bytes)); + } + else + { + updatableRecord.SetValue(i, val); + } } } resultSet.Insert(updatableRecord); diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs index 0b564212ce..9c5edf595b 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlCeSyntaxProvider.cs @@ -261,6 +261,13 @@ where table_name=@0 and column_name=@1", tableName, columnName).FirstOrDefault() return "NTEXT"; return base.GetSpecialDbType(dbTypes); } - + public override SqlDbType GetSqlDbType(DbType dbType) + { + if (DbType.Binary == dbType) + { + return SqlDbType.Image; + } + return base.GetSqlDbType(dbType); + } } } From d425fbe05e97db92f9ea33a2e800f95e4bfa2d27 Mon Sep 17 00:00:00 2001 From: nzdev <834725+nzdev@users.noreply.github.com> Date: Fri, 12 Feb 2021 15:41:46 +1300 Subject: [PATCH 078/147] rebuild nucache if serializer changes --- .../NuCache/NuCacheSerializerComponent.cs | 66 +++++++++++++++++++ .../NuCache/NuCacheSerializerComposer.cs | 21 ++++++ src/Umbraco.Web/Umbraco.Web.csproj | 2 + src/umbraco.sln | 24 ++++++- 4 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComposer.cs diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs new file mode 100644 index 0000000000..4b4ccd295a --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Composing; +using Umbraco.Core.Logging; +using Umbraco.Core.Services; + +namespace Umbraco.Web.PublishedCache.NuCache +{ + /// + /// Rebuilds the database cache if required when the serializer changes + /// + public class NuCacheSerializerComponent : IComponent + { + private const string Nucache_Serializer_Key = "Umbraco.Web.PublishedCache.NuCache.Serializer"; + private const string JSON_SERIALIZER_VALUE = "JSON"; + private readonly IPublishedSnapshotService _service; + private readonly IKeyValueService _keyValueService; + private readonly IProfilingLogger _profilingLogger; + + public NuCacheSerializerComponent(IPublishedSnapshotService service, IKeyValueService keyValueService,IProfilingLogger profilingLogger) + { + // service: nothing - this just ensures that the service is created at boot time + _service = service; + _keyValueService = keyValueService; + _profilingLogger = profilingLogger; + } + + public void Initialize() + { + RebuildDatabaseCacheIfSerializerChanged(); + } + + private void RebuildDatabaseCacheIfSerializerChanged() + { + var serializer = ConfigurationManager.AppSettings[Nucache_Serializer_Key]; + var currentSerializer = _keyValueService.GetValue(Nucache_Serializer_Key); + + if (currentSerializer == null) + { + currentSerializer = JSON_SERIALIZER_VALUE; + } + if (serializer == null) + { + serializer = JSON_SERIALIZER_VALUE; + } + + if (serializer != currentSerializer) + { + _profilingLogger.Info($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) + { + + _service.Rebuild(); + _keyValueService.SetValue(Nucache_Serializer_Key, serializer); + } + } + } + + public void Terminate() + { } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComposer.cs new file mode 100644 index 0000000000..59a206bc47 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComposer.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core; +using Umbraco.Core.Composing; + +namespace Umbraco.Web.PublishedCache.NuCache +{ + + [ComposeAfter(typeof(NuCacheComposer))] + [RuntimeLevel(MinLevel = RuntimeLevel.Run)] + public class NuCacheSerializerComposer : ICoreComposer + { + public void Compose(Composition composition) + { + composition.Components().Append(); + } + } +} diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 088fb0eeb3..0878867162 100755 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -251,6 +251,8 @@ + + diff --git a/src/umbraco.sln b/src/umbraco.sln index 63fb856b5d..72b06a517a 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -58,8 +58,24 @@ Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Web.UI.Client", "ht StartServerOnDebug = "false" EndProjectSection EndProject -Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest\", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" +Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest(1)", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" ProjectSection(WebsiteProperties) = preProject + TargetFrameworkMoniker = ".NETFramework,Version%3Dv4.0" + Debug.AspNetCompiler.VirtualPath = "/localhost_62926" + Debug.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" + Debug.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_62926\" + Debug.AspNetCompiler.Updateable = "true" + Debug.AspNetCompiler.ForceOverwrite = "true" + Debug.AspNetCompiler.FixedNames = "false" + Debug.AspNetCompiler.Debug = "True" + Release.AspNetCompiler.VirtualPath = "/localhost_62926" + Release.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" + Release.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_62926\" + Release.AspNetCompiler.Updateable = "true" + Release.AspNetCompiler.ForceOverwrite = "true" + Release.AspNetCompiler.FixedNames = "false" + Release.AspNetCompiler.Debug = "False" + VWDPort = "62926" SlnRelativePath = "Umbraco.Tests.AcceptanceTest\" EndProjectSection EndProject @@ -123,6 +139,10 @@ Global {4C4C194C-B5E4-4991-8F87-4373E24CC19F}.Release|Any CPU.Build.0 = Release|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Release|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.ActiveCfg = Debug|Any CPU + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -157,6 +177,7 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {227C3B55-80E5-4E7E-A802-BE16C5128B9D} = {2849E9D4-3B4E-40A3-A309-F3CB4F0E125F} + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {5D3B8245-ADA6-453F-A008-50ED04BFE770} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {E3F9F378-AFE1-40A5-90BD-82833375DBFE} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} {5B03EF4E-E0AC-4905-861B-8C3EC1A0D458} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} @@ -164,7 +185,6 @@ Global {3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {C7311C00-2184-409B-B506-52A5FAEA8736} = {FD962632-184C-4005-A5F3-E705D92FC645} {FB5676ED-7A69-492C-B802-E7B24144C0FC} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7A0F2E34-D2AF-4DAB-86A0-7D7764B3D0EC} From bfa6bc43d8c55818b021da5f49ca1fbb6d681a58 Mon Sep 17 00:00:00 2001 From: nzdev Date: Tue, 16 Feb 2021 20:56:44 +1300 Subject: [PATCH 079/147] Implement selectfields for examine --- build/NuSpecs/UmbracoCms.Web.nuspec | 2 +- src/Umbraco.Examine/Umbraco.Examine.csproj | 2 +- src/Umbraco.Examine/UmbracoContentIndex.cs | 7 +++-- src/Umbraco.Tests/Umbraco.Tests.csproj | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 2 +- src/Umbraco.Web/IPublishedContentQuery.cs | 25 ++++++++++++++++ src/Umbraco.Web/PublishedContentQuery.cs | 12 ++++++-- src/Umbraco.Web/Search/ExamineComponent.cs | 4 ++- .../Search/GenericIndexDiagnostics.cs | 4 ++- .../Search/IUmbracoTreeSearcherFields2.cs | 29 +++++++++++++++++++ src/Umbraco.Web/Search/UmbracoTreeSearcher.cs | 16 +++++++++- .../Search/UmbracoTreeSearcherFields.cs | 26 ++++++++++++++++- src/Umbraco.Web/Umbraco.Web.csproj | 3 +- 13 files changed, 120 insertions(+), 14 deletions(-) create mode 100644 src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs diff --git a/build/NuSpecs/UmbracoCms.Web.nuspec b/build/NuSpecs/UmbracoCms.Web.nuspec index 82d15d2b95..59bd7d75aa 100644 --- a/build/NuSpecs/UmbracoCms.Web.nuspec +++ b/build/NuSpecs/UmbracoCms.Web.nuspec @@ -28,7 +28,7 @@ - + diff --git a/src/Umbraco.Examine/Umbraco.Examine.csproj b/src/Umbraco.Examine/Umbraco.Examine.csproj index 0e0ee62139..517edf354c 100644 --- a/src/Umbraco.Examine/Umbraco.Examine.csproj +++ b/src/Umbraco.Examine/Umbraco.Examine.csproj @@ -49,7 +49,7 @@ - + 1.0.0-beta2-19324-01 runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/Umbraco.Examine/UmbracoContentIndex.cs b/src/Umbraco.Examine/UmbracoContentIndex.cs index 88033b1407..468e794110 100644 --- a/src/Umbraco.Examine/UmbracoContentIndex.cs +++ b/src/Umbraco.Examine/UmbracoContentIndex.cs @@ -11,7 +11,7 @@ using Lucene.Net.Store; using Umbraco.Core.Composing; using Umbraco.Core.Logging; using Examine.LuceneEngine; - +using Examine.Search; namespace Umbraco.Examine { /// @@ -21,7 +21,7 @@ namespace Umbraco.Examine { public const string VariesByCultureFieldName = SpecialFieldPrefix + "VariesByCulture"; protected ILocalizationService LanguageService { get; } - + private readonly ISet _idOnlyFieldSet = new HashSet { "id" }; #region Constructors /// @@ -131,8 +131,9 @@ namespace Umbraco.Examine var searcher = GetSearcher(); var c = searcher.CreateQuery(); var filtered = c.NativeQuery(rawQuery); - var results = filtered.Execute(); + var selectedFields = filtered.SelectFields(_idOnlyFieldSet); + var results = selectedFields.Execute(); ProfilingLogger.Debug(GetType(), "DeleteFromIndex with query: {Query} (found {TotalItems} results)", rawQuery, results.TotalItemCount); //need to queue a delete item for each one found diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 2ac28aa7d7..97604df0c6 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -79,7 +79,7 @@ - + 1.8.14 diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 01d029cae0..69bdeba643 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -88,7 +88,7 @@ - + diff --git a/src/Umbraco.Web/IPublishedContentQuery.cs b/src/Umbraco.Web/IPublishedContentQuery.cs index 7066475dc9..f513c1ac02 100644 --- a/src/Umbraco.Web/IPublishedContentQuery.cs +++ b/src/Umbraco.Web/IPublishedContentQuery.cs @@ -9,6 +9,31 @@ using Umbraco.Core.Xml; namespace Umbraco.Web { using Examine = global::Examine; + public interface IPublishedContentQuery2 : IPublishedContentQuery + { + /// + /// Searches content. + /// + /// The term to search. + /// The amount of results to skip. + /// The amount of results to take/return. + /// The total amount of records. + /// The culture (defaults to a culture insensitive search). + /// The name of the index to search (defaults to ). + /// The fields to load in the results of the search (defaults to all fields loaded). + /// + /// The search results. + /// + /// + /// + /// When the is not specified or is *, all cultures are searched. + /// To search for only invariant documents and fields use null. + /// When searching on a specific culture, all culture specific fields are searched for the provided culture and all invariant fields for all documents. + /// + /// While enumerating results, the ambient culture is changed to be the searched culture. + /// + IEnumerable Search(string term, int skip, int take, out long totalRecords, string culture = "*", string indexName = Umbraco.Core.Constants.UmbracoIndexes.ExternalIndexName, ISet loadedFields = null); + } /// /// Query methods used for accessing strongly typed content in templates diff --git a/src/Umbraco.Web/PublishedContentQuery.cs b/src/Umbraco.Web/PublishedContentQuery.cs index d697898f33..760a4b1c82 100644 --- a/src/Umbraco.Web/PublishedContentQuery.cs +++ b/src/Umbraco.Web/PublishedContentQuery.cs @@ -16,7 +16,7 @@ namespace Umbraco.Web /// /// A class used to query for published content, media items /// - public class PublishedContentQuery : IPublishedContentQuery + public class PublishedContentQuery : IPublishedContentQuery2 { private readonly IPublishedSnapshot _publishedSnapshot; private readonly IVariationContextAccessor _variationContextAccessor; @@ -190,6 +190,10 @@ namespace Umbraco.Web /// public IEnumerable Search(string term, int skip, int take, out long totalRecords, string culture = "*", string indexName = Constants.UmbracoIndexes.ExternalIndexName) + => Search(term, skip, take, out totalRecords, culture, indexName, null); + + /// + public IEnumerable Search(string term, int skip, int take, out long totalRecords, string culture = "*", string indexName = Constants.UmbracoIndexes.ExternalIndexName, ISet loadedFields = null) { if (skip < 0) { @@ -212,7 +216,7 @@ namespace Umbraco.Web } var query = umbIndex.GetSearcher().CreateQuery(IndexTypes.Content); - + IQueryExecutor queryExecutor; if (culture == "*") { @@ -231,6 +235,10 @@ namespace Umbraco.Web var fields = umbIndex.GetCultureAndInvariantFields(culture).ToArray(); // Get all index fields suffixed with the culture name supplied queryExecutor = query.ManagedQuery(term, fields); } + if (loadedFields != null && queryExecutor is IBooleanOperation booleanOperation) + { + queryExecutor = booleanOperation.SelectFields(loadedFields); + } var results = skip == 0 && take == 0 ? queryExecutor.Execute() diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index c9d7b7cf56..4fa2427fac 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -19,6 +19,7 @@ using Umbraco.Core.Composing; using System.ComponentModel; using System.Threading; using Umbraco.Web.Scheduling; +using Examine.Search; namespace Umbraco.Web.Search { @@ -39,6 +40,7 @@ namespace Umbraco.Web.Search private readonly IProfilingLogger _logger; private readonly IUmbracoIndexesCreator _indexCreator; private readonly BackgroundTaskRunner _indexItemTaskRunner; + private readonly ISet _idOnlyFieldSet = new HashSet { "id" }; // the default enlist priority is 100 @@ -425,7 +427,7 @@ namespace Umbraco.Web.Search while (page * pageSize < total) { //paging with examine, see https://shazwazza.com/post/paging-with-examine/ - var results = searcher.CreateQuery().Field("nodeType", id.ToInvariantString()).Execute(maxResults: pageSize * (page + 1)); + var results = searcher.CreateQuery().Field("nodeType", id.ToInvariantString()).SelectFields(_idOnlyFieldSet).Execute(maxResults: pageSize * (page + 1)); total = results.TotalItemCount; var paged = results.Skip(page * pageSize); diff --git a/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs b/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs index cb25e1242a..c8d3a72a96 100644 --- a/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs +++ b/src/Umbraco.Web/Search/GenericIndexDiagnostics.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using Examine; +using Examine.Search; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Examine; @@ -17,6 +18,7 @@ namespace Umbraco.Web.Search private readonly IIndex _index; private static readonly string[] IgnoreProperties = { "Description" }; + private readonly ISet _idOnlyFieldSet = new HashSet { "id" }; public GenericIndexDiagnostics(IIndex index) { _index = index; @@ -34,7 +36,7 @@ namespace Umbraco.Web.Search try { var searcher = _index.GetSearcher(); - var result = searcher.Search("test"); + var result = searcher.CreateQuery().ManagedQuery("test").SelectFields(_idOnlyFieldSet).Execute(1); return Attempt.Succeed(); //if we can search we'll assume it's healthy } catch (Exception e) diff --git a/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs b/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs new file mode 100644 index 0000000000..42f592a965 --- /dev/null +++ b/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Web.Search +{ + public interface IUmbracoTreeSearcherFields2 : IUmbracoTreeSearcherFields + { + /// + /// Set of fields for all node types to be loaded + /// + ISet GetBackOfficeFieldsToLoad(); + /// + /// Set list of fields for Members to be loaded + /// + ISet GetBackOfficeMembersFieldsToLoad(); + /// + /// Set of fields for Media to be loaded + /// + ISet GetBackOfficeMediaFieldsToLoad(); + + /// + /// Set of fields for Documents to be loaded + /// + ISet GetBackOfficeDocumentFieldsToLoad(); + } +} diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs index 410b654e32..44ba0766e0 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using Examine; +using Examine.Search; using Umbraco.Core; using Umbraco.Core.Mapping; using Umbraco.Core.Models; @@ -82,6 +83,7 @@ namespace Umbraco.Web.Search string type; var indexName = Constants.UmbracoIndexes.InternalIndexName; var fields = _umbracoTreeSearcherFields.GetBackOfficeFields().ToList(); + ISet fieldsToLoad = null; // TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string // manipulation for things like start paths, member types, etc... @@ -102,6 +104,10 @@ namespace Umbraco.Web.Search indexName = Constants.UmbracoIndexes.MembersIndexName; type = "member"; fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMembersFields()); + if (_umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 umbracoTreeSearcherFieldMember) + { + fieldsToLoad = umbracoTreeSearcherFieldMember.GetBackOfficeMembersFieldsToLoad(); + } if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1") { sb.Append("+__NodeTypeAlias:"); @@ -112,12 +118,20 @@ namespace Umbraco.Web.Search case UmbracoEntityTypes.Media: type = "media"; fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMediaFields()); + if (_umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 umbracoTreeSearcherFieldsMedia) + { + fieldsToLoad = umbracoTreeSearcherFieldsMedia.GetBackOfficeMediaFieldsToLoad(); + } var allMediaStartNodes = _umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService); AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService); break; case UmbracoEntityTypes.Document: type = "content"; fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeDocumentFields()); + if (_umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 umbracoTreeSearcherFieldsDocument) + { + fieldsToLoad = umbracoTreeSearcherFieldsDocument.GetBackOfficeDocumentFieldsToLoad(); + } var allContentStartNodes = _umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(_entityService); AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService); break; @@ -136,7 +150,7 @@ namespace Umbraco.Web.Search return Enumerable.Empty(); } - var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString()) + var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString()).SelectFields(fieldsToLoad) //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested .Execute(Convert.ToInt32(pageSize * (pageIndex + 1))); diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs index f90d7bc6b6..5a2fd91d18 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs @@ -4,7 +4,7 @@ using Umbraco.Examine; namespace Umbraco.Web.Search { - public class UmbracoTreeSearcherFields : IUmbracoTreeSearcherFields + public class UmbracoTreeSearcherFields : IUmbracoTreeSearcherFields2 { private IReadOnlyList _backOfficeFields = new List {"id", "__NodeId", "__Key"}; public IEnumerable GetBackOfficeFields() @@ -27,5 +27,29 @@ namespace Umbraco.Web.Search { return Enumerable.Empty(); } + + private readonly ISet _backOfficeFieldsToLoad = new HashSet { "id", "__NodeId", "__Key" }; + public ISet GetBackOfficeFieldsToLoad() + { + return _backOfficeFieldsToLoad; + } + + private readonly ISet _backOfficeMembersFieldsToLoad = new HashSet { "id", "__NodeId", "__Key", "email", "loginName" }; + public ISet GetBackOfficeMembersFieldsToLoad() + { + return _backOfficeMembersFieldsToLoad; + } + + private readonly ISet _backOfficeMediaFieldsToLoad = new HashSet { "id", "__NodeId", "__Key", UmbracoExamineIndex.UmbracoFileFieldName }; + public ISet GetBackOfficeMediaFieldsToLoad() + { + return _backOfficeMediaFieldsToLoad; + } + private readonly ISet _backOfficeDocumentFieldsToLoad = new HashSet { "id", "__NodeId", "__Key" }; + + public ISet GetBackOfficeDocumentFieldsToLoad() + { + return _backOfficeDocumentFieldsToLoad; + } } } diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 8890d9cf25..189d2e02d1 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -63,7 +63,7 @@ - + 4.0.217 @@ -284,6 +284,7 @@ + From 92d11845f511a97ea00b4836af3672ec22803689 Mon Sep 17 00:00:00 2001 From: Shannon Deminick Date: Fri, 12 Mar 2021 14:00:25 +1100 Subject: [PATCH 080/147] Reverting accidental change --- src/umbraco.sln | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/umbraco.sln b/src/umbraco.sln index 72b06a517a..4d630a8ead 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -58,24 +58,8 @@ Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Web.UI.Client", "ht StartServerOnDebug = "false" EndProjectSection EndProject -Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest(1)", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" +Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Umbraco.Tests.AcceptanceTest\", "Umbraco.Tests.AcceptanceTest\", "{9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}" ProjectSection(WebsiteProperties) = preProject - TargetFrameworkMoniker = ".NETFramework,Version%3Dv4.0" - Debug.AspNetCompiler.VirtualPath = "/localhost_62926" - Debug.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" - Debug.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_62926\" - Debug.AspNetCompiler.Updateable = "true" - Debug.AspNetCompiler.ForceOverwrite = "true" - Debug.AspNetCompiler.FixedNames = "false" - Debug.AspNetCompiler.Debug = "True" - Release.AspNetCompiler.VirtualPath = "/localhost_62926" - Release.AspNetCompiler.PhysicalPath = "Umbraco.Tests.AcceptanceTest\" - Release.AspNetCompiler.TargetPath = "PrecompiledWeb\localhost_62926\" - Release.AspNetCompiler.Updateable = "true" - Release.AspNetCompiler.ForceOverwrite = "true" - Release.AspNetCompiler.FixedNames = "false" - Release.AspNetCompiler.Debug = "False" - VWDPort = "62926" SlnRelativePath = "Umbraco.Tests.AcceptanceTest\" EndProjectSection EndProject From 097a46189f3d72fdc5c42877c3900185cf88df70 Mon Sep 17 00:00:00 2001 From: Shannon Deminick Date: Fri, 12 Mar 2021 14:01:04 +1100 Subject: [PATCH 081/147] Reverting accidental change --- src/umbraco.sln | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/umbraco.sln b/src/umbraco.sln index 4d630a8ead..eec774344b 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -123,10 +123,6 @@ Global {4C4C194C-B5E4-4991-8F87-4373E24CC19F}.Release|Any CPU.Build.0 = Release|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {3819A550-DCEC-4153-91B4-8BA9F7F0B9B4}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.ActiveCfg = Debug|Any CPU - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817}.Release|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Debug|Any CPU.Build.0 = Debug|Any CPU {651E1350-91B6-44B7-BD60-7207006D7003}.Release|Any CPU.ActiveCfg = Release|Any CPU From b606b925dd5554731798a4eea551f7065ba9ca08 Mon Sep 17 00:00:00 2001 From: Shannon Deminick Date: Fri, 12 Mar 2021 14:01:51 +1100 Subject: [PATCH 082/147] Reverting accidental change --- src/umbraco.sln | 1 - 1 file changed, 1 deletion(-) diff --git a/src/umbraco.sln b/src/umbraco.sln index eec774344b..9f2b19c2dd 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -157,7 +157,6 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {227C3B55-80E5-4E7E-A802-BE16C5128B9D} = {2849E9D4-3B4E-40A3-A309-F3CB4F0E125F} - {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {5D3B8245-ADA6-453F-A008-50ED04BFE770} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {E3F9F378-AFE1-40A5-90BD-82833375DBFE} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} {5B03EF4E-E0AC-4905-861B-8C3EC1A0D458} = {227C3B55-80E5-4E7E-A802-BE16C5128B9D} From efbfa0ba30ed1ab11ba86a5ab75c886a3a9c877d Mon Sep 17 00:00:00 2001 From: Shannon Deminick Date: Fri, 12 Mar 2021 14:02:48 +1100 Subject: [PATCH 083/147] Reverting accidental change --- src/umbraco.sln | 1 + 1 file changed, 1 insertion(+) diff --git a/src/umbraco.sln b/src/umbraco.sln index 9f2b19c2dd..63fb856b5d 100644 --- a/src/umbraco.sln +++ b/src/umbraco.sln @@ -164,6 +164,7 @@ Global {3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} {C7311C00-2184-409B-B506-52A5FAEA8736} = {FD962632-184C-4005-A5F3-E705D92FC645} {FB5676ED-7A69-492C-B802-E7B24144C0FC} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} + {9E4C8A12-FBE0-4673-8CE2-DF99D5D57817} = {B5BD12C1-A454-435E-8A46-FF4A364C0382} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7A0F2E34-D2AF-4DAB-86A0-7D7764B3D0EC} From a62112ab823f266ba40503cb3eb02dce4aa85837 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 12 Mar 2021 16:33:09 +1100 Subject: [PATCH 084/147] Don't call the same method over and over --- .../PublishedCache/NuCache/PublishedSnapshotService.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index dbbcb4ee9b..d26e0c67f7 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1492,9 +1492,12 @@ namespace Umbraco.Web.PublishedCache.NuCache scope.ReadLock(Constants.Locks.ContentTree); scope.ReadLock(Constants.Locks.MediaTree); scope.ReadLock(Constants.Locks.MemberTree); - RebuildContentDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); - RebuildMediaDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); - RebuildMemberDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); + + var groupSize = GetSqlPagingSize(); + + RebuildContentDbCacheLocked(serializer, scope, groupSize, null); + RebuildMediaDbCacheLocked(serializer, scope, groupSize, null); + RebuildMemberDbCacheLocked(serializer, scope, groupSize, null); scope.Complete(); } } From e6b56d9a3c88d4cf9ee1ec83dd18097420d7b350 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 12 Mar 2021 16:33:37 +1100 Subject: [PATCH 085/147] missing null check --- src/Umbraco.Web/UmbracoApplicationBase.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Umbraco.Web/UmbracoApplicationBase.cs b/src/Umbraco.Web/UmbracoApplicationBase.cs index 32a949e972..ef773ab56d 100644 --- a/src/Umbraco.Web/UmbracoApplicationBase.cs +++ b/src/Umbraco.Web/UmbracoApplicationBase.cs @@ -169,6 +169,8 @@ namespace Umbraco.Web { var exception = Server.GetLastError(); + if (exception == null) return; + // ignore HTTP errors if (exception.GetType() == typeof(HttpException)) return; From 8c7abc60aa4b5c53265dba228b1dbba9d8c23269 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 12 Mar 2021 16:50:29 +1100 Subject: [PATCH 086/147] inject lazy service just to be safe --- .../NuCache/NuCacheSerializerComponent.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs index 4b4ccd295a..19499dacf9 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheSerializerComponent.cs @@ -17,13 +17,15 @@ namespace Umbraco.Web.PublishedCache.NuCache { private const string Nucache_Serializer_Key = "Umbraco.Web.PublishedCache.NuCache.Serializer"; private const string JSON_SERIALIZER_VALUE = "JSON"; - private readonly IPublishedSnapshotService _service; + private readonly Lazy _service; private readonly IKeyValueService _keyValueService; private readonly IProfilingLogger _profilingLogger; - public NuCacheSerializerComponent(IPublishedSnapshotService service, IKeyValueService keyValueService,IProfilingLogger profilingLogger) + public NuCacheSerializerComponent(Lazy service, IKeyValueService keyValueService, IProfilingLogger profilingLogger) { - // service: nothing - this just ensures that the service is created at boot time + // We are using lazy here as a work around because the service does quite a lot of initialization in the ctor which + // we want to avoid where possible. Since we only need the service if we are rebuilding, we don't want to eagerly + // initialize anything unless we need to. _service = service; _keyValueService = keyValueService; _profilingLogger = profilingLogger; @@ -50,11 +52,11 @@ namespace Umbraco.Web.PublishedCache.NuCache if (serializer != currentSerializer) { - _profilingLogger.Info($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); + _profilingLogger.Warn($"Database NuCache was serialized using {currentSerializer}. Currently configured NuCache serializer {serializer}. Rebuilding Nucache"); + using (_profilingLogger.TraceDuration($"Rebuilding NuCache database with {currentSerializer} serializer")) { - - _service.Rebuild(); + _service.Value.Rebuild(); _keyValueService.SetValue(Nucache_Serializer_Key, serializer); } } From 5f45e6fd83a957b9df357722f624caf81c4e550e Mon Sep 17 00:00:00 2001 From: Chad Currie Date: Tue, 20 Apr 2021 22:42:24 +1200 Subject: [PATCH 087/147] Provide capacity values for DictionaryOfPropertyDataSerializer --- .../BTree.DictionaryOfPropertyDataSerializer.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index aa5dc9eb30..e5d8bfe780 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -10,10 +10,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public IDictionary ReadFrom(Stream stream) { - var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); // read properties count var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + var dict = new Dictionary(pcount,StringComparer.InvariantCultureIgnoreCase); // read each property for (var i = 0; i < pcount; i++) @@ -25,13 +25,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); // create pdata and add to the dictionary - var pdatas = new List(); + var pdatas = new PropertyData[vcount]; // for each value, read and add to pdata for (var j = 0; j < vcount; j++) { var pdata = new PropertyData(); - pdatas.Add(pdata); + pdatas[j] = pdata; // everything that can be null is read/written as object // even though - culture and segment should never be null here, as 'null' represents @@ -43,7 +43,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Value = ReadObject(stream); } - dict[key] = pdatas.ToArray(); + dict[key] = pdatas; } return dict; } From 98d8971ce9170571a90a334e5180d8dfac28af1f Mon Sep 17 00:00:00 2001 From: Chad Date: Wed, 21 Apr 2021 11:05:01 +1200 Subject: [PATCH 088/147] Merge in v8/contrib to V8/feature/nucache perf sync (#10151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version to 8.6.8 * Initial rework of Lock dictionaries * [Issue 5277-146] accessibility - Close 'X' icon next to language drop… (#9264) * [Issue 5277-146] accessibility - Close 'X' icon next to language drop down is identified as "link" - screen reader * add new loacalization key * Fix issue with SqlMainDomLock that cannot use implicit lock timeouts … (#9973) * Fix issue with SqlMainDomLock that cannot use implicit lock timeouts … (#9973) (cherry picked from commit da5351dfcf23daad69fcd73eb74811456ffc34c0) * Adjust unit tests and apply fixes to scope * Add more unit tests, showing current issue * Counting Umbraco.ModelsBuilder and ModelsBuilder.Umbraco namespaces as external providers * Fix dead lock with TypeLoader * Fix errors shown in unit tests * Throw error if all scopes hasn't been disposed * Clean * Fixes and Updates for DB Scope and Ambient Context leaks (#9953) * Adds some scope tests (ported back from netcore) and provides a much better error message, ensure execution context is not flowed to child tasks that shouldn't leak any current ambient context * updates comment * Ensure SqlMainDomLock suppresses execution context too * Since we're awaiting a task in a library method, ConfigureAwait(false) * missing null check Co-authored-by: Elitsa Marinovska * Adds additional error checking and reporting to MainDom/SqlMainDomLock (#9954) Co-authored-by: Elitsa Marinovska * Add copy logic to Media Picker (#9957) * Add copy logic to Media Picker * Add action for copy all * Fix for selectable media item * Wrap calls to map in scopes * Autocomplete scopes * Remove unnecessary aria-hidden attribute from * Remove scope from method that calls another method that has a scope * Fixes #9993 - Cannot save empty image in Grid * Clean * Revert "The Value() method for IPublishedContent was not working with the defaultValue parameter" (#9989) * Use a hashset to keep track of acquired locks This simplifies disposing/checking for locks greatly. * Add images in grid - fixes 9982 (#9987) Co-authored-by: Sebastiaan Janssen * Only create the dicts and hashset when a lock is requested * Clean * Adds a config for configuring the access rules on the content dashboard - by default it granted for all user groups * Adds additional params indicating whether user is admin * Add images in grid - fixes 9982 (#9987) Co-authored-by: Sebastiaan Janssen (cherry picked from commit e2019777fbfc1f9221d040cb9f0b82c57f8552b9) * Bump version to 8.12.2 * #9964 Removed unneeded check for HttpContext * Fix for #9950 - HttpsCheck will now retry using the login background image if inital request returns 301/302. Excessvie Headers check will now check the root url instead of the backoffice * Merge pull request #9994 from umbraco/v8/bugfix/9993 Fixes #9993 - Cannot save empty image in Grid (cherry picked from commit 0ecc933921f2dea9a2a16d6f395b44a039663ec6) * Apply suggestions from review * Fixes #9983 - Getting kicked, if document type has a Umbraco.UserPicker property (#10002) * Fixes #9983 Temporary fix for this issue. using the entityservice like before. * Needed to remove the call to usersResource here as well for displaying the picked items * Don't need usersResource for now * Fixes #9983 - Getting kicked, if document type has a Umbraco.UserPicker property (#10002) * Fixes #9983 Temporary fix for this issue. using the entityservice like before. * Needed to remove the call to usersResource here as well for displaying the picked items * Don't need usersResource for now (cherry picked from commit 45de0a101eaa2b8f16e21a765f32928c7cb968be) * 8539: Allow alias in image cropper (#9266) Co-authored-by: Owain Williams * Wrap dumping dictionaries in a method. * Create method for generating log message And remove forgotten comments. * Fix swedish translation for somethingElse. * Copy member type (#10020) * Add copy dialog for member type * Implement copy action for member type * Create specific localization for content type, media type and member type * Handle "foldersonly" querystring * Add button type attribute * Add a few missing changes of anchor to button element * Null check on scope and options to ensure backward compatibility * Improve performance, readability and handling of FollowInternalRedirects (#9889) * Improve performance, readability and handling of FollowInternalRedirects * Logger didn't like string param Passing string param to _logger.Debug wasn't happy. Changed to pass existing internalRedirectAsInt variable. Co-authored-by: Nathan Woulfe * Update casing of listview layout name * 9097 add contextual password helper (#9256) * update back-office forms * Display tip on reset password page as well * add directive for password tip * integrate directove in login screen * forgot the ng-keyup :-) * adapt tooltip directive to potential different Members and Users password settings * remove watcher Co-authored-by: Nathan Woulfe * Unbind listener Listening for splitViewRequest was only unbound if the split view editor was opened. Not cleaning up the listener caused a memory leak when changing between nodes as the spit view editor was detached but not garbage-collected * Replace icon in date picker with umb-icon component (#10040) * Replace icon in date picker with component * Adjust height of clear button * Update cypress and fix tests * Listview config icons (#10036) * Update icons to use component * Simplify markup and use disabled button * Use move cursor style on sortable handle * Add class for action column * Update setting auto focus * Increase font size of umb-panel-header-icon * Anchor noopener (#10009) * Set rel="noopener" for anchors with target="_blank" * Reverted unwanted changes to Default.cshtml * Align 'Add language' test to netcore * Add new cypress tests * Add indentation * Getting rid of the config file and implementing an appSetting instead * Implementation for IContentDashboardSettings * Cleanup * bool.Try * Taking AllowContentDashboardAccessToAllUsers prop from GlobalSettings to ContentDashboardSettings and saving AccessRulesFromConfig into a backing field * Handling multiple values per field in Examine Management * Add Root and Breadcrumbs extension methods for IPublishedContent (#9033) * Fix usage of obsolete CreatorName and WriterName properties * Add generic Root extension method * Add Breadcrumbs extension methods * Orders member type grouping of members alphabetically, matching the listing of member types. * Revert updating deprecated WriterName/CreatorName refs Changing the properties to use the extensions is a good thing (given the props are deprecated), but causes issues deep in tests. I'm reverting that change to fix the tests, and all refs to the deprecated properties should be updated in one sweep, to deal with any other test issues that might crop up. * Handle Invalid format for Upgrade check * Fixes tabbing-mode remains active after closing modal #9790 (#10074) * Allow to pass in boolean to preventEnterSubmit directive (#8639) * Pass in value to preventEnterSubmit directive * Set enabled similar to preventDefault and preventEnterSubmit directives * Update prevent enter submit value * Init value from controller * Use a different default input id prefix for umb-search-filter * Fix typo * Check for truthly value * Revert "Set enabled similar to preventDefault and preventEnterSubmit directives" This reverts commit 536ce855c4545ead82cea77b4013bf9010a8687b. * None pointer events when clicking icon * Use color variable * Fixes tabbing-mode remains active after closing modal #9790 (#10074) (cherry picked from commit c881fa9e7d08c11954e18489827f70cdafceb947) * Null check on scope and options to ensure backward compatibility (cherry picked from commit fe8cd239d2f4c528c1a8a3cf4c50e90bb43cacfc) * Fix validation of step size in integer/numeric field * 9962: Use $allowedEditors instead of allowed (#10086) * 9962: Use $allowedEditors instead of allowed * 9962: Remove redundant statement * fixes #10021 adds ng-form and val-form-manager to the documentation * Improved accessibility of link picker (#10099) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Added control ids for the link picker * Add French translation * Accessibility: Alerts the user how many results have been returned on a tree search (#10100) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Tree search details the number of search items returned * Add French translations * Updated LightInject to v6.4.0 * Remove HtmlSanitizer once more - see #9803 * Also make sure NuGet installs the correct version of the CodePages dependency * Bump version to 8.13 RC * Fixed copy preserving sort order (#10091) * Revert "Updated LightInject to v6.4.0" This reverts commit fc77252ec756cf90bb74e7fbbe6dd6d75cbdacfc. * Revert "Add copy logic to Media Picker (#9957)" This reverts commit f7c032af65cac83182782c758a3ab79c86b92e70. * Reintroduce old constructor to make non-breaking * Update cypress test to make macros in the grid work again * Attributes could be multiple items, test specifically if `Directory` is an attribute * Accessibility: Adding label fors and control ids for the macro picker (#10101) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Added support for label fors for the macro picker and also gave the ,acro search box a title * Now displays a count of the matching macros returned. Please note the language file amends shared with #10100 * Removed src-only class for the display of the count of messages * Updating typo * Removed top-margin from switcher icon * Allow KeepAlive controller Ping method to be requested by non local requests (#10126) * Allow KeepAlive controller Ping method to be requested by non local requests and accept head requests * removed unused references * fix csproj Co-authored-by: Mole Co-authored-by: Sebastiaan Janssen Co-authored-by: Justin Shearer Co-authored-by: Bjarke Berg Co-authored-by: Callum Whyte Co-authored-by: Shannon Co-authored-by: Elitsa Marinovska Co-authored-by: patrickdemooij9 Co-authored-by: Bjarne Fyrstenborg Co-authored-by: Michael Latouche Co-authored-by: Nathan Woulfe Co-authored-by: Markus Johansson Co-authored-by: Jeavon Leopold Co-authored-by: Benjamin Carleski Co-authored-by: Owain Williams Co-authored-by: Jesper Löfgren Co-authored-by: Martin Bentancour Co-authored-by: Ronald Barendse Co-authored-by: Andy Butland Co-authored-by: BeardinaSuit Co-authored-by: Mads Rasmussen Co-authored-by: Rachel Breeze Co-authored-by: Dave de Moel Co-authored-by: ric <60885685+ricbrady@users.noreply.github.com> Co-authored-by: Carole Rennie Logan Co-authored-by: Dennis Öhman --- build/NuSpecs/UmbracoCms.Web.nuspec | 2 +- src/SolutionInfo.cs | 4 +- src/Umbraco.Core/Composing/TypeLoader.cs | 117 +++++-- src/Umbraco.Core/ConfigsExtensions.cs | 4 +- .../Configuration/GlobalSettings.cs | 41 ++- src/Umbraco.Core/Constants-AppSettings.cs | 5 + src/Umbraco.Core/Constants-Security.cs | 2 +- src/Umbraco.Core/Constants-SvgSanitizer.cs | 23 -- .../Dashboards/ContentDashboardSettings.cs | 24 ++ .../Dashboards/IContentDashboardSettings.cs | 14 + src/Umbraco.Core/IO/IOHelper.cs | 5 +- src/Umbraco.Core/Mapping/UmbracoMapper.cs | 38 +- .../Implement/UpgradeCheckRepository.cs | 5 + src/Umbraco.Core/Runtime/MainDom.cs | 9 +- src/Umbraco.Core/Runtime/SqlMainDomLock.cs | 108 +++--- src/Umbraco.Core/Scoping/Scope.cs | 331 +++++++++--------- src/Umbraco.Core/Scoping/ScopeProvider.cs | 4 + .../Services/Implement/ContentService.cs | 10 +- src/Umbraco.Core/StringExtensions.cs | 41 ++- src/Umbraco.Core/Umbraco.Core.csproj | 3 +- src/Umbraco.Examine/UmbracoExamineIndex.cs | 21 +- .../Compose/ModelsBuilderComposer.cs | 35 +- .../cypress/integration/Content/content.ts | 204 ++++++++++- .../cypress/integration/Settings/languages.ts | 5 +- .../cypress/integration/Settings/templates.ts | 8 +- .../integration/Tour/backofficeTour.ts | 2 +- src/Umbraco.Tests.AcceptanceTest/package.json | 4 +- src/Umbraco.Tests/Mapping/MappingTests.cs | 37 +- src/Umbraco.Tests/Scoping/ScopeTests.cs | 114 ++++++ src/Umbraco.Tests/Scoping/ScopeUnitTests.cs | 294 +++++++++++++--- .../Services/ContentServiceTests.cs | 26 ++ .../Importing/StandardMvc-Package.xml | 6 +- .../Services/PerformanceTests.cs | 4 +- .../Services/ThreadSafetyServiceTest.cs | 22 +- .../Testing/TestingTests/MockTests.cs | 18 +- .../application/umblogin.directive.js | 6 +- .../application/umbpasswordtip.directive.js | 71 ++++ .../umbvariantcontenteditors.directive.js | 3 + .../forms/prevententersubmit.directive.js | 2 +- .../forms/umbfocuslock.directive.js | 4 - .../forms/umbsearchfilter.directive.js | 10 +- .../components/tabs/umbtabsnav.directive.js | 37 +- .../tree/umbtreesearchbox.directive.js | 1 + .../users/changepassword.directive.js | 9 + .../mocks/services/localization.mocks.js | 6 +- .../common/resources/contenttype.resource.js | 8 +- .../common/resources/mediatype.resource.js | 4 +- .../common/resources/membertype.resource.js | 25 +- .../common/services/listviewhelper.service.js | 4 +- .../services/umbrequesthelper.service.js | 4 +- .../src/installer/installer.service.js | 6 +- .../src/installer/steps/upgrade.html | 2 +- .../src/less/application/umb-outline.less | 1 + .../src/less/buttons.less | 17 +- .../editor/umb-variant-switcher.less | 1 - .../less/components/umb-search-filter.less | 3 +- src/Umbraco.Web.UI.Client/src/less/forms.less | 17 +- src/Umbraco.Web.UI.Client/src/less/main.less | 4 + src/Umbraco.Web.UI.Client/src/less/panel.less | 1 + .../src/less/property-editors.less | 17 +- .../linkpicker/linkpicker.html | 67 ++-- .../macropicker/macropicker.controller.js | 31 +- .../macropicker/macropicker.html | 24 +- .../userpicker/userpicker.controller.js | 16 +- .../components/application/umb-login.html | 3 +- .../editor/umb-editor-content-header.html | 1 + .../components/forms/umb-search-filter.html | 31 +- .../components/tree/umb-tree-search-box.html | 1 + .../tree/umb-tree-search-results.html | 49 ++- .../components/users/change-password.html | 4 +- .../dashboard/media/mediadashboardvideos.html | 2 +- .../settings/examinemanagementresults.html | 4 +- .../src/views/documenttypes/copy.html | 2 +- .../src/views/documenttypes/export.html | 4 +- .../documenttypes/importdocumenttype.html | 6 +- .../src/views/documenttypes/move.html | 2 +- .../src/views/mediatypes/copy.html | 2 +- .../src/views/mediatypes/move.html | 2 +- .../src/views/membertypes/copy.controller.js | 61 ++++ .../src/views/membertypes/copy.html | 53 +++ .../changepassword.controller.js | 3 + .../datepicker/datepicker.html | 10 +- .../views/propertyeditors/email/email.html | 5 +- .../grid/editors/media.controller.js | 40 +-- .../propertyeditors/grid/grid.controller.js | 4 +- .../propertyeditors/integer/integer.html | 9 +- .../listview/includeproperties.prevalues.html | 4 +- .../listview/layouts.prevalues.html | 18 +- .../propertyeditors/listview/listview.html | 1 - .../userpicker/userpicker.controller.js | 29 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 5 +- .../Umbraco/Views/Default.cshtml | 7 +- src/Umbraco.Web.UI/Umbraco/config/lang/cs.xml | 10 +- src/Umbraco.Web.UI/Umbraco/config/lang/cy.xml | 12 +- src/Umbraco.Web.UI/Umbraco/config/lang/da.xml | 21 +- src/Umbraco.Web.UI/Umbraco/config/lang/de.xml | 12 +- src/Umbraco.Web.UI/Umbraco/config/lang/en.xml | 31 +- .../Umbraco/config/lang/en_us.xml | 31 +- src/Umbraco.Web.UI/Umbraco/config/lang/es.xml | 10 +- src/Umbraco.Web.UI/Umbraco/config/lang/fr.xml | 31 +- src/Umbraco.Web.UI/Umbraco/config/lang/he.xml | 4 +- src/Umbraco.Web.UI/Umbraco/config/lang/it.xml | 4 +- src/Umbraco.Web.UI/Umbraco/config/lang/ja.xml | 4 +- src/Umbraco.Web.UI/Umbraco/config/lang/ko.xml | 4 +- src/Umbraco.Web.UI/Umbraco/config/lang/nb.xml | 4 +- src/Umbraco.Web.UI/Umbraco/config/lang/pl.xml | 6 +- src/Umbraco.Web.UI/Umbraco/config/lang/pt.xml | 4 +- src/Umbraco.Web.UI/Umbraco/config/lang/ru.xml | 14 +- src/Umbraco.Web.UI/Umbraco/config/lang/sv.xml | 6 +- src/Umbraco.Web.UI/Umbraco/config/lang/tr.xml | 24 +- src/Umbraco.Web.UI/Umbraco/config/lang/zh.xml | 6 +- .../Umbraco/config/lang/zh_tw.xml | 6 +- .../config/splashes/noNodes.aspx | 4 +- src/Umbraco.Web.UI/web.Template.config | 1 + .../Dashboards/ContentDashboard.cs | 59 +++- .../Editors/BackOfficeServerVariables.cs | 6 +- .../Binders/ContentModelBinderHelper.cs | 9 +- src/Umbraco.Web/Editors/CodeFileController.cs | 5 +- .../Editors/DashboardController.cs | 4 +- .../Editors/ExamineManagementController.cs | 13 +- .../Editors/KeepAliveController.cs | 4 +- .../Editors/MemberTypeController.cs | 12 + .../Checks/Security/ExcessiveHeadersCheck.cs | 4 +- .../HealthCheck/Checks/Security/HttpsCheck.cs | 21 +- .../Models/ContentEditing/SearchResult.cs | 2 +- .../PropertyEditors/GridPropertyEditor.cs | 4 +- .../PropertyEditors/ListViewConfiguration.cs | 2 +- src/Umbraco.Web/PublishedContentExtensions.cs | 90 ++++- src/Umbraco.Web/PublishedPropertyExtension.cs | 35 +- src/Umbraco.Web/Routing/PublishedRouter.cs | 62 ++-- src/Umbraco.Web/Runtime/WebInitialComposer.cs | 10 - .../Scheduling/BackgroundTaskRunner.cs | 19 +- src/Umbraco.Web/Services/IconService.cs | 8 +- src/Umbraco.Web/Trees/MemberTreeController.cs | 10 +- .../MemberTypeAndGroupTreeControllerBase.cs | 12 +- .../Trees/MemberTypeTreeController.cs | 1 + src/Umbraco.Web/Umbraco.Web.csproj | 3 - 137 files changed, 2137 insertions(+), 860 deletions(-) delete mode 100644 src/Umbraco.Core/Constants-SvgSanitizer.cs create mode 100644 src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs create mode 100644 src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs create mode 100644 src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbpasswordtip.directive.js create mode 100644 src/Umbraco.Web.UI.Client/src/views/membertypes/copy.controller.js create mode 100644 src/Umbraco.Web.UI.Client/src/views/membertypes/copy.html diff --git a/build/NuSpecs/UmbracoCms.Web.nuspec b/build/NuSpecs/UmbracoCms.Web.nuspec index de634b4884..46165df087 100644 --- a/build/NuSpecs/UmbracoCms.Web.nuspec +++ b/build/NuSpecs/UmbracoCms.Web.nuspec @@ -42,7 +42,7 @@ - + diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index c064920d34..2a7386cb45 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.12.0")] -[assembly: AssemblyInformationalVersion("8.12.0")] +[assembly: AssemblyFileVersion("8.13.0")] +[assembly: AssemblyInformationalVersion("8.13.0-rc")] diff --git a/src/Umbraco.Core/Composing/TypeLoader.cs b/src/Umbraco.Core/Composing/TypeLoader.cs index f5c75ff607..bee6436cd6 100644 --- a/src/Umbraco.Core/Composing/TypeLoader.cs +++ b/src/Umbraco.Core/Composing/TypeLoader.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -45,7 +45,7 @@ namespace Umbraco.Core.Composing private IEnumerable _assemblies; private bool _reportedChange; private readonly string _localTempPath; - private string _fileBasePath; + private readonly Lazy _fileBasePath; /// /// Initializes a new instance of the class. @@ -70,6 +70,8 @@ namespace Umbraco.Core.Composing _localTempPath = localTempPath; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _fileBasePath = new Lazy(GetFileBasePath); + if (detectChanges) { //first check if the cached hash is string.Empty, if it is then we need @@ -160,7 +162,8 @@ namespace Umbraco.Core.Composing return _cachedAssembliesHash; var typesHashFilePath = GetTypesHashFilePath(); - if (!File.Exists(typesHashFilePath)) return string.Empty; + if (!File.Exists(typesHashFilePath)) + return string.Empty; var hash = File.ReadAllText(typesHashFilePath, Encoding.UTF8); @@ -339,7 +342,9 @@ namespace Umbraco.Core.Composing var typesListFilePath = GetTypesListFilePath(); if (File.Exists(typesListFilePath) == false) + { return cache; + } using (var stream = GetFileStream(typesListFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, ListFileOpenReadTimeout)) using (var reader = new StreamReader(stream)) @@ -347,11 +352,21 @@ namespace Umbraco.Core.Composing while (true) { var baseType = reader.ReadLine(); - if (baseType == null) return cache; // exit - if (baseType.StartsWith("<")) break; // old xml + if (baseType == null) + { + return cache; // exit + } + + if (baseType.StartsWith("<")) + { + break; // old xml + } var attributeType = reader.ReadLine(); - if (attributeType == null) break; + if (attributeType == null) + { + break; + } var types = new List(); while (true) @@ -370,7 +385,10 @@ namespace Umbraco.Core.Composing types.Add(type); } - if (types == null) break; + if (types == null) + { + break; + } } } @@ -379,28 +397,31 @@ namespace Umbraco.Core.Composing } // internal for tests - internal string GetTypesListFilePath() => GetFileBasePath() + ".list"; + internal string GetTypesListFilePath() => _fileBasePath.Value + ".list"; - private string GetTypesHashFilePath() => GetFileBasePath() + ".hash"; + private string GetTypesHashFilePath() => _fileBasePath.Value + ".hash"; + /// + /// Used to produce the Lazy value of _fileBasePath + /// + /// private string GetFileBasePath() { - lock (_locko) + var fileBasePath = Path.Combine(_localTempPath, "TypesCache", "umbraco-types." + NetworkHelper.FileSafeMachineName); + + // ensure that the folder exists + var directory = Path.GetDirectoryName(fileBasePath); + if (directory == null) { - if (_fileBasePath != null) - return _fileBasePath; - - _fileBasePath = Path.Combine(_localTempPath, "TypesCache", "umbraco-types." + NetworkHelper.FileSafeMachineName); - - // ensure that the folder exists - var directory = Path.GetDirectoryName(_fileBasePath); - if (directory == null) - throw new InvalidOperationException($"Could not determine folder for path \"{_fileBasePath}\"."); - if (Directory.Exists(directory) == false) - Directory.CreateDirectory(directory); - - return _fileBasePath; + throw new InvalidOperationException($"Could not determine folder for path \"{fileBasePath}\"."); } + + if (Directory.Exists(directory) == false) + { + Directory.CreateDirectory(directory); + } + + return fileBasePath; } // internal for tests @@ -416,7 +437,10 @@ namespace Umbraco.Core.Composing writer.WriteLine(typeList.BaseType == null ? string.Empty : typeList.BaseType.FullName); writer.WriteLine(typeList.AttributeType == null ? string.Empty : typeList.AttributeType.FullName); foreach (var type in typeList.Types) + { writer.WriteLine(type.AssemblyQualifiedName); + } + writer.WriteLine(); } } @@ -434,16 +458,22 @@ namespace Umbraco.Core.Composing WriteCache(); } catch { /* bah - just don't die */ } - if (!_timing) _timer = null; + if (!_timing) + _timer = null; } } lock (_timerLock) { if (_timer == null) + { _timer = new Timer(TimerRelease, null, ListFileWriteThrottle, Timeout.Infinite); + } else + { _timer.Change(ListFileWriteThrottle, Timeout.Infinite); + } + _timing = true; } } @@ -476,7 +506,9 @@ namespace Umbraco.Core.Composing catch { if (--attempts == 0) + { throw; + } _logger.Debug("Attempted to get filestream for file {Path} failed, {NumberOfAttempts} attempts left, pausing for {PauseMilliseconds} milliseconds", path, attempts, pauseMilliseconds); Thread.Sleep(pauseMilliseconds); @@ -543,7 +575,8 @@ namespace Umbraco.Core.Composing /// attributeTypes public IEnumerable GetAssemblyAttributes(params Type[] attributeTypes) { - if (attributeTypes == null) throw new ArgumentNullException(nameof(attributeTypes)); + if (attributeTypes == null) + throw new ArgumentNullException(nameof(attributeTypes)); return AssembliesToScan.SelectMany(a => attributeTypes.SelectMany(at => a.GetCustomAttributes(at))).ToList(); } @@ -563,7 +596,9 @@ namespace Umbraco.Core.Composing public IEnumerable GetTypes(bool cache = true, IEnumerable specificAssemblies = null) { if (_logger == null) + { throw new InvalidOperationException("Cannot get types from a test/blank type loader."); + } // do not cache anything from specific assemblies cache &= specificAssemblies == null; @@ -583,7 +618,7 @@ namespace Umbraco.Core.Composing // get IDiscoverable and always cache var discovered = GetTypesInternal( - typeof (IDiscoverable), null, + typeof(IDiscoverable), null, () => TypeFinder.FindClassesOfType(AssembliesToScan), "scanning assemblies", true); @@ -594,9 +629,9 @@ namespace Umbraco.Core.Composing // filter the cached discovered types (and maybe cache the result) return GetTypesInternal( - typeof (T), null, + typeof(T), null, () => discovered - .Where(x => typeof (T).IsAssignableFrom(x)), + .Where(x => typeof(T).IsAssignableFrom(x)), "filtering IDiscoverable", cache); } @@ -614,7 +649,9 @@ namespace Umbraco.Core.Composing where TAttribute : Attribute { if (_logger == null) + { throw new InvalidOperationException("Cannot get types from a test/blank type loader."); + } // do not cache anything from specific assemblies cache &= specificAssemblies == null; @@ -633,7 +670,7 @@ namespace Umbraco.Core.Composing // get IDiscoverable and always cache var discovered = GetTypesInternal( - typeof (IDiscoverable), null, + typeof(IDiscoverable), null, () => TypeFinder.FindClassesOfType(AssembliesToScan), "scanning assemblies", true); @@ -644,7 +681,7 @@ namespace Umbraco.Core.Composing // filter the cached discovered types (and maybe cache the result) return GetTypesInternal( - typeof (T), typeof (TAttribute), + typeof(T), typeof(TAttribute), () => discovered .Where(x => typeof(T).IsAssignableFrom(x)) .Where(x => x.GetCustomAttributes(false).Any()), @@ -664,7 +701,9 @@ namespace Umbraco.Core.Composing where TAttribute : Attribute { if (_logger == null) + { throw new InvalidOperationException("Cannot get types from a test/blank type loader."); + } // do not cache anything from specific assemblies cache &= specificAssemblies == null; @@ -673,7 +712,7 @@ namespace Umbraco.Core.Composing _logger.Debug("Running a full, non-cached, scan for types / attribute {AttributeName} (slow).", typeof(TAttribute).FullName); return GetTypesInternal( - typeof (object), typeof (TAttribute), + typeof(object), typeof(TAttribute), () => TypeFinder.FindClassesWithAttribute(specificAssemblies ?? AssembliesToScan), "scanning assemblies", cache); @@ -693,12 +732,14 @@ namespace Umbraco.Core.Composing var name = GetName(baseType, attributeType); lock (_locko) - using (_logger.DebugDuration( + { + using (_logger.DebugDuration( "Getting " + name, "Got " + name)) // cannot contain typesFound.Count as it's evaluated before the find - { - // get within a lock & timer - return GetTypesInternalLocked(baseType, attributeType, finder, action, cache); + { + // get within a lock & timer + return GetTypesInternalLocked(baseType, attributeType, finder, action, cache); + } } } @@ -720,7 +761,9 @@ namespace Umbraco.Core.Composing var listKey = new CompositeTypeTypeKey(baseType ?? tobject, attributeType ?? tobject); TypeList typeList = null; if (cache) + { _types.TryGetValue(listKey, out typeList); // else null + } // if caching and found, return if (typeList != null) @@ -795,7 +838,9 @@ namespace Umbraco.Core.Composing _logger.Debug("Getting {TypeName}: " + action + ".", GetName(baseType, attributeType)); foreach (var t in finder()) + { typeList.Add(t); + } } // if we are to cache the results, do so @@ -807,7 +852,9 @@ namespace Umbraco.Core.Composing _types[listKey] = typeList; //if we are scanning then update the cache file if (scan) + { UpdateCache(); + } } _logger.Debug("Got {TypeName}, caching ({CacheType}).", GetName(baseType, attributeType), added.ToString().ToLowerInvariant()); diff --git a/src/Umbraco.Core/ConfigsExtensions.cs b/src/Umbraco.Core/ConfigsExtensions.cs index d1672c6c7f..10594fc970 100644 --- a/src/Umbraco.Core/ConfigsExtensions.cs +++ b/src/Umbraco.Core/ConfigsExtensions.cs @@ -1,10 +1,10 @@ using System.IO; using Umbraco.Core.Cache; -using Umbraco.Core.Composing; using Umbraco.Core.Configuration; using Umbraco.Core.Configuration.Grid; using Umbraco.Core.Configuration.HealthChecks; using Umbraco.Core.Configuration.UmbracoSettings; +using Umbraco.Core.Dashboards; using Umbraco.Core.IO; using Umbraco.Core.Logging; using Umbraco.Core.Manifest; @@ -48,6 +48,8 @@ namespace Umbraco.Core configDir, factory.GetInstance(), factory.GetInstance().Debug)); + + configs.Add(() => new ContentDashboardSettings()); } } } diff --git a/src/Umbraco.Core/Configuration/GlobalSettings.cs b/src/Umbraco.Core/Configuration/GlobalSettings.cs index b7dce21285..c844abe75e 100644 --- a/src/Umbraco.Core/Configuration/GlobalSettings.cs +++ b/src/Umbraco.Core/Configuration/GlobalSettings.cs @@ -395,7 +395,6 @@ namespace Umbraco.Core.Configuration } } - /// /// An int value representing the time in milliseconds to lock the database for a write operation /// @@ -409,26 +408,34 @@ namespace Umbraco.Core.Configuration { if (_sqlWriteLockTimeOut != default) return _sqlWriteLockTimeOut; - var timeOut = 5000; // 5 seconds - var appSettingSqlWriteLockTimeOut = ConfigurationManager.AppSettings[Constants.AppSettings.SqlWriteLockTimeOut]; - if(int.TryParse(appSettingSqlWriteLockTimeOut, out var configuredTimeOut)) - { - // Only apply this setting if it's not excessively high or low - const int minimumTimeOut = 100; - const int maximumTimeOut = 20000; - if (configuredTimeOut >= minimumTimeOut && configuredTimeOut <= maximumTimeOut) // between 0.1 and 20 seconds - { - timeOut = configuredTimeOut; - } - else - { - Current.Logger.Warn($"The `{Constants.AppSettings.SqlWriteLockTimeOut}` setting in web.config is not between the minimum of {minimumTimeOut} ms and maximum of {maximumTimeOut} ms, defaulting back to {timeOut}"); - } - } + var timeOut = GetSqlWriteLockTimeoutFromConfigFile(Current.Logger); _sqlWriteLockTimeOut = timeOut; return _sqlWriteLockTimeOut; } } + + internal static int GetSqlWriteLockTimeoutFromConfigFile(ILogger logger) + { + var timeOut = 5000; // 5 seconds + var appSettingSqlWriteLockTimeOut = ConfigurationManager.AppSettings[Constants.AppSettings.SqlWriteLockTimeOut]; + if (int.TryParse(appSettingSqlWriteLockTimeOut, out var configuredTimeOut)) + { + // Only apply this setting if it's not excessively high or low + const int minimumTimeOut = 100; + const int maximumTimeOut = 20000; + if (configuredTimeOut >= minimumTimeOut && configuredTimeOut <= maximumTimeOut) // between 0.1 and 20 seconds + { + timeOut = configuredTimeOut; + } + else + { + logger.Warn( + $"The `{Constants.AppSettings.SqlWriteLockTimeOut}` setting in web.config is not between the minimum of {minimumTimeOut} ms and maximum of {maximumTimeOut} ms, defaulting back to {timeOut}"); + } + } + + return timeOut; + } } } diff --git a/src/Umbraco.Core/Constants-AppSettings.cs b/src/Umbraco.Core/Constants-AppSettings.cs index f04f0e1f5f..1f096ab9f9 100644 --- a/src/Umbraco.Core/Constants-AppSettings.cs +++ b/src/Umbraco.Core/Constants-AppSettings.cs @@ -110,6 +110,11 @@ namespace Umbraco.Core /// public const string UseHttps = "Umbraco.Core.UseHttps"; + /// + /// A true/false value indicating whether the content dashboard should be visible for all user groups. + /// + public const string AllowContentDashboardAccessToAllUsers = "Umbraco.Core.AllowContentDashboardAccessToAllUsers"; + /// /// TODO: FILL ME IN /// diff --git a/src/Umbraco.Core/Constants-Security.cs b/src/Umbraco.Core/Constants-Security.cs index f900288ef5..2b6244debb 100644 --- a/src/Umbraco.Core/Constants-Security.cs +++ b/src/Umbraco.Core/Constants-Security.cs @@ -25,7 +25,7 @@ namespace Umbraco.Core /// /// The name of the 'unknown' user. /// - public const string UnknownUserName = "SYTEM"; + public const string UnknownUserName = "SYSTEM"; public const string AdminGroupAlias = "admin"; public const string EditorGroupAlias = "editor"; diff --git a/src/Umbraco.Core/Constants-SvgSanitizer.cs b/src/Umbraco.Core/Constants-SvgSanitizer.cs deleted file mode 100644 index c92b9f56c7..0000000000 --- a/src/Umbraco.Core/Constants-SvgSanitizer.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Collections.Generic; - -namespace Umbraco.Core -{ - public static partial class Constants - { - /// - /// Defines the alias identifiers for Umbraco's core application sections. - /// - public static class SvgSanitizer - { - /// - /// Allowlist for SVG attributes. - /// - public static readonly IList Attributes = new [] { "accent-height", "accumulate", "additive", "alignment-baseline", "allowReorder", "alphabetic", "amplitude", "arabic-form", "ascent", "attributeName", "attributeType", "autoReverse", "azimuth", "baseFrequency", "baseline-shift", "baseProfile", "bbox", "begin", "bias", "by", "calcMode", "cap-height", "class", "clip", "clipPathUnits", "clip-path", "clip-rule", "color", "color-interpolation", "color-interpolation-filters", "color-profile", "color-rendering", "contentScriptType", "contentStyleType", "cursor", "cx", "cy", "d", "decelerate", "descent", "diffuseConstant", "direction", "display", "divisor", "dominant-baseline", "dur", "dx", "dy", "edgeMode", "elevation", "enable-background", "end", "exponent", "externalResourcesRequired", "Section", "fill", "fill-opacity", "fill-rule", "filter", "filterRes", "filterUnits", "flood-color", "flood-opacity", "font-family", "font-size", "font-size-adjust", "font-stretch", "font-style", "font-variant", "font-weight", "format", "from", "fr", "fx", "fy", "g1", "g2", "glyph-name", "glyph-orientation-horizontal", "glyph-orientation-vertical", "glyphRef", "gradientTransform", "gradientUnits", "hanging", "height", "href", "hreflang", "horiz-adv-x", "horiz-origin-x", "ISection", "id", "ideographic", "image-rendering", "in", "in2", "intercept", "k", "k1", "k2", "k3", "k4", "kernelMatrix", "kernelUnitLength", "kerning", "keyPoints", "keySplines", "keyTimes", "lang", "lengthAdjust", "letter-spacing", "lighting-color", "limitingConeAngle", "local", "MSection", "marker-end", "marker-mid", "marker-start", "markerHeight", "markerUnits", "markerWidth", "mask", "maskContentUnits", "maskUnits", "mathematical", "max", "media", "method", "min", "mode", "NSection", "name", "numOctaves", "offset", "opacity", "operator", "order", "orient", "orientation", "origin", "overflow", "overline-position", "overline-thickness", "panose-1", "paint-order", "path", "pathLength", "patternContentUnits", "patternTransform", "patternUnits", "ping", "pointer-events", "points", "pointsAtX", "pointsAtY", "pointsAtZ", "preserveAlpha", "preserveAspectRatio", "primitiveUnits", "r", "radius", "referrerPolicy", "refX", "refY", "rel", "rendering-intent", "repeatCount", "repeatDur", "requiredExtensions", "requiredFeatures", "restart", "result", "rotate", "rx", "ry", "scale", "seed", "shape-rendering", "slope", "spacing", "specularConstant", "specularExponent", "speed", "spreadMethod", "startOffset", "stdDeviation", "stemh", "stemv", "stitchTiles", "stop-color", "stop-opacity", "strikethrough-position", "strikethrough-thickness", "string", "stroke", "stroke-dasharray", "stroke-dashoffset", "stroke-linecap", "stroke-linejoin", "stroke-miterlimit", "stroke-opacity", "stroke-width", "style", "surfaceScale", "systemLanguage", "tabindex", "tableValues", "target", "targetX", "targetY", "text-anchor", "text-decoration", "text-rendering", "textLength", "to", "transform", "type", "u1", "u2", "underline-position", "underline-thickness", "unicode", "unicode-bidi", "unicode-range", "units-per-em", "v-alphabetic", "v-hanging", "v-ideographic", "v-mathematical", "values", "vector-effect", "version", "vert-adv-y", "vert-origin-x", "vert-origin-y", "viewBox", "viewTarget", "visibility", "width", "widths", "word-spacing", "writing-mode", "x", "x-height", "x1", "x2", "xChannelSelector", "xlink:actuate", "xlink:arcrole", "xlink:href", "xlink:role", "xlink:show", "xlink:title", "xlink:type", "xml:base", "xml:lang", "xml:space", "y", "y1", "y2", "yChannelSelector", "z", "zoomAndPan" }; - - /// - /// Allowlist for SVG tabs. - /// - public static readonly IList Tags = new [] { "a", "altGlyph", "altGlyphDef", "altGlyphItem", "animate", "animateColor", "animateMotion", "animateTransform", "circle", "clipPath", "color-profile", "cursor", "defs", "desc", "discard", "ellipse", "feBlend", "feColorMatrix", "feComponentTransfer", "feComposite", "feConvolveMatrix", "feDiffuseLighting", "feDisplacementMap", "feDistantLight", "feDropShadow", "feFlood", "feFuncA", "feFuncB", "feFuncG", "feFuncR", "feGaussianBlur", "feImage", "feMerge", "feMergeNode", "feMorphology", "feOffset", "fePointLight", "feSpecularLighting", "feSpotLight", "feTile", "feTurbulence", "filter", "font", "font-face", "font-face-format", "font-face-name", "font-face-src", "font-face-uri", "foreignObject", "g", "glyph", "glyphRef", "hatch", "hatchpath", "hkern", "image", "line", "linearGradient", "marker", "mask", "mesh", "meshgradient", "meshpatch", "meshrow", "metadata", "missing-glyph", "mpath", "path", "pattern", "polygon", "polyline", "radialGradient", "rect", "set", "solidcolor", "stop", "svg", "switch", "symbol", "text", "textPath", "title", "tref", "tspan", "unknown", "use", "view", "vkern" }; - } - } -} diff --git a/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs b/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs new file mode 100644 index 0000000000..f8fb5c7b06 --- /dev/null +++ b/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs @@ -0,0 +1,24 @@ +using System.Configuration; + +namespace Umbraco.Core.Dashboards +{ + public class ContentDashboardSettings: IContentDashboardSettings + { + + /// + /// Gets a value indicating whether the content dashboard should be available to all users. + /// + /// + /// true if the dashboard is visible for all user groups; otherwise, false + /// and the default access rules for that dashboard will be in use. + /// + public bool AllowContentDashboardAccessToAllUsers + { + get + { + bool.TryParse(ConfigurationManager.AppSettings[Constants.AppSettings.AllowContentDashboardAccessToAllUsers], out var value); + return value; + } + } + } +} diff --git a/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs b/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs new file mode 100644 index 0000000000..862a28b90e --- /dev/null +++ b/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs @@ -0,0 +1,14 @@ +namespace Umbraco.Core.Dashboards +{ + public interface IContentDashboardSettings + { + /// + /// Gets a value indicating whether the content dashboard should be available to all users. + /// + /// + /// true if the dashboard is visible for all user groups; otherwise, false + /// and the default access rules for that dashboard will be in use. + /// + bool AllowContentDashboardAccessToAllUsers { get; } + } +} diff --git a/src/Umbraco.Core/IO/IOHelper.cs b/src/Umbraco.Core/IO/IOHelper.cs index 8661f73fb1..69ce50de9c 100644 --- a/src/Umbraco.Core/IO/IOHelper.cs +++ b/src/Umbraco.Core/IO/IOHelper.cs @@ -81,6 +81,7 @@ namespace Umbraco.Core.IO public static string MapPath(string path, bool useHttpContext) { if (path == null) throw new ArgumentNullException("path"); + useHttpContext = useHttpContext && IsHosted; // Check if the path is already mapped @@ -89,10 +90,8 @@ namespace Umbraco.Core.IO { return path; } - // Check that we even have an HttpContext! otherwise things will fail anyways - // http://umbraco.codeplex.com/workitem/30946 - if (useHttpContext && HttpContext.Current != null) + if (useHttpContext) { //string retval; if (String.IsNullOrEmpty(path) == false && (path.StartsWith("~") || path.StartsWith(SystemDirectories.Root))) diff --git a/src/Umbraco.Core/Mapping/UmbracoMapper.cs b/src/Umbraco.Core/Mapping/UmbracoMapper.cs index e62825101c..36e3f9eab9 100644 --- a/src/Umbraco.Core/Mapping/UmbracoMapper.cs +++ b/src/Umbraco.Core/Mapping/UmbracoMapper.cs @@ -3,7 +3,9 @@ using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using Umbraco.Core.Composing; using Umbraco.Core.Exceptions; +using Umbraco.Core.Scoping; namespace Umbraco.Core.Mapping { @@ -42,16 +44,29 @@ namespace Umbraco.Core.Mapping private readonly ConcurrentDictionary>> _maps = new ConcurrentDictionary>>(); + private readonly IScopeProvider _scopeProvider; + /// /// Initializes a new instance of the class. /// /// - public UmbracoMapper(MapDefinitionCollection profiles) + /// + public UmbracoMapper(MapDefinitionCollection profiles, IScopeProvider scopeProvider) { + _scopeProvider = scopeProvider; + foreach (var profile in profiles) profile.DefineMaps(this); } + /// + /// Initializes a new instance of the class. + /// + /// + [Obsolete("This constructor is no longer used and will be removed in future versions, use the other constructor instead")] + public UmbracoMapper(MapDefinitionCollection profiles) : this(profiles, Current.ScopeProvider) + {} + #region Define private static TTarget ThrowCtor(TSource source, MapperContext context) @@ -203,7 +218,10 @@ namespace Umbraco.Core.Mapping if (ctor != null && map != null) { var target = ctor(source, context); - map(source, target, context); + using (var scope = _scopeProvider.CreateScope(autoComplete: true)) + { + map(source, target, context); + } return (TTarget)target; } @@ -248,11 +266,14 @@ namespace Umbraco.Core.Mapping { var targetList = (IList)Activator.CreateInstance(typeof(List<>).MakeGenericType(targetGenericArg)); - foreach (var sourceItem in source) + using (var scope = _scopeProvider.CreateScope(autoComplete: true)) { - var targetItem = ctor(sourceItem, context); - map(sourceItem, targetItem, context); - targetList.Add(targetItem); + foreach (var sourceItem in source) + { + var targetItem = ctor(sourceItem, context); + map(sourceItem, targetItem, context); + targetList.Add(targetItem); + } } object target = targetList; @@ -315,7 +336,10 @@ namespace Umbraco.Core.Mapping // if there is a direct map, map if (map != null) { - map(source, target, context); + using (var scope = _scopeProvider.CreateScope(autoComplete: true)) + { + map(source, target, context); + } return target; } diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/UpgradeCheckRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/UpgradeCheckRepository.cs index 365e8ba481..95f699d952 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/UpgradeCheckRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/UpgradeCheckRepository.cs @@ -24,6 +24,11 @@ namespace Umbraco.Core.Persistence.Repositories.Implement return result ?? new UpgradeResult("None", "", ""); } + catch (UnsupportedMediaTypeException) + { + // this occurs if the server for Our is up but doesn't return a valid result (ex. content type) + return new UpgradeResult("None", "", ""); + } catch (HttpRequestException) { // this occurs if the server for Our is down or cannot be reached diff --git a/src/Umbraco.Core/Runtime/MainDom.cs b/src/Umbraco.Core/Runtime/MainDom.cs index d930841289..d784560f2c 100644 --- a/src/Umbraco.Core/Runtime/MainDom.cs +++ b/src/Umbraco.Core/Runtime/MainDom.cs @@ -179,7 +179,14 @@ namespace Umbraco.Core.Runtime _listenTask = _mainDomLock.ListenAsync(); _listenCompleteTask = _listenTask.ContinueWith(t => { - _logger.Debug("Listening task completed with {TaskStatus}", _listenTask.Status); + if (_listenTask.Exception != null) + { + _logger.Warn("Listening task completed with {TaskStatus}, Exception: {Exception}", _listenTask.Status, _listenTask.Exception); + } + else + { + _logger.Debug("Listening task completed with {TaskStatus}", _listenTask.Status); + } OnSignal("signal"); }, TaskScheduler.Default); // Must explicitly specify this, see https://blog.stephencleary.com/2013/10/continuewith-is-dangerous-too.html diff --git a/src/Umbraco.Core/Runtime/SqlMainDomLock.cs b/src/Umbraco.Core/Runtime/SqlMainDomLock.cs index d98d62cb20..12359c96d1 100644 --- a/src/Umbraco.Core/Runtime/SqlMainDomLock.cs +++ b/src/Umbraco.Core/Runtime/SqlMainDomLock.cs @@ -1,5 +1,6 @@ using NPoco; using System; +using System.Configuration; using System.Data; using System.Data.SqlClient; using System.Diagnostics; @@ -7,6 +8,7 @@ using System.Linq; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using Umbraco.Core.Configuration; using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Dtos; @@ -18,6 +20,7 @@ namespace Umbraco.Core.Runtime { internal class SqlMainDomLock : IMainDomLock { + private readonly TimeSpan _lockTimeout; private string _lockId; private const string MainDomKeyPrefix = "Umbraco.Core.Runtime.SqlMainDom"; private const string UpdatedSuffix = "_updated"; @@ -40,6 +43,8 @@ namespace Umbraco.Core.Runtime Constants.System.UmbracoConnectionName, _logger, new Lazy(() => new MapperCollection(Enumerable.Empty()))); + + _lockTimeout = TimeSpan.FromMilliseconds(GlobalSettings.GetSqlWriteLockTimeoutFromConfigFile(logger)); } public async Task AcquireLockAsync(int millisecondsTimeout) @@ -81,7 +86,7 @@ namespace Umbraco.Core.Runtime // wait to get a write lock _sqlServerSyntax.WriteLock(db, TimeSpan.FromMilliseconds(millisecondsTimeout), Constants.Locks.MainDom); } - catch(SqlException ex) + catch (SqlException ex) { if (IsLockTimeoutException(ex)) { @@ -121,7 +126,7 @@ namespace Umbraco.Core.Runtime } - return await WaitForExistingAsync(tempId, millisecondsTimeout); + return await WaitForExistingAsync(tempId, millisecondsTimeout).ConfigureAwait(false); } public Task ListenAsync() @@ -134,13 +139,15 @@ namespace Umbraco.Core.Runtime // Create a long running task (dedicated thread) // to poll to check if we are still the MainDom registered in the DB - return Task.Factory.StartNew( - ListeningLoop, - _cancellationTokenSource.Token, - TaskCreationOptions.LongRunning, - // Must explicitly specify this, see https://blog.stephencleary.com/2013/10/continuewith-is-dangerous-too.html - TaskScheduler.Default); - + using (ExecutionContext.SuppressFlow()) + { + return Task.Factory.StartNew( + ListeningLoop, + _cancellationTokenSource.Token, + TaskCreationOptions.LongRunning, + // Must explicitly specify this, see https://blog.stephencleary.com/2013/10/continuewith-is-dangerous-too.html + TaskScheduler.Default); + } } /// @@ -198,7 +205,7 @@ namespace Umbraco.Core.Runtime db.BeginTransaction(IsolationLevel.ReadCommitted); // get a read lock - _sqlServerSyntax.ReadLock(db, Constants.Locks.MainDom); + _sqlServerSyntax.ReadLock(db, _lockTimeout, Constants.Locks.MainDom); if (!IsMainDomValue(_lockId, db)) { @@ -222,11 +229,29 @@ namespace Umbraco.Core.Runtime } finally { - db?.CompleteTransaction(); - db?.Dispose(); + // Even if any of the above fail like BeginTransaction, or even a query after the + // Transaction is started, the calls below will not throw. I've tried all sorts of + // combinations to see if I can make this throw but I can't. In any case, we'll be + // extra safe and try/catch/log + try + { + db?.CompleteTransaction(); + } + catch (Exception ex) + { + _logger.Error(ex, "Unexpected error completing transaction."); + } + + try + { + db?.Dispose(); + } + catch (Exception ex) + { + _logger.Error(ex, "Unexpected error completing disposing."); + } } } - } } @@ -240,37 +265,40 @@ namespace Umbraco.Core.Runtime { var updatedTempId = tempId + UpdatedSuffix; - return Task.Run(() => + using (ExecutionContext.SuppressFlow()) { - try + return Task.Run(() => { - using var db = _dbFactory.CreateDatabase(); - - var watch = new Stopwatch(); - watch.Start(); - while (true) + try { - // poll very often, we need to take over as fast as we can - // local testing shows the actual query to be executed from client/server is approx 300ms but would change depending on environment/IO - Thread.Sleep(1000); + using var db = _dbFactory.CreateDatabase(); - var acquired = TryAcquire(db, tempId, updatedTempId); - if (acquired.HasValue) - return acquired.Value; - - if (watch.ElapsedMilliseconds >= millisecondsTimeout) + var watch = new Stopwatch(); + watch.Start(); + while (true) { - return AcquireWhenMaxWaitTimeElapsed(db); + // poll very often, we need to take over as fast as we can + // local testing shows the actual query to be executed from client/server is approx 300ms but would change depending on environment/IO + Thread.Sleep(1000); + + var acquired = TryAcquire(db, tempId, updatedTempId); + if (acquired.HasValue) + return acquired.Value; + + if (watch.ElapsedMilliseconds >= millisecondsTimeout) + { + return AcquireWhenMaxWaitTimeElapsed(db); + } } } - } - catch (Exception ex) - { - _logger.Error(ex, "An error occurred trying to acquire and waiting for existing SqlMainDomLock to shutdown"); - return false; - } + catch (Exception ex) + { + _logger.Error(ex, "An error occurred trying to acquire and waiting for existing SqlMainDomLock to shutdown"); + return false; + } - }, _cancellationTokenSource.Token); + }, _cancellationTokenSource.Token); + } } private bool? TryAcquire(IUmbracoDatabase db, string tempId, string updatedTempId) @@ -284,7 +312,7 @@ namespace Umbraco.Core.Runtime { transaction = db.GetTransaction(IsolationLevel.ReadCommitted); // get a read lock - _sqlServerSyntax.ReadLock(db, Constants.Locks.MainDom); + _sqlServerSyntax.ReadLock(db, _lockTimeout, Constants.Locks.MainDom); // the row var mainDomRows = db.Fetch("SELECT * FROM umbracoKeyValue WHERE [key] = @key", new { key = MainDomKey }); @@ -296,7 +324,7 @@ namespace Umbraco.Core.Runtime // which indicates that we // can acquire it and it has shutdown. - _sqlServerSyntax.WriteLock(db, Constants.Locks.MainDom); + _sqlServerSyntax.WriteLock(db, _lockTimeout, Constants.Locks.MainDom); // so now we update the row with our appdomain id InsertLockRecord(_lockId, db); @@ -355,7 +383,7 @@ namespace Umbraco.Core.Runtime { transaction = db.GetTransaction(IsolationLevel.ReadCommitted); - _sqlServerSyntax.WriteLock(db, Constants.Locks.MainDom); + _sqlServerSyntax.WriteLock(db, _lockTimeout, Constants.Locks.MainDom); // so now we update the row with our appdomain id InsertLockRecord(_lockId, db); @@ -438,7 +466,7 @@ namespace Umbraco.Core.Runtime db.BeginTransaction(IsolationLevel.ReadCommitted); // get a write lock - _sqlServerSyntax.WriteLock(db, Constants.Locks.MainDom); + _sqlServerSyntax.WriteLock(db, _lockTimeout, Constants.Locks.MainDom); // When we are disposed, it means we have released the MainDom lock // and called all MainDom release callbacks, in this case diff --git a/src/Umbraco.Core/Scoping/Scope.cs b/src/Umbraco.Core/Scoping/Scope.cs index c4c8d08622..4c058cbdb7 100644 --- a/src/Umbraco.Core/Scoping/Scope.cs +++ b/src/Umbraco.Core/Scoping/Scope.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Text; using Umbraco.Core.Cache; using Umbraco.Core.Composing; using Umbraco.Core.Events; @@ -36,11 +37,10 @@ namespace Umbraco.Core.Scoping private IEventDispatcher _eventDispatcher; private object _dictionaryLocker; - - // ReadLocks and WriteLocks if we're the outer most scope it's those owned by the entire chain - // If we're a child scope it's those that we have requested. - internal readonly Dictionary ReadLocks; - internal readonly Dictionary WriteLocks; + private HashSet _readLocks; + private HashSet _writeLocks; + internal Dictionary> ReadLocks; + internal Dictionary> WriteLocks; // initializes a new scope private Scope(ScopeProvider scopeProvider, @@ -67,8 +67,6 @@ namespace Umbraco.Core.Scoping Detachable = detachable; _dictionaryLocker = new object(); - ReadLocks = new Dictionary(); - WriteLocks = new Dictionary(); #if DEBUG_SCOPES _scopeProvider.RegisterScope(this); @@ -345,6 +343,8 @@ namespace Umbraco.Core.Scoping if (this != _scopeProvider.AmbientScope) { + var failedMessage = $"The {nameof(Scope)} {this.InstanceId} being disposed is not the Ambient {nameof(Scope)} {(_scopeProvider.AmbientScope?.InstanceId.ToString() ?? "NULL")}. This typically indicates that a child {nameof(Scope)} was not disposed, or flowed to a child thread that was not awaited, or concurrent threads are accessing the same {nameof(Scope)} (Ambient context) which is not supported. If using Task.Run (or similar) as a fire and forget tasks or to run threads in parallel you must suppress execution context flow with ExecutionContext.SuppressFlow() and ExecutionContext.RestoreFlow()."; + #if DEBUG_SCOPES var ambient = _scopeProvider.AmbientScope; _logger.Debug("Dispose error (" + (ambient == null ? "no" : "other") + " ambient)"); @@ -356,24 +356,21 @@ namespace Umbraco.Core.Scoping + "- ambient ctor ->\r\n" + ambientInfos.CtorStack + "\r\n" + "- dispose ctor ->\r\n" + disposeInfos.CtorStack + "\r\n"); #else - throw new InvalidOperationException("Not the ambient scope."); + throw new InvalidOperationException(failedMessage); #endif } // Decrement the lock counters on the parent if any. - if (ParentScope != null) + ClearLocks(InstanceId); + if (ParentScope is null) { - lock (_dictionaryLocker) + // We're the parent scope, make sure that locks of all scopes has been cleared + // Since we're only reading we don't have to be in a lock + if (ReadLocks?.Count > 0 || WriteLocks?.Count > 0) { - foreach (var readLockPair in ReadLocks) - { - DecrementReadLock(readLockPair.Key, readLockPair.Value); - } - - foreach (var writeLockPair in WriteLocks) - { - DecrementWriteLock(writeLockPair.Key, writeLockPair.Value); - } + var exception = new InvalidOperationException($"All scopes has not been disposed from parent scope: {InstanceId}, see log for more details."); + _logger.Error(exception, GenerateUnclearedScopesLogMessage()); + throw exception; } } @@ -396,6 +393,42 @@ namespace Umbraco.Core.Scoping GC.SuppressFinalize(this); } + /// + /// Generates a log message with all scopes that hasn't cleared their locks, including how many, and what locks they have requested. + /// + /// Log message. + private string GenerateUnclearedScopesLogMessage() + { + // Dump the dicts into a message for the locks. + StringBuilder builder = new StringBuilder(); + builder.AppendLine($"Lock counters aren't empty, suggesting a scope hasn't been properly disposed, parent id: {InstanceId}"); + WriteLockDictionaryToString(ReadLocks, builder, "read locks"); + WriteLockDictionaryToString(WriteLocks, builder, "write locks"); + return builder.ToString(); + } + + /// + /// Writes a locks dictionary to a for logging purposes. + /// + /// Lock dictionary to report on. + /// String builder to write to. + /// The name to report the dictionary as. + private void WriteLockDictionaryToString(Dictionary> dict, StringBuilder builder, string dictName) + { + if (dict?.Count > 0) + { + builder.AppendLine($"Remaining {dictName}:"); + foreach (var instance in dict) + { + builder.AppendLine($"Scope {instance.Key}"); + foreach (var lockCounter in instance.Value) + { + builder.AppendLine($"\tLock ID: {lockCounter.Key} - times requested: {lockCounter.Value}"); + } + } + } + } + private void DisposeLastScope() { // figure out completed @@ -516,207 +549,157 @@ namespace Umbraco.Core.Scoping ?? (_logUncompletedScopes = Current.Configs.CoreDebug().LogUncompletedScopes)).Value; /// - /// Decrements the count of the ReadLocks with a specific lock object identifier we currently hold + /// Increment the counter of a locks dictionary, either ReadLocks or WriteLocks, + /// for a specific scope instance and lock identifier. Must be called within a lock. /// - /// Lock object identifier to decrement - /// Amount to decrement the lock count with - public void DecrementReadLock(int lockId, int amountToDecrement) + /// Lock ID to increment. + /// Instance ID of the scope requesting the lock. + /// Reference to the dictionary to increment on + private void IncrementLock(int lockId, Guid instanceId, ref Dictionary> locks) { - // If we aren't the outermost scope, pass it on to the parent. - if (ParentScope != null) - { - ParentScope.DecrementReadLock(lockId, amountToDecrement); - return; - } + // Since we've already checked that we're the parent in the WriteLockInner method, we don't need to check again. + // If it's the very first time a lock has been requested the WriteLocks dict hasn't been instantiated yet. + locks ??= new Dictionary>(); - lock (_dictionaryLocker) + // Try and get the dict associated with the scope id. + var locksDictFound = locks.TryGetValue(instanceId, out var locksDict); + if (locksDictFound) { - ReadLocks[lockId] -= amountToDecrement; + locksDict.TryGetValue(lockId, out var value); + locksDict[lockId] = value + 1; + } + else + { + // The scope hasn't requested a lock yet, so we have to create a dict for it. + locks.Add(instanceId, new Dictionary()); + locks[instanceId][lockId] = 1; } } /// - /// Decrements the count of the WriteLocks with a specific lock object identifier we currently hold. + /// Clears all lock counters for a given scope instance, signalling that the scope has been disposed. /// - /// Lock object identifier to decrement. - /// Amount to decrement the lock count with - public void DecrementWriteLock(int lockId, int amountToDecrement) + /// Instance ID of the scope to clear. + private void ClearLocks(Guid instanceId) { - // If we aren't the outermost scope, pass it on to the parent. - if (ParentScope != null) + if (ParentScope is not null) { - ParentScope.DecrementWriteLock(lockId, amountToDecrement); - return; + ParentScope.ClearLocks(instanceId); } - - lock (_dictionaryLocker) + else { - WriteLocks[lockId] -= amountToDecrement; - } - } - - /// - /// Increment the count of the read locks we've requested - /// - /// - /// This should only be done on child scopes since it's then used to decrement the count later. - /// - /// - private void IncrementRequestedReadLock(params int[] lockIds) - { - // We need to keep track of what lockIds we have requested locks for to be able to decrement them. - if (ParentScope != null) - { - foreach (var lockId in lockIds) + lock (_dictionaryLocker) { - lock (_dictionaryLocker) - { - if (ReadLocks.ContainsKey(lockId)) - { - ReadLocks[lockId] += 1; - } - else - { - ReadLocks[lockId] = 1; - } - } - } - } - } - - /// - /// Increment the count of the write locks we've requested - /// - /// - /// This should only be done on child scopes since it's then used to decrement the count later. - /// - /// - private void IncrementRequestedWriteLock(params int[] lockIds) - { - // We need to keep track of what lockIds we have requested locks for to be able to decrement them. - if (ParentScope != null) - { - foreach (var lockId in lockIds) - { - lock (_dictionaryLocker) - { - if (WriteLocks.ContainsKey(lockId)) - { - WriteLocks[lockId] += 1; - } - else - { - WriteLocks[lockId] = 1; - } - } + ReadLocks?.Remove(instanceId); + WriteLocks?.Remove(instanceId); } } } /// - public void ReadLock(params int[] lockIds) - { - IncrementRequestedReadLock(lockIds); - ReadLockInner(null, lockIds); - } + public void ReadLock(params int[] lockIds) => ReadLockInner(InstanceId, null, lockIds); /// - public void ReadLock(TimeSpan timeout, int lockId) - { - IncrementRequestedReadLock(lockId); - ReadLockInner(timeout, lockId); - } + public void ReadLock(TimeSpan timeout, int lockId) => ReadLockInner(InstanceId, timeout, lockId); /// - public void WriteLock(params int[] lockIds) - { - IncrementRequestedWriteLock(lockIds); - WriteLockInner(null, lockIds); - } + public void WriteLock(params int[] lockIds) => WriteLockInner(InstanceId, null, lockIds); /// - public void WriteLock(TimeSpan timeout, int lockId) - { - IncrementRequestedWriteLock(lockId); - WriteLockInner(timeout, lockId); - } + public void WriteLock(TimeSpan timeout, int lockId) => WriteLockInner(InstanceId, timeout, lockId); /// /// Handles acquiring a read lock, will delegate it to the parent if there are any. /// + /// Instance ID of the requesting scope. /// Optional database timeout in milliseconds. /// Array of lock object identifiers. - internal void ReadLockInner(TimeSpan? timeout = null, params int[] lockIds) + private void ReadLockInner(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) { - if (ParentScope != null) + if (ParentScope is not null) { - // Delegate acquiring the lock to the parent if any. - ParentScope.ReadLockInner(timeout, lockIds); - return; + // If we have a parent we delegate lock creation to parent. + ParentScope.ReadLockInner(instanceId, timeout, lockIds); } - - // If we are the parent, then handle the lock request. - foreach (var lockId in lockIds) + else { - lock (_dictionaryLocker) - { - // Only acquire the lock if we haven't done so yet. - if (!ReadLocks.ContainsKey(lockId)) - { - if (timeout is null) - { - // We want a lock with a custom timeout - ObtainReadLock(lockId); - } - else - { - // We just want an ordinary lock. - ObtainTimoutReadLock(lockId, timeout.Value); - } - // Add the lockId as a key to the dict. - ReadLocks[lockId] = 0; - } - - ReadLocks[lockId] += 1; - } + // We are the outermost scope, handle the lock request. + LockInner(instanceId, ref ReadLocks, ref _readLocks, ObtainReadLock, ObtainTimeoutReadLock, timeout, lockIds); } } /// /// Handles acquiring a write lock with a specified timeout, will delegate it to the parent if there are any. /// + /// Instance ID of the requesting scope. /// Optional database timeout in milliseconds. /// Array of lock object identifiers. - internal void WriteLockInner(TimeSpan? timeout = null, params int[] lockIds) + private void WriteLockInner(Guid instanceId, TimeSpan? timeout = null, params int[] lockIds) { - if (ParentScope != null) + if (ParentScope is not null) { // If we have a parent we delegate lock creation to parent. - ParentScope.WriteLockInner(timeout, lockIds); - return; + ParentScope.WriteLockInner(instanceId, timeout, lockIds); } - - foreach (var lockId in lockIds) + else { - lock (_dictionaryLocker) - { - // Only acquire lock if we haven't yet (WriteLocks not containing the key) - if (!WriteLocks.ContainsKey(lockId)) - { - if (timeout is null) - { - ObtainWriteLock(lockId); - } - else - { - ObtainTimeoutWriteLock(lockId, timeout.Value); - } - // Add the lockId as a key to the dict. - WriteLocks[lockId] = 0; - } + // We are the outermost scope, handle the lock request. + LockInner(instanceId, ref WriteLocks, ref _writeLocks, ObtainWriteLock, ObtainTimeoutWriteLock, timeout, lockIds); + } + } - // Increment count of the lock by 1. - WriteLocks[lockId] += 1; + /// + /// Handles acquiring a lock, this should only be called from the outermost scope. + /// + /// Instance ID of the scope requesting the lock. + /// Reference to the applicable locks dictionary (ReadLocks or WriteLocks). + /// Reference to the applicable locks hashset (_readLocks or _writeLocks). + /// Delegate used to request the lock from the database without a timeout. + /// Delegate used to request the lock from the database with a timeout. + /// Optional timeout parameter to specify a timeout. + /// Lock identifiers to lock on. + private void LockInner(Guid instanceId, ref Dictionary> locks, ref HashSet locksSet, + Action obtainLock, Action obtainLockTimeout, TimeSpan? timeout = null, + params int[] lockIds) + { + lock (_dictionaryLocker) + { + locksSet ??= new HashSet(); + foreach (var lockId in lockIds) + { + // Only acquire the lock if we haven't done so yet. + if (!locksSet.Contains(lockId)) + { + IncrementLock(lockId, instanceId, ref locks); + locksSet.Add(lockId); + try + { + if (timeout is null) + { + // We just want an ordinary lock. + obtainLock(lockId); + } + else + { + // We want a lock with a custom timeout + obtainLockTimeout(lockId, timeout.Value); + } + } + catch + { + // Something went wrong and we didn't get the lock + // Since we at this point have determined that we haven't got any lock with an ID of LockID, it's safe to completely remove it instead of decrementing. + locks[instanceId].Remove(lockId); + // It needs to be removed from the HashSet as well, because that's how we determine to acquire a lock. + locksSet.Remove(lockId); + throw; + } + } + else + { + // We already have a lock, but need to update the dictionary for debugging purposes. + IncrementLock(lockId, instanceId, ref locks); + } } } } @@ -735,10 +718,10 @@ namespace Umbraco.Core.Scoping /// /// Lock object identifier to lock. /// TimeSpan specifying the timout period. - private void ObtainTimoutReadLock(int lockId, TimeSpan timeout) + private void ObtainTimeoutReadLock(int lockId, TimeSpan timeout) { var syntax2 = Database.SqlContext.SqlSyntax as ISqlSyntaxProvider2; - if (syntax2 == null) + if (syntax2 is null) { throw new InvalidOperationException($"{Database.SqlContext.SqlSyntax.GetType()} is not of type {typeof(ISqlSyntaxProvider2)}"); } @@ -763,7 +746,7 @@ namespace Umbraco.Core.Scoping private void ObtainTimeoutWriteLock(int lockId, TimeSpan timeout) { var syntax2 = Database.SqlContext.SqlSyntax as ISqlSyntaxProvider2; - if (syntax2 == null) + if (syntax2 is null) { throw new InvalidOperationException($"{Database.SqlContext.SqlSyntax.GetType()} is not of type {typeof(ISqlSyntaxProvider2)}"); } diff --git a/src/Umbraco.Core/Scoping/ScopeProvider.cs b/src/Umbraco.Core/Scoping/ScopeProvider.cs index bf4e27bdb6..a1cc128181 100644 --- a/src/Umbraco.Core/Scoping/ScopeProvider.cs +++ b/src/Umbraco.Core/Scoping/ScopeProvider.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel; using System.Data; using System.Runtime.Remoting.Messaging; using System.Web; @@ -240,6 +241,9 @@ namespace Umbraco.Core.Scoping var value = GetHttpContextObject(ContextItemKey, false); return value ?? GetCallContextObject(ContextItemKey); } + + [Obsolete("This setter is not used and will be removed in future versions")] + [EditorBrowsable(EditorBrowsableState.Never)] set { // clear both diff --git a/src/Umbraco.Core/Services/Implement/ContentService.cs b/src/Umbraco.Core/Services/Implement/ContentService.cs index e5363d0e2b..d8e99663ea 100644 --- a/src/Umbraco.Core/Services/Implement/ContentService.cs +++ b/src/Umbraco.Core/Services/Implement/ContentService.cs @@ -1415,7 +1415,7 @@ namespace Umbraco.Core.Services.Implement var result = CommitDocumentChangesInternal(scope, d, saveEventArgs, allLangs.Value, d.WriterId); if (result.Success == false) - Logger.Error(null, "Failed to publish document id={DocumentId}, reason={Reason}.", d.Id, result.Result); + Logger.Error(null, "Failed to publish document id={DocumentId}, reason={Reason}.", d.Id, result.Result); results.Add(result); } @@ -2201,7 +2201,7 @@ namespace Umbraco.Core.Services.Implement while (page * pageSize < total) { var descendants = GetPagedDescendants(content.Id, page++, pageSize, out total); - foreach (var descendant in descendants) + foreach (var descendant in descendants.OrderBy(x => x.Level).ThenBy(y => y.SortOrder)) { // if parent has not been copied, skip, else gets its copy id if (idmap.TryGetValue(descendant.ParentId, out parentId) == false) continue; @@ -2420,7 +2420,7 @@ namespace Umbraco.Core.Services.Implement if (report.FixedIssues.Count > 0) { //The event args needs a content item so we'll make a fake one with enough properties to not cause a null ref - var root = new Content("root", -1, new ContentType(-1)) {Id = -1, Key = Guid.Empty}; + var root = new Content("root", -1, new ContentType(-1)) { Id = -1, Key = Guid.Empty }; scope.Events.Dispatch(TreeChanged, this, new TreeChange.EventArgs(new TreeChange(root, TreeChangeTypes.RefreshAll))); } @@ -3169,7 +3169,7 @@ namespace Umbraco.Core.Services.Implement if (rollbackSaveResult.Success == false) { //Log the error/warning - Logger.Error("User '{UserId}' was unable to rollback content '{ContentId}' to version '{VersionId}'", userId, id, versionId); + Logger.Error("User '{UserId}' was unable to rollback content '{ContentId}' to version '{VersionId}'", userId, id, versionId); } else { @@ -3178,7 +3178,7 @@ namespace Umbraco.Core.Services.Implement scope.Events.Dispatch(RolledBack, this, rollbackEventArgs); //Logging & Audit message - Logger.Info("User '{UserId}' rolled back content '{ContentId}' to version '{VersionId}'", userId, id, versionId); + Logger.Info("User '{UserId}' rolled back content '{ContentId}' to version '{VersionId}'", userId, id, versionId); Audit(AuditType.RollBack, userId, id, $"Content '{content.Name}' was rolled back to version '{versionId}'"); } diff --git a/src/Umbraco.Core/StringExtensions.cs b/src/Umbraco.Core/StringExtensions.cs index 80ef81f36d..0e16c2c852 100644 --- a/src/Umbraco.Core/StringExtensions.cs +++ b/src/Umbraco.Core/StringExtensions.cs @@ -22,7 +22,7 @@ namespace Umbraco.Core /// public static class StringExtensions { - + private const char DefaultEscapedStringEscapeChar = '\\'; private static readonly char[] ToCSharpHexDigitLower = "0123456789abcdef".ToCharArray(); private static readonly char[] ToCSharpEscapeChars; @@ -1490,5 +1490,44 @@ namespace Umbraco.Core /// public static string NullOrWhiteSpaceAsNull(this string text) => string.IsNullOrWhiteSpace(text) ? null : text; + + /// + /// Splits a string with an escape character that allows for the split character to exist in a string + /// + /// The string to split + /// The character to split on + /// The character which can be used to escape the character to split on + /// The string split into substrings delimited by the split character + public static IEnumerable EscapedSplit(this string value, char splitChar, char escapeChar = DefaultEscapedStringEscapeChar) + { + if (value == null) yield break; + + var sb = new StringBuilder(value.Length); + var escaped = false; + + foreach (var chr in value.ToCharArray()) + { + if (escaped) + { + escaped = false; + sb.Append(chr); + } + else if (chr == splitChar) + { + yield return sb.ToString(); + sb.Clear(); + } + else if (chr == escapeChar) + { + escaped = true; + } + else + { + sb.Append(chr); + } + } + + yield return sb.ToString(); + } } } diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index f0ba7f66d8..1160881304 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -133,6 +133,8 @@ + + @@ -270,7 +272,6 @@ - diff --git a/src/Umbraco.Examine/UmbracoExamineIndex.cs b/src/Umbraco.Examine/UmbracoExamineIndex.cs index cc97178e5c..511d78db92 100644 --- a/src/Umbraco.Examine/UmbracoExamineIndex.cs +++ b/src/Umbraco.Examine/UmbracoExamineIndex.cs @@ -24,8 +24,8 @@ namespace Umbraco.Examine // note // wrapping all operations that end up calling base.SafelyProcessQueueItems in a safe call // context because they will fork a thread/task/whatever which should *not* capture our - // call context (and the database it can contain)! ideally we should be able to override - // SafelyProcessQueueItems but that's not possible in the current version of Examine. + // call context (and the database it can contain)! + // TODO: FIX Examine to not flow the ExecutionContext so callers don't need to worry about this! /// /// Used to store the path of a content object @@ -99,6 +99,9 @@ namespace Umbraco.Examine { if (CanInitialize()) { + // Use SafeCallContext to prevent the current CallContext flow to child + // tasks executed in the base class so we don't leak Scopes. + // TODO: See notes at the top of this class using (new SafeCallContext()) { base.PerformDeleteFromIndex(itemIds, onComplete); @@ -106,6 +109,20 @@ namespace Umbraco.Examine } } + protected override void PerformIndexItems(IEnumerable values, Action onComplete) + { + if (CanInitialize()) + { + // Use SafeCallContext to prevent the current CallContext flow to child + // tasks executed in the base class so we don't leak Scopes. + // TODO: See notes at the top of this class + using (new SafeCallContext()) + { + base.PerformIndexItems(values, onComplete); + } + } + } + /// /// Returns true if the Umbraco application is in a state that we can initialize the examine indexes /// diff --git a/src/Umbraco.ModelsBuilder.Embedded/Compose/ModelsBuilderComposer.cs b/src/Umbraco.ModelsBuilder.Embedded/Compose/ModelsBuilderComposer.cs index 45c4de5d2a..01010cca66 100644 --- a/src/Umbraco.ModelsBuilder.Embedded/Compose/ModelsBuilderComposer.cs +++ b/src/Umbraco.ModelsBuilder.Embedded/Compose/ModelsBuilderComposer.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using System.Reflection; using Umbraco.Core; using Umbraco.Core.Logging; @@ -20,14 +21,11 @@ namespace Umbraco.ModelsBuilder.Embedded.Compose { public void Compose(Composition composition) { - var isLegacyModelsBuilderInstalled = IsLegacyModelsBuilderInstalled(); - - composition.Configs.Add(() => new ModelsBuilderConfig()); - if (isLegacyModelsBuilderInstalled) + if (IsExternalModelsBuilderInstalled() == true) { - ComposeForLegacyModelsBuilder(composition); + ComposeForExternalModelsBuilder(composition); return; } @@ -45,22 +43,35 @@ namespace Umbraco.ModelsBuilder.Embedded.Compose ComposeForDefaultModelsFactory(composition); } - private static bool IsLegacyModelsBuilderInstalled() + private static bool IsExternalModelsBuilderInstalled() { - Assembly legacyMbAssembly = null; + var assemblyNames = new[] + { + "Umbraco.ModelsBuider", + "ModelsBuilder.Umbraco" + }; + try { - legacyMbAssembly = Assembly.Load("Umbraco.ModelsBuilder"); + foreach (var name in assemblyNames) + { + var assembly = Assembly.Load(name); + + if (assembly != null) + { + return true; + } + } } - catch (System.Exception) + catch (Exception) { //swallow exception, DLL must not be there } - return legacyMbAssembly != null; + return false; } - private void ComposeForLegacyModelsBuilder(Composition composition) + private void ComposeForExternalModelsBuilder(Composition composition) { composition.Logger.Info("ModelsBuilder.Embedded is disabled, the external ModelsBuilder was detected."); composition.Components().Append(); diff --git a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts index 1a40e8451f..0cec374c5d 100644 --- a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts +++ b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts @@ -1,5 +1,13 @@ /// -import { DocumentTypeBuilder, ContentBuilder, AliasHelper } from 'umbraco-cypress-testhelpers'; +import { + DocumentTypeBuilder, + ContentBuilder, + AliasHelper, + GridDataTypeBuilder, + PartialViewMacroBuilder, + MacroBuilder +} from 'umbraco-cypress-testhelpers'; + context('Content', () => { beforeEach(() => { @@ -14,6 +22,23 @@ context('Content', () => { cy.get('.umb-tree-item__inner').should('exist', {timeout: 10000}); } + function createSimpleMacro(name){ + const insertMacro = new PartialViewMacroBuilder() + .withName(name) + .withContent(`@inherits Umbraco.Web.Macros.PartialViewMacroPage +

Acceptance test

`) + .build(); + + const macroWithPartial = new MacroBuilder() + .withName(name) + .withPartialViewMacro(insertMacro) + .withRenderInEditor() + .withUseInEditor() + .build(); + + cy.saveMacroWithPartial(macroWithPartial); + } + it('Copy content', () => { const rootDocTypeName = "Test document type"; const childDocTypeName = "Child test document type"; @@ -596,4 +621,181 @@ context('Content', () => { cy.umbracoEnsureTemplateNameNotExists(pickerDocTypeName); cy.umbracoEnsureDocumentTypeNameNotExists(pickedDocTypeName); }); + + it('Content with macro in RTE', () => { + const viewMacroName = 'Content with macro in RTE'; + const partialFileName = viewMacroName + '.cshtml'; + + cy.umbracoEnsureMacroNameNotExists(viewMacroName); + cy.umbracoEnsurePartialViewMacroFileNameNotExists(partialFileName); + cy.umbracoEnsureDocumentTypeNameNotExists(viewMacroName); + cy.umbracoEnsureTemplateNameNotExists(viewMacroName); + cy.deleteAllContent(); + + // First thing first we got to create the macro we will be inserting + createSimpleMacro(viewMacroName); + + // Now we need to create a document type with a rich text editor where we can insert the macro + // The document type must have a template as well in order to ensure that the content is displayed correctly + const alias = AliasHelper.toAlias(viewMacroName); + const docType = new DocumentTypeBuilder() + .withName(viewMacroName) + .withAlias(alias) + .withAllowAsRoot(true) + .withDefaultTemplate(alias) + .addGroup() + .addRichTextProperty() + .withAlias('text') + .done() + .done() + .build(); + + cy.saveDocumentType(docType).then((generatedDocType) => { + // Might as wel initally create the content here, the less GUI work during the test the better + const contentNode = new ContentBuilder() + .withContentTypeAlias(generatedDocType["alias"]) + .withAction('saveNew') + .addVariant() + .withName(viewMacroName) + .withSave(true) + .done() + .build(); + + cy.saveContent(contentNode); + }); + + // Edit the macro template in order to have something to verify on when rendered. + cy.editTemplate(viewMacroName, `@inherits Umbraco.Web.Mvc.UmbracoViewPage +@using ContentModels = Umbraco.Web.PublishedModels; +@{ + Layout = null; +} +@{ + if (Model.HasValue("text")){ + @(Model.Value("text")) + } +} `); + + // Enter content + refreshContentTree(); + cy.umbracoTreeItem("content", [viewMacroName]).click(); + + // Insert macro + cy.get('#mceu_13-button').click(); + cy.get('.umb-card-grid-item').contains(viewMacroName).click(); + + // Save and publish + cy.umbracoButtonByLabelKey('buttons_saveAndPublish').click(); + cy.umbracoSuccessNotification().should('be.visible'); + + // Ensure that the view gets rendered correctly + const expected = `

Acceptance test

 

`; + cy.umbracoVerifyRenderedViewContent('/', expected, true).should('be.true'); + + // Cleanup + cy.umbracoEnsureMacroNameNotExists(viewMacroName); + cy.umbracoEnsurePartialViewMacroFileNameNotExists(partialFileName); + cy.umbracoEnsureDocumentTypeNameNotExists(viewMacroName); + cy.umbracoEnsureTemplateNameNotExists(viewMacroName); + }); + + it('Content with macro in grid', () => { + const name = 'Content with macro in grid'; + const macroName = 'Grid macro'; + const macroFileName = macroName + '.cshtml'; + + cy.umbracoEnsureDataTypeNameNotExists(name); + cy.umbracoEnsureDocumentTypeNameNotExists(name); + cy.umbracoEnsureTemplateNameNotExists(name); + cy.umbracoEnsureMacroNameNotExists(macroName); + cy.umbracoEnsurePartialViewMacroFileNameNotExists(macroFileName); + cy.deleteAllContent(); + + createSimpleMacro(macroName); + + const grid = new GridDataTypeBuilder() + .withName(name) + .withDefaultGrid() + .build(); + + const alias = AliasHelper.toAlias(name); + // Save grid and get the ID + cy.saveDataType(grid).then((dataType) => { + // Create a document type using the data type + const docType = new DocumentTypeBuilder() + .withName(name) + .withAlias(alias) + .withAllowAsRoot(true) + .withDefaultTemplate(alias) + .addGroup() + .addCustomProperty(dataType['id']) + .withAlias('grid') + .done() + .done() + .build(); + + cy.saveDocumentType(docType).then((generatedDocType) => { + const contentNode = new ContentBuilder() + .withContentTypeAlias(generatedDocType["alias"]) + .addVariant() + .withName(name) + .withSave(true) + .done() + .build(); + + cy.saveContent(contentNode); + }); + }); + + // Edit the template to allow us to verify the rendered view + cy.editTemplate(name, `@inherits Umbraco.Web.Mvc.UmbracoViewPage +@using ContentModels = Umbraco.Web.PublishedModels; +@{ + Layout = null; +} +@Html.GetGridHtml(Model, "grid")`); + + // Act + // Enter content + refreshContentTree(); + cy.umbracoTreeItem("content", [name]).click(); + // Click add + cy.get(':nth-child(2) > .preview-row > .preview-col > .preview-cell').click(); // Choose 1 column layout. + cy.get('.umb-column > .templates-preview > :nth-child(2) > small').click(); // Choose headline + cy.get('.umb-cell-placeholder').click(); + // Click macro + cy.get(':nth-child(4) > .umb-card-grid-item > :nth-child(1)').click(); + // Select the macro + cy.get('.umb-card-grid-item').contains(macroName).click(); + + // Save and publish + cy.umbracoButtonByLabelKey('buttons_saveAndPublish').click(); + cy.umbracoSuccessNotification().should('be.visible'); + + const expected = ` +
+
+
+
+
+
+
+

Acceptance test

+
+
+
+
+
+
+
` + + cy.umbracoVerifyRenderedViewContent('/', expected, true).should('be.true'); + + // Clean + cy.umbracoEnsureDataTypeNameNotExists(name); + cy.umbracoEnsureDocumentTypeNameNotExists(name); + cy.umbracoEnsureTemplateNameNotExists(name); + cy.umbracoEnsureMacroNameNotExists(macroName); + cy.umbracoEnsurePartialViewMacroFileNameNotExists(macroFileName); + }); }); diff --git a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/languages.ts b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/languages.ts index 49bcf94943..336e5793d9 100644 --- a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/languages.ts +++ b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/languages.ts @@ -6,7 +6,10 @@ context('Languages', () => { }); it('Add language', () => { - const name = "Kyrgyz (Kyrgyzstan)"; // Must be an option in the select box + // For some reason the languages to chose fom seems to be translated differently than normal, as an example: + // My system is set to EN (US), but most languages are translated into Danish for some reason + // Aghem seems untranslated though? + const name = "Aghem"; // Must be an option in the select box cy.umbracoEnsureLanguageNameNotExists(name); diff --git a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/templates.ts b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/templates.ts index c586384af7..65d03e5a78 100644 --- a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/templates.ts +++ b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Settings/templates.ts @@ -23,14 +23,18 @@ context('Templates', () => { cy.umbracoEnsureTemplateNameNotExists(name); createTemplate(); + // We have to wait for the ace editor to load, because when the editor is loading it will "steal" the focus briefly, + // which causes the save event to fire if we've added something to the header field, causing errors. + cy.wait(500); + //Type name cy.umbracoEditorHeaderName(name); // Save // We must drop focus for the auto save event to occur. cy.get('.btn-success').focus(); // And then wait for the auto save event to finish by finding the page in the tree view. - // This is a bit of a roundabout way to find items in a treev view since we dont use umbracoTreeItem - // but we must be able to wait for the save evnent to finish, and we can't do that with umbracoTreeItem + // This is a bit of a roundabout way to find items in a tree view since we dont use umbracoTreeItem + // but we must be able to wait for the save event to finish, and we can't do that with umbracoTreeItem cy.get('[data-element="tree-item-templates"] > :nth-child(2) > .umb-animated > .umb-tree-item__inner > .umb-tree-item__label') .contains(name).should('be.visible', { timeout: 10000 }); // Now that the auto save event has finished we can save diff --git a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Tour/backofficeTour.ts b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Tour/backofficeTour.ts index 9bc1fff488..d3950d7d19 100644 --- a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Tour/backofficeTour.ts +++ b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Tour/backofficeTour.ts @@ -49,7 +49,7 @@ function resetTourData() { { "alias": "umbIntroIntroduction", "completed": false, - "disabled": false + "disabled": true }; cy.getCookie('UMB-XSRF-TOKEN', { log: false }).then((token) => { diff --git a/src/Umbraco.Tests.AcceptanceTest/package.json b/src/Umbraco.Tests.AcceptanceTest/package.json index 378fe719fc..caf75638e6 100644 --- a/src/Umbraco.Tests.AcceptanceTest/package.json +++ b/src/Umbraco.Tests.AcceptanceTest/package.json @@ -7,10 +7,10 @@ }, "devDependencies": { "cross-env": "^7.0.2", - "cypress": "^6.0.1", + "cypress": "^6.8.0", "ncp": "^2.0.0", "prompt": "^1.0.0", - "umbraco-cypress-testhelpers": "^1.0.0-beta-52" + "umbraco-cypress-testhelpers": "^1.0.0-beta-53" }, "dependencies": { "typescript": "^3.9.2" diff --git a/src/Umbraco.Tests/Mapping/MappingTests.cs b/src/Umbraco.Tests/Mapping/MappingTests.cs index e6a382692c..35f64cac62 100644 --- a/src/Umbraco.Tests/Mapping/MappingTests.cs +++ b/src/Umbraco.Tests/Mapping/MappingTests.cs @@ -1,17 +1,40 @@ using System; using System.Collections.Generic; +using System.Data; using System.Linq; using System.Threading; +using Moq; using NUnit.Framework; +using Umbraco.Core.Events; using Umbraco.Core.Mapping; using Umbraco.Core.Models; +using Umbraco.Core.Scoping; using Umbraco.Web.Models.ContentEditing; +using PropertyCollection = Umbraco.Core.Models.PropertyCollection; namespace Umbraco.Tests.Mapping { [TestFixture] public class MappingTests { + private IScopeProvider _scopeProvider; + + [SetUp] + public void MockScopeProvider() + { + var scopeMock = new Mock(); + scopeMock.Setup(x => x.CreateScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Mock.Of); + + _scopeProvider = scopeMock.Object; + } + [Test] public void SimpleMap() { @@ -19,7 +42,7 @@ namespace Umbraco.Tests.Mapping { new MapperDefinition1(), }); - var mapper = new UmbracoMapper(definitions); + var mapper = new UmbracoMapper(definitions, _scopeProvider); var thing1 = new Thing1 { Value = "value" }; var thing2 = mapper.Map(thing1); @@ -44,7 +67,7 @@ namespace Umbraco.Tests.Mapping { new MapperDefinition1(), }); - var mapper = new UmbracoMapper(definitions); + var mapper = new UmbracoMapper(definitions, _scopeProvider); var thing1A = new Thing1 { Value = "valueA" }; var thing1B = new Thing1 { Value = "valueB" }; @@ -78,7 +101,7 @@ namespace Umbraco.Tests.Mapping { new MapperDefinition1(), }); - var mapper = new UmbracoMapper(definitions); + var mapper = new UmbracoMapper(definitions, _scopeProvider); var thing3 = new Thing3 { Value = "value" }; var thing2 = mapper.Map(thing3); @@ -103,7 +126,7 @@ namespace Umbraco.Tests.Mapping { new MapperDefinition2(), }); - var mapper = new UmbracoMapper(definitions); + var mapper = new UmbracoMapper(definitions, _scopeProvider); // can map a PropertyCollection var source = new PropertyCollection(); @@ -119,7 +142,7 @@ namespace Umbraco.Tests.Mapping new MapperDefinition1(), new MapperDefinition3(), }); - var mapper = new UmbracoMapper(definitions); + var mapper = new UmbracoMapper(definitions, _scopeProvider); // the mapper currently has a map from Thing1 to Thing2 // because Thing3 inherits from Thing1, it will map a Thing3 instance, @@ -179,7 +202,7 @@ namespace Umbraco.Tests.Mapping { new MapperDefinition4(), }); - var mapper = new UmbracoMapper(definitions); + var mapper = new UmbracoMapper(definitions, _scopeProvider); var thing5 = new Thing5() { @@ -203,7 +226,7 @@ namespace Umbraco.Tests.Mapping { new MapperDefinition5(), }); - var mapper = new UmbracoMapper(definitions); + var mapper = new UmbracoMapper(definitions, _scopeProvider); var thing7 = new Thing7(); diff --git a/src/Umbraco.Tests/Scoping/ScopeTests.cs b/src/Umbraco.Tests/Scoping/ScopeTests.cs index 6c5e9a74b5..7d8984baad 100644 --- a/src/Umbraco.Tests/Scoping/ScopeTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopeTests.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Runtime.Remoting.Messaging; using System.Threading; +using System.Threading.Tasks; using NUnit.Framework; using Umbraco.Core; using Umbraco.Core.Persistence; @@ -24,6 +25,119 @@ namespace Umbraco.Tests.Scoping Assert.IsNull(ScopeProvider.AmbientScope); // gone } + [Test] + public void GivenUncompletedScopeOnChildThread_WhenTheParentCompletes_TheTransactionIsRolledBack() + { + ScopeProvider scopeProvider = ScopeProvider; + + Assert.IsNull(ScopeProvider.AmbientScope); + IScope mainScope = scopeProvider.CreateScope(); + + var t = Task.Run(() => + { + IScope nested = scopeProvider.CreateScope(); + Thread.Sleep(2000); + nested.Dispose(); + }); + + Thread.Sleep(1000); // mimic some long running operation that is shorter than the other thread + mainScope.Complete(); + Assert.Throws(() => mainScope.Dispose()); + + Task.WaitAll(t); + } + + [Test] + public void GivenNonDisposedChildScope_WhenTheParentDisposes_ThenInvalidOperationExceptionThrows() + { + // this all runs in the same execution context so the AmbientScope reference isn't a copy + + ScopeProvider scopeProvider = ScopeProvider; + + Assert.IsNull(ScopeProvider.AmbientScope); + IScope mainScope = scopeProvider.CreateScope(); + + IScope nested = scopeProvider.CreateScope(); // not disposing + + InvalidOperationException ex = Assert.Throws(() => mainScope.Dispose()); + Console.WriteLine(ex); + } + + [Test] + public void GivenChildThread_WhenParentDisposedBeforeChild_ParentScopeThrows() + { + // The ambient context is NOT thread safe, even though it has locks, etc... + // This all just goes to show that concurrent threads with scopes is a no-go. + var childWait = new ManualResetEventSlim(false); + var parentWait = new ManualResetEventSlim(false); + + ScopeProvider scopeProvider = ScopeProvider; + + Assert.IsNull(ScopeProvider.AmbientScope); + IScope mainScope = scopeProvider.CreateScope(); + + var t = Task.Run(() => + { + Console.WriteLine("Child Task start: " + scopeProvider.AmbientScope.InstanceId); + // This will evict the parent from the ScopeProvider.StaticCallContextObjects + // and replace it with the child + IScope nested = scopeProvider.CreateScope(); + childWait.Set(); + Console.WriteLine("Child Task scope created: " + scopeProvider.AmbientScope.InstanceId); + parentWait.Wait(); // wait for the parent thread + Console.WriteLine("Child Task before dispose: " + scopeProvider.AmbientScope.InstanceId); + // This will evict the child from the ScopeProvider.StaticCallContextObjects + // and replace it with the parent + nested.Dispose(); + Console.WriteLine("Child Task after dispose: " + scopeProvider.AmbientScope.InstanceId); + }); + + childWait.Wait(); // wait for the child to start and create the scope + // This is a confusing thing (this is not the case in netcore), this is NULL because the + // parent thread's scope ID was evicted from the ScopeProvider.StaticCallContextObjects + // so now the ambient context is null because the GUID in the CallContext doesn't match + // the GUID in the ScopeProvider.StaticCallContextObjects. + Assert.IsNull(scopeProvider.AmbientScope); + // now dispose the main without waiting for the child thread to join + // This will throw because at this stage a child scope has been created which means + // it is the Ambient (top) scope but here we're trying to dispose the non top scope. + Assert.Throws(() => mainScope.Dispose()); + parentWait.Set(); // tell child thread to proceed + Task.WaitAll(t); // wait for the child to dispose + mainScope.Dispose(); // now it's ok + Console.WriteLine("Parent Task disposed: " + scopeProvider.AmbientScope?.InstanceId); + } + + [Test] + public void GivenChildThread_WhenChildDisposedBeforeParent_OK() + { + ScopeProvider scopeProvider = ScopeProvider; + + Assert.IsNull(ScopeProvider.AmbientScope); + IScope mainScope = scopeProvider.CreateScope(); + + // Task.Run will flow the execution context unless ExecutionContext.SuppressFlow() is explicitly called. + // This is what occurs in normal async behavior since it is expected to await (and join) the main thread, + // but if Task.Run is used as a fire and forget thread without being done correctly then the Scope will + // flow to that thread. + var t = Task.Run(() => + { + Console.WriteLine("Child Task start: " + scopeProvider.AmbientScope.InstanceId); + IScope nested = scopeProvider.CreateScope(); + Console.WriteLine("Child Task before dispose: " + scopeProvider.AmbientScope.InstanceId); + nested.Dispose(); + Console.WriteLine("Child Task after disposed: " + scopeProvider.AmbientScope.InstanceId); + }); + + Console.WriteLine("Parent Task waiting: " + scopeProvider.AmbientScope?.InstanceId); + Task.WaitAll(t); + Console.WriteLine("Parent Task disposing: " + scopeProvider.AmbientScope.InstanceId); + mainScope.Dispose(); + Console.WriteLine("Parent Task disposed: " + scopeProvider.AmbientScope?.InstanceId); + + Assert.Pass(); + } + [Test] public void SimpleCreateScope() { diff --git a/src/Umbraco.Tests/Scoping/ScopeUnitTests.cs b/src/Umbraco.Tests/Scoping/ScopeUnitTests.cs index 32bd7e2afe..038376f71c 100644 --- a/src/Umbraco.Tests/Scoping/ScopeUnitTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopeUnitTests.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using Moq; using NPoco; using NUnit.Framework; @@ -9,6 +10,7 @@ using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.SqlSyntax; using Umbraco.Core.Scoping; +using Umbraco.Tests.TestHelpers; namespace Umbraco.Tests.Scoping { @@ -72,6 +74,30 @@ namespace Umbraco.Tests.Scoping syntaxProviderMock.Verify(x => x.WriteLock(It.IsAny(), Constants.Locks.Languages), Times.Once); } + [Test] + public void WriteLock_Acquired_Only_Once_When_InnerScope_Disposed() + { + var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + + using (var outerScope = scopeProvider.CreateScope()) + { + outerScope.WriteLock(Constants.Locks.Languages); + + using (var innerScope = scopeProvider.CreateScope()) + { + innerScope.WriteLock(Constants.Locks.Languages); + innerScope.WriteLock(Constants.Locks.ContentTree); + innerScope.Complete(); + } + + outerScope.WriteLock(Constants.Locks.ContentTree); + outerScope.Complete(); + } + + syntaxProviderMock.Verify(x => x.WriteLock(It.IsAny(), Constants.Locks.Languages), Times.Once); + syntaxProviderMock.Verify(x => x.WriteLock(It.IsAny(), Constants.Locks.ContentTree), Times.Once); + } + [Test] public void WriteLock_With_Timeout_Acquired_Only_Once_Per_Key(){ var scopeProvider = GetScopeProvider(out var syntaxProviderMock); @@ -176,31 +202,58 @@ namespace Umbraco.Tests.Scoping syntaxProviderMock.Verify(x => x.ReadLock(It.IsAny(), timeOut, Constants.Locks.Languages), Times.Once); } + [Test] + public void ReadLock_Acquired_Only_Once_When_InnerScope_Disposed() + { + var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + + using (var outerScope = scopeProvider.CreateScope()) + { + outerScope.ReadLock(Constants.Locks.Languages); + + using (var innerScope = scopeProvider.CreateScope()) + { + innerScope.ReadLock(Constants.Locks.Languages); + innerScope.ReadLock(Constants.Locks.ContentTree); + innerScope.Complete(); + } + + outerScope.ReadLock(Constants.Locks.ContentTree); + outerScope.Complete(); + } + + syntaxProviderMock.Verify(x => x.ReadLock(It.IsAny(), Constants.Locks.Languages), Times.Once); + syntaxProviderMock.Verify(x => x.ReadLock(It.IsAny(), Constants.Locks.ContentTree), Times.Once); + } + [Test] public void WriteLocks_Count_correctly_If_Lock_Requested_Twice_In_Scope() { var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + Guid innerscopeId; using (var outerscope = scopeProvider.CreateScope()) { var realOuterScope = (Scope) outerscope; outerscope.WriteLock(Constants.Locks.ContentTree); outerscope.WriteLock(Constants.Locks.ContentTree); - Assert.AreEqual(2, realOuterScope.WriteLocks[Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.WriteLocks[outerscope.InstanceId][Constants.Locks.ContentTree]); using (var innerScope = scopeProvider.CreateScope()) { + innerscopeId = innerScope.InstanceId; innerScope.WriteLock(Constants.Locks.ContentTree); innerScope.WriteLock(Constants.Locks.ContentTree); - Assert.AreEqual(4, realOuterScope.WriteLocks[Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.WriteLocks[outerscope.InstanceId][Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.WriteLocks[innerscopeId][Constants.Locks.ContentTree]); innerScope.WriteLock(Constants.Locks.Languages); innerScope.WriteLock(Constants.Locks.Languages); - Assert.AreEqual(2, realOuterScope.WriteLocks[Constants.Locks.Languages]); + Assert.AreEqual(2, realOuterScope.WriteLocks[innerScope.InstanceId][Constants.Locks.Languages]); innerScope.Complete(); } - Assert.AreEqual(0, realOuterScope.WriteLocks[Constants.Locks.Languages]); - Assert.AreEqual(2, realOuterScope.WriteLocks[Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.WriteLocks[realOuterScope.InstanceId][Constants.Locks.ContentTree]); + Assert.IsFalse(realOuterScope.WriteLocks.ContainsKey(innerscopeId)); outerscope.Complete(); } } @@ -209,27 +262,32 @@ namespace Umbraco.Tests.Scoping public void ReadLocks_Count_correctly_If_Lock_Requested_Twice_In_Scope() { var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + Guid innerscopeId; using (var outerscope = scopeProvider.CreateScope()) { var realOuterScope = (Scope) outerscope; outerscope.ReadLock(Constants.Locks.ContentTree); outerscope.ReadLock(Constants.Locks.ContentTree); - Assert.AreEqual(2, realOuterScope.ReadLocks[Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.ReadLocks[outerscope.InstanceId][Constants.Locks.ContentTree]); using (var innerScope = scopeProvider.CreateScope()) { + innerscopeId = innerScope.InstanceId; innerScope.ReadLock(Constants.Locks.ContentTree); innerScope.ReadLock(Constants.Locks.ContentTree); - Assert.AreEqual(4, realOuterScope.ReadLocks[Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.ReadLocks[outerscope.InstanceId][Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.ReadLocks[innerScope.InstanceId][Constants.Locks.ContentTree]); innerScope.ReadLock(Constants.Locks.Languages); innerScope.ReadLock(Constants.Locks.Languages); - Assert.AreEqual(2, realOuterScope.ReadLocks[Constants.Locks.Languages]); + Assert.AreEqual(2, realOuterScope.ReadLocks[innerScope.InstanceId][Constants.Locks.Languages]); innerScope.Complete(); } - Assert.AreEqual(0, realOuterScope.ReadLocks[Constants.Locks.Languages]); - Assert.AreEqual(2, realOuterScope.ReadLocks[Constants.Locks.ContentTree]); + Assert.AreEqual(2, realOuterScope.ReadLocks[outerscope.InstanceId][Constants.Locks.ContentTree]); + Assert.IsFalse(realOuterScope.ReadLocks.ContainsKey(innerscopeId)); + + outerscope.Complete(); } } @@ -238,51 +296,61 @@ namespace Umbraco.Tests.Scoping public void Nested_Scopes_WriteLocks_Count_Correctly() { var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + Guid innerScope1Id, innerScope2Id; - using (var outerScope = scopeProvider.CreateScope()) + using (var parentScope = scopeProvider.CreateScope()) { - var parentScope = (Scope) outerScope; - outerScope.WriteLock(Constants.Locks.ContentTree); - outerScope.WriteLock(Constants.Locks.ContentTypes); + var realParentScope = (Scope) parentScope; + parentScope.WriteLock(Constants.Locks.ContentTree); + parentScope.WriteLock(Constants.Locks.ContentTypes); - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.ContentTree], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.ContentTypes], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); using (var innerScope1 = scopeProvider.CreateScope()) { + innerScope1Id = innerScope1.InstanceId; innerScope1.WriteLock(Constants.Locks.ContentTree); innerScope1.WriteLock(Constants.Locks.ContentTypes); innerScope1.WriteLock(Constants.Locks.Languages); - Assert.AreEqual(2, parentScope.WriteLocks[Constants.Locks.ContentTree], $"childScope1 after locks acquired: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(2, parentScope.WriteLocks[Constants.Locks.ContentTypes], $"childScope1 after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.Languages], $"childScope1 after locks acquired: {nameof(Constants.Locks.Languages)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"innerScope1, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.ContentTree], $"innerScope1, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.Languages], $"innerScope1, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.Languages)}"); using (var innerScope2 = scopeProvider.CreateScope()) { + innerScope2Id = innerScope2.InstanceId; innerScope2.WriteLock(Constants.Locks.ContentTree); innerScope2.WriteLock(Constants.Locks.MediaTypes); - Assert.AreEqual(3, parentScope.WriteLocks[Constants.Locks.ContentTree], $"childScope2 after locks acquired: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(2, parentScope.WriteLocks[Constants.Locks.ContentTypes], $"childScope2 after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.Languages], $"childScope2 after locks acquired: {nameof(Constants.Locks.Languages)}"); - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.MediaTypes], $"childScope2 after locks acquired: {nameof(Constants.Locks.MediaTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"innerScope2, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"innerScope2, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.ContentTree], $"innerScope2, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.ContentTypes], $"innerScope2, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.Languages], $"innerScope2, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.Languages)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope2.InstanceId][Constants.Locks.ContentTree], $"innerScope2, innerScope2 instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope2.InstanceId][Constants.Locks.MediaTypes], $"innerScope2, innerScope2 instance, after locks acquired: {nameof(Constants.Locks.MediaTypes)}"); innerScope2.Complete(); } - Assert.AreEqual(2, parentScope.WriteLocks[Constants.Locks.ContentTree], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(2, parentScope.WriteLocks[Constants.Locks.ContentTypes], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.Languages], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.Languages)}"); - Assert.AreEqual(0, parentScope.WriteLocks[Constants.Locks.MediaTypes], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.MediaTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"innerScope1, parent instance, after innserScope2 disposed: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, parent instance, after innserScope2 disposed: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.ContentTree], $"innerScope1, innerScope1 instance, after innserScope2 disposed: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, innerScope1 instance, after innserScope2 disposed: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[innerScope1.InstanceId][Constants.Locks.Languages], $"innerScope1, innerScope1 instance, after innserScope2 disposed: {nameof(Constants.Locks.Languages)}"); + Assert.IsFalse(realParentScope.WriteLocks.ContainsKey(innerScope2Id)); innerScope1.Complete(); } - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.ContentTree], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(1, parentScope.WriteLocks[Constants.Locks.ContentTypes], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(0, parentScope.WriteLocks[Constants.Locks.Languages], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.Languages)}"); - Assert.AreEqual(0, parentScope.WriteLocks[Constants.Locks.MediaTypes], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.MediaTypes)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.WriteLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.ContentTypes)}"); + Assert.IsFalse(realParentScope.WriteLocks.ContainsKey(innerScope2Id)); + Assert.IsFalse(realParentScope.WriteLocks.ContainsKey(innerScope1Id)); - outerScope.Complete(); + parentScope.Complete(); } } @@ -290,48 +358,166 @@ namespace Umbraco.Tests.Scoping public void Nested_Scopes_ReadLocks_Count_Correctly() { var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + Guid innerScope1Id, innerScope2Id; - using (var outerScope = scopeProvider.CreateScope()) + using (var parentScope = scopeProvider.CreateScope()) { - var parentScope = (Scope) outerScope; - outerScope.ReadLock(Constants.Locks.ContentTree); - outerScope.ReadLock(Constants.Locks.ContentTypes); - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.ContentTree], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.ContentTypes], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + var realParentScope = (Scope) parentScope; + parentScope.ReadLock(Constants.Locks.ContentTree); + parentScope.ReadLock(Constants.Locks.ContentTypes); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"parentScope after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); using (var innserScope1 = scopeProvider.CreateScope()) { + innerScope1Id = innserScope1.InstanceId; innserScope1.ReadLock(Constants.Locks.ContentTree); innserScope1.ReadLock(Constants.Locks.ContentTypes); innserScope1.ReadLock(Constants.Locks.Languages); - Assert.AreEqual(2, parentScope.ReadLocks[Constants.Locks.ContentTree], $"childScope1 after locks acquired: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(2, parentScope.ReadLocks[Constants.Locks.ContentTypes], $"childScope1 after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.Languages], $"childScope1 after locks acquired: {nameof(Constants.Locks.Languages)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"innerScope1, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.ContentTree], $"innerScope1, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.Languages], $"innerScope1, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.Languages)}"); using (var innerScope2 = scopeProvider.CreateScope()) { + innerScope2Id = innerScope2.InstanceId; innerScope2.ReadLock(Constants.Locks.ContentTree); innerScope2.ReadLock(Constants.Locks.MediaTypes); - Assert.AreEqual(3, parentScope.ReadLocks[Constants.Locks.ContentTree], $"childScope2 after locks acquired: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(2, parentScope.ReadLocks[Constants.Locks.ContentTypes], $"childScope2 after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.Languages], $"childScope2 after locks acquired: {nameof(Constants.Locks.Languages)}"); - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.MediaTypes], $"childScope2 after locks acquired: {nameof(Constants.Locks.MediaTypes)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"innerScope2, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"innerScope2, parent instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.ContentTree], $"innerScope2, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.ContentTypes], $"innerScope2, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.Languages], $"innerScope2, innerScope1 instance, after locks acquired: {nameof(Constants.Locks.Languages)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innerScope2.InstanceId][Constants.Locks.ContentTree], $"innerScope2, innerScope2 instance, after locks acquired: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innerScope2.InstanceId][Constants.Locks.MediaTypes], $"innerScope2, innerScope2 instance, after locks acquired: {nameof(Constants.Locks.MediaTypes)}"); innerScope2.Complete(); } - Assert.AreEqual(2, parentScope.ReadLocks[Constants.Locks.ContentTree], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(2, parentScope.ReadLocks[Constants.Locks.ContentTypes], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.Languages], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.Languages)}"); - Assert.AreEqual(0, parentScope.ReadLocks[Constants.Locks.MediaTypes], $"childScope1 after inner scope disposed: {nameof(Constants.Locks.MediaTypes)}"); + + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"innerScope1, parent instance, after innerScope2 disposed: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, parent instance, after innerScope2 disposed: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.ContentTree], $"innerScope1, innerScope1 instance, after innerScope2 disposed: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.ContentTypes], $"innerScope1, innerScope1 instance, after innerScope2 disposed: {nameof(Constants.Locks.ContentTypes)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[innserScope1.InstanceId][Constants.Locks.Languages], $"innerScope1, innerScope1 instance, after innerScope2 disposed: {nameof(Constants.Locks.Languages)}"); + Assert.IsFalse(realParentScope.ReadLocks.ContainsKey(innerScope2Id)); innserScope1.Complete(); } - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.ContentTree], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.ContentTree)}"); - Assert.AreEqual(1, parentScope.ReadLocks[Constants.Locks.ContentTypes], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.ContentTypes)}"); - Assert.AreEqual(0, parentScope.ReadLocks[Constants.Locks.Languages], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.Languages)}"); - Assert.AreEqual(0, parentScope.ReadLocks[Constants.Locks.MediaTypes], $"parentScope after inner scopes disposed: {nameof(Constants.Locks.MediaTypes)}"); - outerScope.Complete(); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTree], $"parentScope after innerScope1 disposed: {nameof(Constants.Locks.ContentTree)}"); + Assert.AreEqual(1, realParentScope.ReadLocks[realParentScope.InstanceId][Constants.Locks.ContentTypes], $"parentScope after innerScope1 disposed: {nameof(Constants.Locks.ContentTypes)}"); + Assert.IsFalse(realParentScope.ReadLocks.ContainsKey(innerScope2Id)); + Assert.IsFalse(realParentScope.ReadLocks.ContainsKey(innerScope1Id)); + + parentScope.Complete(); + } + } + + [Test] + public void WriteLock_Doesnt_Increment_On_Error() + { + var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + syntaxProviderMock.Setup(x => x.WriteLock(It.IsAny(), It.IsAny())).Throws(new Exception("Boom")); + + using (var scope = scopeProvider.CreateScope()) + { + var realScope = (Scope) scope; + + Assert.Throws(() => scope.WriteLock(Constants.Locks.Languages)); + Assert.IsFalse(realScope.WriteLocks[scope.InstanceId].ContainsKey(Constants.Locks.Languages)); + scope.Complete(); + } + } + + [Test] + public void ReadLock_Doesnt_Increment_On_Error() + { + var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + syntaxProviderMock.Setup(x => x.ReadLock(It.IsAny(), It.IsAny())).Throws(new Exception("Boom")); + + using (var scope = scopeProvider.CreateScope()) + { + var realScope = (Scope) scope; + + Assert.Throws(() => scope.ReadLock(Constants.Locks.Languages)); + Assert.IsFalse(realScope.ReadLocks[scope.InstanceId].ContainsKey(Constants.Locks.Languages)); + scope.Complete(); + } + } + + [Test] + public void Scope_Throws_If_ReadLocks_Not_Cleared() + { + var scopeprovider = GetScopeProvider(out var syntaxProviderMock); + var scope = (Scope) scopeprovider.CreateScope(); + + try + { + // Request a lock to create the ReadLocks dict. + scope.ReadLock(Constants.Locks.Domains); + + var readDict = new Dictionary(); + readDict[Constants.Locks.Languages] = 1; + scope.ReadLocks[Guid.NewGuid()] = readDict; + + Assert.Throws(() => scope.Dispose()); + } + finally + { + // We have to clear so we can properly dispose the scope, otherwise it'll mess with other tests. + scope.ReadLocks?.Clear(); + scope.Dispose(); + } + } + + [Test] + public void Scope_Throws_If_WriteLocks_Not_Cleared() + { + var scopeprovider = GetScopeProvider(out var syntaxProviderMock); + var scope = (Scope) scopeprovider.CreateScope(); + + try + { + // Request a lock to create the WriteLocks dict. + scope.WriteLock(Constants.Locks.Domains); + + var writeDict = new Dictionary(); + writeDict[Constants.Locks.Languages] = 1; + scope.WriteLocks[Guid.NewGuid()] = writeDict; + + Assert.Throws(() => scope.Dispose()); + } + finally + { + // We have to clear so we can properly dispose the scope, otherwise it'll mess with other tests. + scope.WriteLocks?.Clear(); + scope.Dispose(); + } + } + + [Test] + public void WriteLocks_Not_Created_Until_First_Lock() + { + var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + + using (var scope = scopeProvider.CreateScope()) + { + var realScope = (Scope) scope; + Assert.IsNull(realScope.WriteLocks); + } + } + + [Test] + public void ReadLocks_Not_Created_Until_First_Lock() + { + var scopeProvider = GetScopeProvider(out var syntaxProviderMock); + + using (var scope = scopeProvider.CreateScope()) + { + var realScope = (Scope) scope; + Assert.IsNull(realScope.ReadLocks); } } } diff --git a/src/Umbraco.Tests/Services/ContentServiceTests.cs b/src/Umbraco.Tests/Services/ContentServiceTests.cs index 008c24fcbf..0faa4af316 100644 --- a/src/Umbraco.Tests/Services/ContentServiceTests.cs +++ b/src/Umbraco.Tests/Services/ContentServiceTests.cs @@ -2082,6 +2082,32 @@ namespace Umbraco.Tests.Services Assert.AreEqual("world", copiedTags[1].Text); } + [Test] + public void Copy_Recursive_Preserves_Sort_Order() + { + // Arrange + var contentService = ServiceContext.ContentService; + var temp = contentService.GetById(NodeDto.NodeIdSeed + 2); + Assert.AreEqual("Home", temp.Name); + Assert.AreEqual(3, contentService.CountChildren(temp.Id)); + var reversedChildren = contentService.GetPagedChildren(temp.Id, 0, 10, out var total1).Reverse().ToArray(); + contentService.Sort(reversedChildren); + + // Act + var copy = contentService.Copy(temp, temp.ParentId, false, true, Constants.Security.SuperUserId); + var content = contentService.GetById(NodeDto.NodeIdSeed + 2); + + // Assert + Assert.That(copy, Is.Not.Null); + Assert.That(copy.Id, Is.Not.EqualTo(content.Id)); + Assert.AreNotSame(content, copy); + Assert.AreEqual(3, contentService.CountChildren(copy.Id)); + + var copiedChildren = contentService.GetPagedChildren(copy.Id, 0, 10, out var total2).OrderBy(c => c.SortOrder).ToArray(); + Assert.AreEqual(reversedChildren.First().Name, copiedChildren.First().Name); + Assert.AreEqual(reversedChildren.Last().Name, copiedChildren.Last().Name); + } + [Test] public void Can_Rollback_Version_On_Content() { diff --git a/src/Umbraco.Tests/Services/Importing/StandardMvc-Package.xml b/src/Umbraco.Tests/Services/Importing/StandardMvc-Package.xml index ee6f7cea4a..daeb74cc75 100644 --- a/src/Umbraco.Tests/Services/Importing/StandardMvc-Package.xml +++ b/src/Umbraco.Tests/Services/Importing/StandardMvc-Package.xml @@ -210,7 +210,7 @@ Google Maps - A map macro that you can use within Rich Text Areas Built by Creative Founds

Web ApplicationsCreative Founds design and build first class software solutions that deliver big results. We provide ASP.NET web and mobile applications, Umbraco development service & technical consultancy.

-

www.creativefounds.co.uk

]]> +

www.creativefounds.co.uk

]]> Umbraco Development @@ -218,7 +218,7 @@ Google Maps - A map macro that you can use within Rich Text Areas Contact Us -

Contact Us on TwitterWe'd love to hear how this package has helped you and how it can be improved. Get in touch on the project website or via twitter

]]> +

Contact Us on TwitterWe'd love to hear how this package has helped you and how it can be improved. Get in touch on the project website or via twitter

]]>
Standard Website MVC, Company Address, Glasgow, Postcode
@@ -418,7 +418,7 @@ Google Maps - A map macro that you can use within Rich Text Areas Standard Website MVC

Well hello! This website package demonstrates all the standard functionality of Umbraco. It's a great starting point for starting point for further development or as a prototype.

Creative Founds

-

This package was developed by Chris Koiak & Creative Founds. 

]]> +

This package was developed by Chris Koiak & Creative Founds. 

]]> 1 diff --git a/src/Umbraco.Tests/Services/PerformanceTests.cs b/src/Umbraco.Tests/Services/PerformanceTests.cs index 9cf38e1789..718f99ce2f 100644 --- a/src/Umbraco.Tests/Services/PerformanceTests.cs +++ b/src/Umbraco.Tests/Services/PerformanceTests.cs @@ -297,11 +297,11 @@ namespace Umbraco.Tests.Services Built by Creative Founds

Web ApplicationsCreative Founds design and build first class software solutions that deliver big results. We provide ASP.NET web and mobile applications, Umbraco development service & technical consultancy.

-

www.creativefounds.co.uk

]]> +

www.creativefounds.co.uk

]]> Umbraco Development

UmbracoUmbraco the the leading ASP.NET open source CMS, under pinning over 150,000 websites. Our Certified Developers are experts in developing high performance and feature rich websites.

]]> Contact Us -

Contact Us on TwitterWe'd love to hear how this package has helped you and how it can be improved. Get in touch on the project website or via twitter

]]> +

Contact Us on TwitterWe'd love to hear how this package has helped you and how it can be improved. Get in touch on the project website or via twitter

]]>
Standard Website MVC, Company Address, Glasgow, Postcode
Copyright &copy; 2012 Your Company diff --git a/src/Umbraco.Tests/Services/ThreadSafetyServiceTest.cs b/src/Umbraco.Tests/Services/ThreadSafetyServiceTest.cs index 33ee2f737a..5c58b35b6d 100644 --- a/src/Umbraco.Tests/Services/ThreadSafetyServiceTest.cs +++ b/src/Umbraco.Tests/Services/ThreadSafetyServiceTest.cs @@ -104,7 +104,7 @@ namespace Umbraco.Tests.Services var threads = new List(); var exceptions = new List(); - Debug.WriteLine("Starting..."); + Console.WriteLine("Starting..."); var done = TraceLocks(); @@ -114,12 +114,12 @@ namespace Umbraco.Tests.Services { try { - Debug.WriteLine("[{0}] Running...", Thread.CurrentThread.ManagedThreadId); + Console.WriteLine("[{0}] Running...", Thread.CurrentThread.ManagedThreadId); var name1 = "test-" + Guid.NewGuid(); var content1 = contentService.Create(name1, -1, "umbTextpage"); - Debug.WriteLine("[{0}] Saving content #1.", Thread.CurrentThread.ManagedThreadId); + Console.WriteLine("[{0}] Saving content #1.", Thread.CurrentThread.ManagedThreadId); Save(contentService, content1); Thread.Sleep(100); //quick pause for maximum overlap! @@ -127,7 +127,7 @@ namespace Umbraco.Tests.Services var name2 = "test-" + Guid.NewGuid(); var content2 = contentService.Create(name2, -1, "umbTextpage"); - Debug.WriteLine("[{0}] Saving content #2.", Thread.CurrentThread.ManagedThreadId); + Console.WriteLine("[{0}] Saving content #2.", Thread.CurrentThread.ManagedThreadId); Save(contentService, content2); } catch (Exception e) @@ -139,16 +139,16 @@ namespace Umbraco.Tests.Services } // start all threads - Debug.WriteLine("Starting threads"); + Console.WriteLine("Starting threads"); threads.ForEach(x => x.Start()); // wait for all to complete - Debug.WriteLine("Joining threads"); + Console.WriteLine("Joining threads"); threads.ForEach(x => x.Join()); done.Set(); - Debug.WriteLine("Checking exceptions"); + Console.WriteLine("Checking exceptions"); if (exceptions.Count == 0) { //now look up all items, there should be 40! @@ -172,7 +172,7 @@ namespace Umbraco.Tests.Services var threads = new List(); var exceptions = new List(); - Debug.WriteLine("Starting..."); + Console.WriteLine("Starting..."); var done = TraceLocks(); @@ -182,18 +182,18 @@ namespace Umbraco.Tests.Services { try { - Debug.WriteLine("[{0}] Running...", Thread.CurrentThread.ManagedThreadId); + Console.WriteLine("[{0}] Running...", Thread.CurrentThread.ManagedThreadId); var name1 = "test-" + Guid.NewGuid(); var media1 = mediaService.CreateMedia(name1, -1, Constants.Conventions.MediaTypes.Folder); - Debug.WriteLine("[{0}] Saving media #1.", Thread.CurrentThread.ManagedThreadId); + Console.WriteLine("[{0}] Saving media #1.", Thread.CurrentThread.ManagedThreadId); Save(mediaService, media1); Thread.Sleep(100); //quick pause for maximum overlap! var name2 = "test-" + Guid.NewGuid(); var media2 = mediaService.CreateMedia(name2, -1, Constants.Conventions.MediaTypes.Folder); - Debug.WriteLine("[{0}] Saving media #2.", Thread.CurrentThread.ManagedThreadId); + Console.WriteLine("[{0}] Saving media #2.", Thread.CurrentThread.ManagedThreadId); Save(mediaService, media2); } catch (Exception e) diff --git a/src/Umbraco.Tests/Testing/TestingTests/MockTests.cs b/src/Umbraco.Tests/Testing/TestingTests/MockTests.cs index f53b0bfff0..7eacccc8d5 100644 --- a/src/Umbraco.Tests/Testing/TestingTests/MockTests.cs +++ b/src/Umbraco.Tests/Testing/TestingTests/MockTests.cs @@ -1,4 +1,5 @@ using System; +using System.Data; using System.Globalization; using System.Linq; using System.Web.Security; @@ -9,11 +10,13 @@ using Umbraco.Core.Cache; using Umbraco.Core.Composing; using Umbraco.Core.Configuration; using Umbraco.Core.Dictionary; +using Umbraco.Core.Events; using Umbraco.Core.Logging; using Umbraco.Core.Mapping; using Umbraco.Core.Models; using Umbraco.Core.Models.PublishedContent; using Umbraco.Core.Persistence; +using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.TestHelpers.Stubs; @@ -98,10 +101,21 @@ namespace Umbraco.Tests.Testing.TestingTests { var umbracoContext = TestObjects.GetUmbracoContextMock(); + var scopeProvider = new Mock(); + scopeProvider + .Setup(x => x.CreateScope( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Mock.Of); + var membershipHelper = new MembershipHelper(umbracoContext.HttpContext, Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of()); var umbracoHelper = new UmbracoHelper(Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), membershipHelper); - var umbracoMapper = new UmbracoMapper(new MapDefinitionCollection(new[] { Mock.Of() })); - + var umbracoMapper = new UmbracoMapper(new MapDefinitionCollection(new[] { Mock.Of() }), scopeProvider.Object); + // ReSharper disable once UnusedVariable var umbracoApiController = new FakeUmbracoApiController(Mock.Of(), Mock.Of(), Mock.Of(), ServiceContext.CreatePartial(), AppCaches.NoCache, Mock.Of(), Mock.Of(), umbracoHelper, umbracoMapper); diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umblogin.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umblogin.directive.js index 36eeb173d6..ea8b5e8173 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umblogin.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umblogin.directive.js @@ -16,7 +16,7 @@ function UmbLoginController($scope, $location, currentUserResource, formHelper, mediaHelper, umbRequestHelper, Upload, localizationService, userService, externalLoginInfo, externalLoginInfoService, - resetPasswordCodeInfo, $timeout, authResource, $q, $route) { + resetPasswordCodeInfo, authResource, $q) { const vm = this; @@ -72,6 +72,7 @@ vm.loginSubmit = loginSubmit; vm.requestPasswordResetSubmit = requestPasswordResetSubmit; vm.setPasswordSubmit = setPasswordSubmit; + vm.newPasswordKeyUp = newPasswordKeyUp; vm.labels = {}; localizationService.localizeMany([ vm.usernameIsEmail ? "general_email" : "general_username", @@ -362,6 +363,9 @@ }); } + function newPasswordKeyUp(event) { + vm.passwordVal = event.target.value; + } //// diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbpasswordtip.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbpasswordtip.directive.js new file mode 100644 index 0000000000..86e1d3d32f --- /dev/null +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/application/umbpasswordtip.directive.js @@ -0,0 +1,71 @@ +(function () { + 'use strict'; + + angular + .module('umbraco.directives') + .component('umbPasswordTip', { + controller: UmbPasswordTipController, + controllerAs: 'vm', + template: + '{{vm.passwordTip}}', + bindings: { + passwordVal: "<", + minPwdLength: "<", + minPwdNonAlphaNum: "<" + } + }); + + function UmbPasswordTipController(localizationService) { + + let defaultMinPwdLength = Umbraco.Sys.ServerVariables.umbracoSettings.minimumPasswordLength; + let defaultMinPwdNonAlphaNum = Umbraco.Sys.ServerVariables.umbracoSettings.minimumPasswordNonAlphaNum; + + var vm = this; + vm.$onInit = onInit; + vm.$onChanges = onChanges; + + function onInit() { + if (vm.minPwdLength === undefined) { + vm.minPwdLength = defaultMinPwdLength; + } + + if (vm.minPwdNonAlphaNum === undefined) { + vm.minPwdNonAlphaNum = defaultMinPwdNonAlphaNum; + } + + if (vm.minPwdNonAlphaNum > 0) { + localizationService.localize('user_newPasswordFormatNonAlphaTip', [vm.minPwdNonAlphaNum]).then(data => { + vm.passwordNonAlphaTip = data; + updatePasswordTip(0); + }); + } else { + vm.passwordNonAlphaTip = ''; + updatePasswordTip(0); + } + } + + function onChanges(simpleChanges) { + if (simpleChanges.passwordVal) { + if (simpleChanges.passwordVal.currentValue) { + updatePasswordTip(simpleChanges.passwordVal.currentValue.length); + } else { + updatePasswordTip(0); + } + } + } + + const updatePasswordTip = passwordLength => { + const remainingLength = vm.minPwdLength - passwordLength; + if (remainingLength > 0) { + localizationService.localize('user_newPasswordFormatLengthTip', [remainingLength]).then(data => { + vm.passwordTip = data; + if (vm.passwordNonAlphaTip) { + vm.passwordTip += `
${vm.passwordNonAlphaTip}`; + } + }); + } else { + vm.passwordTip = vm.passwordNonAlphaTip; + } + } + } +})(); diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/content/umbvariantcontenteditors.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/content/umbvariantcontenteditors.directive.js index 3e227bfcb3..71bf151b89 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/content/umbvariantcontenteditors.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/content/umbvariantcontenteditors.directive.js @@ -202,6 +202,9 @@ splitViewChanged(); unbindSplitViewRequest(); } + + // if split view was never closed, the listener is not disposed when changing nodes - this unbinds it + $scope.$on('$destroy', () => unbindSplitViewRequest()); /** * Changes the currently selected variant diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/prevententersubmit.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/prevententersubmit.directive.js index 355b02216f..62334387cb 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/prevententersubmit.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/prevententersubmit.directive.js @@ -17,7 +17,7 @@ angular.module("umbraco.directives") } $(element).on("keypress", function (event) { - if (event.which === 13) { + if (event.which === 13 && enabled === true) { event.preventDefault(); } }); diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbfocuslock.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbfocuslock.directive.js index 569f49b88a..f7cd32217e 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbfocuslock.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbfocuslock.directive.js @@ -29,10 +29,6 @@ var defaultFocusedElement = getAutoFocusElement(focusableElements); var firstFocusableElement = focusableElements[0]; var lastFocusableElement = focusableElements[focusableElements.length -1]; - - // We need to add the tabbing-active class in order to highlight the focused button since the default style is - // outline: none; set in the stylesheet specifically - bodyElement.classList.add('tabbing-active'); // If there is no default focused element put focus on the first focusable element in the nodelist if(defaultFocusedElement === null ){ diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbsearchfilter.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbsearchfilter.directive.js index efbc384cb4..2e9f15913c 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbsearchfilter.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/forms/umbsearchfilter.directive.js @@ -24,7 +24,7 @@ @param {boolean} model Set to true or false to set the checkbox to checked or unchecked. @param {string} inputId Set the id of the checkbox. @param {string} text Set the text for the checkbox label. -@param {string} labelKey Set a dictinary/localization string for the checkbox label +@param {string} labelKey Set a dictionary/localization string for the checkbox label @param {callback} onChange Callback when the value of the checkbox change by interaction. @param {boolean} autoFocus Add autofocus to the input field @param {boolean} preventSubmitOnEnter Set the enter prevent directive or not @@ -42,13 +42,15 @@ vm.change = change; function onInit() { - vm.inputId = vm.inputId || "umb-check_" + String.CreateGuid(); + vm.inputId = vm.inputId || "umb-search-filter_" + String.CreateGuid(); + vm.autoFocus = Object.toBoolean(vm.autoFocus) === true; + vm.preventSubmitOnEnter = Object.toBoolean(vm.preventSubmitOnEnter) === true; // If a labelKey is passed let's update the returned text if it's does not contain an opening square bracket [ if (vm.labelKey) { localizationService.localize(vm.labelKey).then(function (data) { - if(data.indexOf('[') === -1){ - vm.text = data; + if (data.indexOf('[') === -1){ + vm.text = data; } }); } diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/tabs/umbtabsnav.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/tabs/umbtabsnav.directive.js index b1c8608124..7a10ff51b5 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/tabs/umbtabsnav.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/tabs/umbtabsnav.directive.js @@ -11,24 +11,27 @@ Use this directive to render a tabs navigation. 
     

 
-        
-        
+        
+        
 
-        
-            

-                
Content of tab 1

-            

-            

-                
Content of tab 2

-            

-        
+            
+            
 
+            
+                

+                    
Content of tab 1

+                

+                

+                    
Content of tab 2

+                

+            
 
+
@@ -37,7 +40,7 @@ Use this directive to render a tabs navigation. (function () { "use strict"; - function Controller() { + function Controller(eventsService) { var vm = this; @@ -62,7 +65,7 @@ Use this directive to render a tabs navigation. selectedTab.active = true; }; - eventsService.on("tab.tabChange", function(name, args){ + eventsService.on("app.tabChange", function(name, args){ console.log("args", args); }); diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/tree/umbtreesearchbox.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/tree/umbtreesearchbox.directive.js index 3e2e7e362e..2ae17fdc6b 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/tree/umbtreesearchbox.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/tree/umbtreesearchbox.directive.js @@ -15,6 +15,7 @@ function treeSearchBox(localizationService, searchService, $q) { datatypeKey: "@", hideSearchCallback: "=", searchCallback: "=", + inputId: "@", autoFocus: "=" }, restrict: "E", // restrict to an element diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/users/changepassword.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/users/changepassword.directive.js index 8cbdabbf75..a9961a7579 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/users/changepassword.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/users/changepassword.directive.js @@ -11,6 +11,7 @@ vm.cancelChange = cancelChange; vm.showOldPass = showOldPass; vm.showCancelBtn = showCancelBtn; + vm.newPasswordKeyUp = newPasswordKeyUp; var unsubscribe = []; @@ -55,6 +56,11 @@ vm.config.minPasswordLength = 0; } + // Check non-alpha pwd settings for tooltip display + if (vm.config.minNonAlphaNumericChars === undefined) { + vm.config.minNonAlphaNumericChars = 0; + } + //set the model defaults if (!Utilities.isObject(vm.passwordValues)) { //if it's not an object then just create a new one @@ -152,6 +158,9 @@ return vm.config.disableToggle !== true && vm.config.hasPassword; }; + function newPasswordKeyUp(event) { + vm.passwordVal = event.target.value; + } } var component = { diff --git a/src/Umbraco.Web.UI.Client/src/common/mocks/services/localization.mocks.js b/src/Umbraco.Web.UI.Client/src/common/mocks/services/localization.mocks.js index ea7f3a6d4c..9a05e3cd7f 100644 --- a/src/Umbraco.Web.UI.Client/src/common/mocks/services/localization.mocks.js +++ b/src/Umbraco.Web.UI.Client/src/common/mocks/services/localization.mocks.js @@ -193,7 +193,7 @@ angular.module('umbraco.mocks'). "defaultdialogs_recycleBinDeleting": "The items in the recycle bin are now being deleted. Please do not close this window while this operation takes place", "defaultdialogs_recycleBinIsEmpty": "The recycle bin is now empty", "defaultdialogs_recycleBinWarning": "When items are deleted from the recycle bin, they will be gone forever", - "defaultdialogs_regexSearchError": "regexlib.com's webservice is currently experiencing some problems, which we have no control over. We are very sorry for this inconvenience.", + "defaultdialogs_regexSearchError": "regexlib.com's webservice is currently experiencing some problems, which we have no control over. We are very sorry for this inconvenience.", "defaultdialogs_regexSearchHelp": "Search for a regular expression to add validation to a form field. Exemple: 'email, 'zip-code' 'url'", "defaultdialogs_removeMacro": "Remove Macro", "defaultdialogs_requiredField": "Required Field", @@ -355,7 +355,7 @@ angular.module('umbraco.mocks'). "installer_databaseHeader": "Database configuration", "installer_databaseInstall": " Press the install button to install the Umbraco %0% database ", "installer_databaseInstallDone": "Umbraco %0% has now been copied to your database. Press Next to proceed.", - "installer_databaseNotFound": "

Database not found! Please check that the information in the 'connection string' of the \"web.config\" file is correct.

To proceed, please edit the 'web.config' file (using Visual Studio or your favourite text editor), scroll to the bottom, add the connection string for your database in the key named 'UmbracoDbDSN' and save the file.

Click the retry button when done.
More information on editing web.config here.

", + "installer_databaseNotFound": "

Database not found! Please check that the information in the 'connection string' of the \"web.config\" file is correct.

To proceed, please edit the 'web.config' file (using Visual Studio or your favourite text editor), scroll to the bottom, add the connection string for your database in the key named 'UmbracoDbDSN' and save the file.

Click the retry button when done.
More information on editing web.config here.

", "installer_databaseText": "To complete this step, you must know some information regarding your database server ('connection string').
Please contact your ISP if necessary. If you're installing on a local machine or server you might need information from your system administrator.", "installer_databaseUpgrade": "

Press the upgrade button to upgrade your database to Umbraco %0%

Don't worry - no content will be deleted and everything will continue working afterwards!

", "installer_databaseUpgradeDone": "Your database has been upgraded to the final version %0%.
Press Next to proceed. ", @@ -420,7 +420,7 @@ angular.module('umbraco.mocks'). "login_greeting6": "Happy friendly Friday", "login_greeting7": "Happy shiny Saturday", "login_instruction": "Log in below:", - "login_bottomText": "

© 2001 - %0%
Umbraco.org

", + "login_bottomText": "

© 2001 - %0%
Umbraco.org

", "main_dashboard": "Dashboard", "main_sections": "Sections", "main_tree": "Content", diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/contenttype.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/contenttype.resource.js index 6acf702546..a3a5b1946d 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/contenttype.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/contenttype.resource.js @@ -432,13 +432,15 @@ function contentTypeResource($q, $http, umbRequestHelper, umbDataFormatter, loca throw "args.id cannot be null"; } + var promise = localizationService.localize("contentType_moveFailed"); + return umbRequestHelper.resourcePromise( $http.post(umbRequestHelper.getApiUrl("contentTypeApiBaseUrl", "PostMove"), { parentId: args.parentId, id: args.id }, { responseType: 'text' }), - 'Failed to move content'); + promise); }, /** @@ -475,13 +477,15 @@ function contentTypeResource($q, $http, umbRequestHelper, umbDataFormatter, loca throw "args.id cannot be null"; } + var promise = localizationService.localize("contentType_copyFailed"); + return umbRequestHelper.resourcePromise( $http.post(umbRequestHelper.getApiUrl("contentTypeApiBaseUrl", "PostCopy"), { parentId: args.parentId, id: args.id }, { responseType: 'text' }), - 'Failed to copy content'); + promise); }, /** diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js index d194ae2c73..e3fab86067 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/mediatype.resource.js @@ -208,7 +208,7 @@ function mediaTypeResource($q, $http, umbRequestHelper, umbDataFormatter, locali throw "args.id cannot be null"; } - var promise = localizationService.localize("media_moveFailed"); + var promise = localizationService.localize("mediaType_moveFailed"); return umbRequestHelper.resourcePromise( $http.post(umbRequestHelper.getApiUrl("mediaTypeApiBaseUrl", "PostMove"), @@ -230,7 +230,7 @@ function mediaTypeResource($q, $http, umbRequestHelper, umbDataFormatter, locali throw "args.id cannot be null"; } - var promise = localizationService.localize("media_copyFailed"); + var promise = localizationService.localize("mediaType_copyFailed"); return umbRequestHelper.resourcePromise( $http.post(umbRequestHelper.getApiUrl("mediaTypeApiBaseUrl", "PostCopy"), diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js index 2314fa6d6c..bf02d9618e 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/membertype.resource.js @@ -3,7 +3,7 @@ * @name umbraco.resources.memberTypeResource * @description Loads in data for member types **/ -function memberTypeResource($q, $http, umbRequestHelper, umbDataFormatter) { +function memberTypeResource($q, $http, umbRequestHelper, umbDataFormatter, localizationService) { return { @@ -102,8 +102,29 @@ function memberTypeResource($q, $http, umbRequestHelper, umbDataFormatter) { return umbRequestHelper.resourcePromise( $http.post(umbRequestHelper.getApiUrl("memberTypeApiBaseUrl", "PostSave"), saveModel), 'Failed to save data for member type id ' + contentType.id); - } + }, + copy: function (args) { + if (!args) { + throw "args cannot be null"; + } + if (!args.parentId) { + throw "args.parentId cannot be null"; + } + if (!args.id) { + throw "args.id cannot be null"; + } + + var promise = localizationService.localize("memberType_copyFailed"); + + return umbRequestHelper.resourcePromise( + $http.post(umbRequestHelper.getApiUrl("memberTypeApiBaseUrl", "PostCopy"), + { + parentId: args.parentId, + id: args.id + }, { responseType: 'text' }), + promise); + } }; } angular.module('umbraco.resources').factory('memberTypeResource', memberTypeResource); diff --git a/src/Umbraco.Web.UI.Client/src/common/services/listviewhelper.service.js b/src/Umbraco.Web.UI.Client/src/common/services/listviewhelper.service.js index 14643dc9cd..f9ebba00ea 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/listviewhelper.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/listviewhelper.service.js @@ -572,13 +572,15 @@ * Method for opening an item in a list view for editing. * * @param {Object} item The item to edit + * @param {Object} scope The scope with options */ function editItem(item, scope) { + if (!item.editPath) { return; } - if (scope.options.useInfiniteEditor) + if (scope && scope.options && scope.options.useInfiniteEditor) { var editorModel = { id: item.id, diff --git a/src/Umbraco.Web.UI.Client/src/common/services/umbrequesthelper.service.js b/src/Umbraco.Web.UI.Client/src/common/services/umbrequesthelper.service.js index 0a4009264d..2b5447cdf6 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/umbrequesthelper.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/umbrequesthelper.service.js @@ -252,10 +252,10 @@ function umbRequestHelper($http, $q, notificationsService, eventsService, formHe //each item has a property alias and the file object, we'll ensure that the alias is suffixed to the key // so we know which property it belongs to on the server side var file = args.files[f]; - var fileKey = "file_" + file.alias + "_" + (file.culture ? file.culture : "") + "_" + (file.segment ? file.segment : ""); + var fileKey = "file_" + (file.alias || '').replace(/_/g, '\\_') + "_" + (file.culture ? file.culture.replace(/_/g, '\\_') : "") + "_" + (file.segment ? file.segment.replace(/_/g, '\\_') : ""); if (Utilities.isArray(file.metaData) && file.metaData.length > 0) { - fileKey += ("_" + file.metaData.join("_")); + fileKey += ("_" + _.map(file.metaData, x => ('' + x).replace(/_/g, '\\_')).join("_")); } formData.append(fileKey, file.file); } diff --git a/src/Umbraco.Web.UI.Client/src/installer/installer.service.js b/src/Umbraco.Web.UI.Client/src/installer/installer.service.js index 74858d652e..d3ab9e519c 100644 --- a/src/Umbraco.Web.UI.Client/src/installer/installer.service.js +++ b/src/Umbraco.Web.UI.Client/src/installer/installer.service.js @@ -19,8 +19,8 @@ angular.module("umbraco.install").factory('installerService', function ($rootSco "Over 500 000 websites are currently powered by Umbraco", "At least 2 people have named their cat 'Umbraco'", "On an average day more than 1000 people download Umbraco", - "umbraco.tv is the premier source of Umbraco video tutorials to get you started", - "You can find the world's friendliest CMS community at our.umbraco.com", + "umbraco.tv is the premier source of Umbraco video tutorials to get you started", + "You can find the world's friendliest CMS community at our.umbraco.com", "You can become a certified Umbraco developer by attending one of the official courses", "Umbraco works really well on tablets", "You have 100% control over your markup and design when crafting a website in Umbraco", @@ -30,7 +30,7 @@ angular.module("umbraco.install").factory('installerService', function ($rootSco "At least 4 people have the Umbraco logo tattooed on them", "'Umbraco' is the Danish name for an allen key", "Umbraco has been around since 2005, that's a looong time in IT", - "More than 700 people from all over the world meet each year in Denmark in May for our annual conference CodeGarden", + "More than 700 people from all over the world meet each year in Denmark in May for our annual conference CodeGarden", "While you are installing Umbraco someone else on the other side of the planet is probably doing it too", "You can extend Umbraco without modifying the source code using either JavaScript or C#", "Umbraco has been installed in more than 198 countries" diff --git a/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html b/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html index 472ceb7135..8ea69b3ee4 100644 --- a/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html +++ b/src/Umbraco.Web.UI.Client/src/installer/steps/upgrade.html @@ -10,7 +10,7 @@ To compare versions and read a report of changes between versions, use the View Report button below.

- View Report + View Report

Simply click continue below to be guided through the rest of the upgrade. diff --git a/src/Umbraco.Web.UI.Client/src/less/application/umb-outline.less b/src/Umbraco.Web.UI.Client/src/less/application/umb-outline.less index 939366d5ac..1f1c2c0e72 100644 --- a/src/Umbraco.Web.UI.Client/src/less/application/umb-outline.less +++ b/src/Umbraco.Web.UI.Client/src/less/application/umb-outline.less @@ -15,6 +15,7 @@ right: 0; border-radius: 3px; box-shadow: 0 0 2px 0px @ui-outline, inset 0 0 2px 2px @ui-outline; + pointer-events: none; } } diff --git a/src/Umbraco.Web.UI.Client/src/less/buttons.less b/src/Umbraco.Web.UI.Client/src/less/buttons.less index c446a02424..d1a426f818 100644 --- a/src/Umbraco.Web.UI.Client/src/less/buttons.less +++ b/src/Umbraco.Web.UI.Client/src/less/buttons.less @@ -65,15 +65,22 @@ // -------------------------------------------------- .btn-reset { - padding: 0; - margin: 0; - border: none; + padding: 0; + margin: 0; + border: none; background: none; - color: currentColor; + color: currentColor; font-family: @baseFontFamily; font-size: @baseFontSize; line-height: @baseLineHeight; - cursor: pointer; + cursor: pointer; + + // Disabled state + &.disabled, + &[disabled], + &:disabled:hover { + cursor: default; + } } // Button Sizes diff --git a/src/Umbraco.Web.UI.Client/src/less/components/editor/umb-variant-switcher.less b/src/Umbraco.Web.UI.Client/src/less/components/editor/umb-variant-switcher.less index 9d2782f184..ce9286e5f5 100644 --- a/src/Umbraco.Web.UI.Client/src/less/components/editor/umb-variant-switcher.less +++ b/src/Umbraco.Web.UI.Client/src/less/components/editor/umb-variant-switcher.less @@ -69,7 +69,6 @@ button.umb-variant-switcher__toggle { .umb-variant-switcher__expand { color: @ui-action-discreet-type; - margin-top: 3px; margin-left: 5px; margin-right: -5px; transition: color 0.2s ease-in-out; diff --git a/src/Umbraco.Web.UI.Client/src/less/components/umb-search-filter.less b/src/Umbraco.Web.UI.Client/src/less/components/umb-search-filter.less index b96d3e8569..b38f5937c7 100644 --- a/src/Umbraco.Web.UI.Client/src/less/components/umb-search-filter.less +++ b/src/Umbraco.Web.UI.Client/src/less/components/umb-search-filter.less @@ -21,11 +21,12 @@ html .umb-search-filter { // "icon-search" class it kept for backward compatibility .umb-icon, .icon-search { - color: #d8d7d9; + color: @gray-8; position: absolute; top: 0; bottom: 0; left: 10px; margin: auto 0; + pointer-events: none; } } diff --git a/src/Umbraco.Web.UI.Client/src/less/forms.less b/src/Umbraco.Web.UI.Client/src/less/forms.less index f38ba8f806..3782fca695 100644 --- a/src/Umbraco.Web.UI.Client/src/less/forms.less +++ b/src/Umbraco.Web.UI.Client/src/less/forms.less @@ -308,7 +308,14 @@ select[size] { input[type="file"], input[type="radio"], input[type="checkbox"] { - .umb-outline(); + &:focus { + border-color: @inputBorderFocus; + outline: 0; + + .tabbing-active & { + outline: 2px solid @ui-outline; + } + } } @@ -582,19 +589,21 @@ table.domains .help-inline { } } .add-on { - display: inline-block; + display: inline-flex; + align-items: center; + justify-content: center; width: auto; height: 22px; min-width: 18px; - padding: 5px 6px 3px 6px; + padding: 4px 6px; font-size: @baseFontSize; font-weight: normal; line-height: @baseLineHeight; text-align: center; - //text-shadow: 0 1px 0 @white; background-color: @white; border: 1px solid @inputBorder; color: @ui-option-type; + &:hover { border-color:@inputBorderFocus; color: @ui-option-type-hover; diff --git a/src/Umbraco.Web.UI.Client/src/less/main.less b/src/Umbraco.Web.UI.Client/src/less/main.less index 31bb8484c4..66afbfd73f 100644 --- a/src/Umbraco.Web.UI.Client/src/less/main.less +++ b/src/Umbraco.Web.UI.Client/src/less/main.less @@ -402,6 +402,10 @@ table thead button:focus{ /* UI interactions */ +.ui-sortable-handle { + cursor: move; +} + .umb-table tbody.ui-sortable tr { cursor:pointer; diff --git a/src/Umbraco.Web.UI.Client/src/less/panel.less b/src/Umbraco.Web.UI.Client/src/less/panel.less index a036267c85..cc87a0edf5 100644 --- a/src/Umbraco.Web.UI.Client/src/less/panel.less +++ b/src/Umbraco.Web.UI.Client/src/less/panel.less @@ -349,6 +349,7 @@ .umb-panel-header-icon { cursor: pointer; + font-size: 2rem; margin-right: 5px; margin-top: -6px; height: 50px; diff --git a/src/Umbraco.Web.UI.Client/src/less/property-editors.less b/src/Umbraco.Web.UI.Client/src/less/property-editors.less index 0d8f270f1b..f5e652aa3d 100644 --- a/src/Umbraco.Web.UI.Client/src/less/property-editors.less +++ b/src/Umbraco.Web.UI.Client/src/less/property-editors.less @@ -838,22 +838,25 @@ // // Date/time picker // -------------------------------------------------- -.bootstrap-datetimepicker-widget .btn{padding: 0;} -.bootstrap-datetimepicker-widget .picker-switch .btn{ background: none; border: none;} -.umb-datepicker .input-append .add-on{cursor: pointer;} -.umb-datepicker .input-append .on-top { - border: 0 none; +.bootstrap-datetimepicker-widget .btn {padding: 0;} +.bootstrap-datetimepicker-widget .picker-switch .btn { background: none; border: none;} +.umb-datepicker .input-append .btn-clear { + border: none; position: absolute; margin-left: -31px; margin-top: 1px; - display: inline-block; - padding: 5px 6px 3px 6px; + display: inline-flex; + align-items: center; + justify-content: center; + height: 30px; + padding: 4px 6px; font-size: @baseFontSize; font-weight: normal; line-height: @baseLineHeight; text-align: center; background-color: @white; color: @ui-option-type; + &:hover { color: @ui-option-type-hover; } diff --git a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/linkpicker/linkpicker.html b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/linkpicker/linkpicker.html index 093e69b5ed..0a2a8223e9 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/linkpicker/linkpicker.html +++ b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/linkpicker/linkpicker.html @@ -16,7 +16,7 @@

- + + ng-disabled="model.target.id || model.target.udi" + id="urlLinkPicker"/> - + + ng-model="model.target.anchor" + id="anchor"/> @@ -41,19 +43,21 @@
- + + ng-model="model.target.name" + id="nodeNameLinkPicker"/> - + + text="{{vm.labels.openInNewWindow}}" + input-id="openInNewWindow"> @@ -61,36 +65,35 @@
Link to page
- + +
- - +
- +
diff --git a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.controller.js b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.controller.js index 40338f2dca..1701553efc 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.controller.js @@ -1,12 +1,13 @@ function MacroPickerController($scope, entityResource, macroResource, umbPropEditorHelper, macroService, formHelper, localizationService) { $scope.macros = []; + $scope.a11yInfo = ""; $scope.model.selectedMacro = null; $scope.model.macroParams = []; - + $scope.displayA11YMessageForFilter = displayA11YMessageForFilter; $scope.wizardStep = "macroSelect"; $scope.noMacroParams = false; - + $scope.model.searchTerm = ""; function onInit() { if (!$scope.model.title) { localizationService.localize("defaultdialogs_selectMacro").then(function (value) { @@ -49,6 +50,7 @@ function MacroPickerController($scope, entityResource, macroResource, umbPropEdi $scope.model.submit($scope.model); } else { $scope.wizardStep = 'macroSelect'; + displayA11yMessages($scope.macros); } } else { @@ -95,6 +97,28 @@ function MacroPickerController($scope, entityResource, macroResource, umbPropEdi }); } + function displayA11yMessages(macros) { + if ($scope.noMacroParams || !macros || macros.length === 0) + localizationService.localize("general_searchNoResult").then(function (value) { + $scope.a11yInfo = value; + }); + else if (macros) { + if (macros.length === 1) { + localizationService.localize("treeSearch_searchResult").then(function(value) { + $scope.a11yInfo = "1 " + value; + }); + } else { + localizationService.localize("treeSearch_searchResults").then(function (value) { + $scope.a11yInfo = macros.length + " " + value; + }); + } + } + } + + function displayA11YMessageForFilter() { + var macros = _.filter($scope.macros, v => v.name.toLowerCase().includes($scope.model.searchTerm.toLowerCase())); + displayA11yMessages(macros); + } //here we check to see if we've been passed a selected macro and if so we'll set the //editor to start with parameter editing if ($scope.model.dialogData && $scope.model.dialogData.macroData) { @@ -141,10 +165,11 @@ function MacroPickerController($scope, entityResource, macroResource, umbPropEdi //we don't have a pre-selected macro so ensure the correct step is set $scope.wizardStep = 'macroSelect'; } + displayA11yMessages($scope.macros); }); onInit(); - + } angular.module("umbraco").controller("Umbraco.Overlays.MacroPickerController", MacroPickerController); diff --git a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.html b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.html index 33d7a471a5..8bda49b328 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.html +++ b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/macropicker/macropicker.html @@ -16,18 +16,18 @@
- - + + - +

    -
  • +
+ position="center"> There are no macros available to insert @@ -53,7 +53,7 @@
  • - + diff --git a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/userpicker/userpicker.controller.js b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/userpicker/userpicker.controller.js index a7021b2867..33d526c3cf 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/userpicker/userpicker.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/userpicker/userpicker.controller.js @@ -1,8 +1,8 @@ (function () { "use strict"; - function UserPickerController($scope, usersResource, localizationService, eventsService) { - + function UserPickerController($scope, entityResource, localizationService, eventsService) { + var vm = this; vm.users = []; @@ -102,17 +102,9 @@ vm.loading = true; // Get users - usersResource.getPagedResults(vm.usersOptions).then(function (users) { - - vm.users = users.items; - - vm.usersOptions.pageNumber = users.pageNumber; - vm.usersOptions.pageSize = users.pageSize; - vm.usersOptions.totalItems = users.totalItems; - vm.usersOptions.totalPages = users.totalPages; - + entityResource.getAll("User").then(function (data) { + vm.users = data; preSelect($scope.model.selection, vm.users); - vm.loading = false; }); } diff --git a/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html b/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html index 1e570b4af6..c4b6a4a2ed 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/application/umb-login.html @@ -226,7 +226,8 @@
    - + +
    diff --git a/src/Umbraco.Web.UI.Client/src/views/components/editor/umb-editor-content-header.html b/src/Umbraco.Web.UI.Client/src/views/components/editor/umb-editor-content-header.html index 6978672e99..6e33633512 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/editor/umb-editor-content-header.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/editor/umb-editor-content-header.html @@ -96,6 +96,7 @@
    diff --git a/src/Umbraco.Web.UI.Client/src/views/components/forms/umb-search-filter.html b/src/Umbraco.Web.UI.Client/src/views/components/forms/umb-search-filter.html index d6fde29090..ab21654f91 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/forms/umb-search-filter.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/forms/umb-search-filter.html @@ -2,27 +2,14 @@
    - - - +
    diff --git a/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree-search-box.html b/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree-search-box.html index 77498cd007..054472e4b6 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree-search-box.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree-search-box.html @@ -2,6 +2,7 @@ - - Sorry, we can not find what you are looking for. - - -
      -
    • -
        -
      • -
        - -
        + +

        Sorry, we can not find what you are looking for.

        +         +

        1 item returned

        +

        {{results.length}} items returned

        +
          +
        • +
            +
          • +
            + +
            +
            • +
          • -
        -
        • -
      +
    • diff --git a/src/Umbraco.Web.UI.Client/src/views/components/users/change-password.html b/src/Umbraco.Web.UI.Client/src/views/components/users/change-password.html index 974f8d6b4e..273f56d256 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/users/change-password.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/users/change-password.html @@ -43,12 +43,14 @@ required val-server-field="password" ng-minlength="{{vm.config.minPasswordLength}}" - no-dirty-check /> + no-dirty-check + ng-keyup="vm.newPasswordKeyUp($event)"/> Required Minimum {{vm.config.minPasswordLength}} characters {{changePasswordForm.password.errorMsg}} + diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/media/mediadashboardvideos.html b/src/Umbraco.Web.UI.Client/src/views/dashboard/media/mediadashboardvideos.html index 96d6a3f40a..670fae2f6e 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/media/mediadashboardvideos.html +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/media/mediadashboardvideos.html @@ -1,5 +1,5 @@

    Hours of Umbraco training videos are only a click away

    -

    Want to master Umbraco? Spend a couple of minutes learning some best practices by watching one of these videos about using Umbraco. And visit umbraco.tv for even more Umbraco videos

    +

    Want to master Umbraco? Spend a couple of minutes learning some best practices by watching one of these videos about using Umbraco. And visit umbraco.tv for even more Umbraco videos

    - + {{key}} - {{val}} + {{values | umbCmsJoinArray:', '}} diff --git a/src/Umbraco.Web.UI.Client/src/views/documenttypes/copy.html b/src/Umbraco.Web.UI.Client/src/views/documenttypes/copy.html index 8b81462ad5..824527be34 100644 --- a/src/Umbraco.Web.UI.Client/src/views/documenttypes/copy.html +++ b/src/Umbraco.Web.UI.Client/src/views/documenttypes/copy.html @@ -20,7 +20,7 @@
    {{source.name}} was copied underneath {{target.name}}
    - +
    diff --git a/src/Umbraco.Web.UI.Client/src/views/documenttypes/export.html b/src/Umbraco.Web.UI.Client/src/views/documenttypes/export.html index 0b9feb3fb6..b9aa4d9d3a 100644 --- a/src/Umbraco.Web.UI.Client/src/views/documenttypes/export.html +++ b/src/Umbraco.Web.UI.Client/src/views/documenttypes/export.html @@ -1,8 +1,8 @@
    - + diff --git a/src/Umbraco.Web.UI.Client/src/views/documenttypes/importdocumenttype.html b/src/Umbraco.Web.UI.Client/src/views/documenttypes/importdocumenttype.html index 8f84a0cceb..9ddc0b7c93 100644 --- a/src/Umbraco.Web.UI.Client/src/views/documenttypes/importdocumenttype.html +++ b/src/Umbraco.Web.UI.Client/src/views/documenttypes/importdocumenttype.html @@ -21,15 +21,13 @@ or - +
    Please choose a .udt file to import
    - -
    diff --git a/src/Umbraco.Web.UI.Client/src/views/documenttypes/move.html b/src/Umbraco.Web.UI.Client/src/views/documenttypes/move.html index fe0bde7f1f..b623b6131d 100644 --- a/src/Umbraco.Web.UI.Client/src/views/documenttypes/move.html +++ b/src/Umbraco.Web.UI.Client/src/views/documenttypes/move.html @@ -20,7 +20,7 @@
    {{source.name}} was moved underneath {{target.name}}
    - +
    diff --git a/src/Umbraco.Web.UI.Client/src/views/mediatypes/copy.html b/src/Umbraco.Web.UI.Client/src/views/mediatypes/copy.html index 9c21f623b5..58968c9dfa 100644 --- a/src/Umbraco.Web.UI.Client/src/views/mediatypes/copy.html +++ b/src/Umbraco.Web.UI.Client/src/views/mediatypes/copy.html @@ -20,7 +20,7 @@
    {{source.name}} was copied underneath {{target.name}}
    - +
    diff --git a/src/Umbraco.Web.UI.Client/src/views/mediatypes/move.html b/src/Umbraco.Web.UI.Client/src/views/mediatypes/move.html index 5225a41a0d..6bb1b6fa10 100644 --- a/src/Umbraco.Web.UI.Client/src/views/mediatypes/move.html +++ b/src/Umbraco.Web.UI.Client/src/views/mediatypes/move.html @@ -20,7 +20,7 @@
    {{source.name}} was moved underneath {{target.name}}
    - +
    diff --git a/src/Umbraco.Web.UI.Client/src/views/membertypes/copy.controller.js b/src/Umbraco.Web.UI.Client/src/views/membertypes/copy.controller.js new file mode 100644 index 0000000000..aa94b4bd04 --- /dev/null +++ b/src/Umbraco.Web.UI.Client/src/views/membertypes/copy.controller.js @@ -0,0 +1,61 @@ +angular.module("umbraco") + .controller("Umbraco.Editors.MemberTypes.CopyController", + function ($scope, memberTypeResource, treeService, navigationService, notificationsService, appState, eventsService) { + + $scope.dialogTreeApi = {}; + $scope.source = _.clone($scope.currentNode); + + function nodeSelectHandler(args) { + args.event.preventDefault(); + args.event.stopPropagation(); + + if ($scope.target) { + //un-select if there's a current one selected + $scope.target.selected = false; + } + + $scope.target = args.node; + $scope.target.selected = true; + } + + $scope.copy = function () { + + $scope.busy = true; + $scope.error = false; + + memberTypeResource.copy({ parentId: $scope.target.id, id: $scope.source.id }) + .then(function (path) { + $scope.error = false; + $scope.success = true; + $scope.busy = false; + + //get the currently edited node (if any) + var activeNode = appState.getTreeState("selectedNode"); + + //we need to do a double sync here: first sync to the copied content - but don't activate the node, + //then sync to the currenlty edited content (note: this might not be the content that was copied!!) + + navigationService.syncTree({ tree: "memberTypes", path: path, forceReload: true, activate: false }).then(function (args) { + if (activeNode) { + var activeNodePath = treeService.getPath(activeNode).join(); + //sync to this node now - depending on what was copied this might already be synced but might not be + navigationService.syncTree({ tree: "memberTypes", path: activeNodePath, forceReload: false, activate: true }); + } + }); + + }, function (err) { + $scope.success = false; + $scope.error = err; + $scope.busy = false; + }); + }; + + $scope.onTreeInit = function () { + $scope.dialogTreeApi.callbacks.treeNodeSelect(nodeSelectHandler); + }; + + $scope.close = function() { + navigationService.hideDialog(); + }; + + }); diff --git a/src/Umbraco.Web.UI.Client/src/views/membertypes/copy.html b/src/Umbraco.Web.UI.Client/src/views/membertypes/copy.html new file mode 100644 index 0000000000..fb7c6b5584 --- /dev/null +++ b/src/Umbraco.Web.UI.Client/src/views/membertypes/copy.html @@ -0,0 +1,53 @@ +
    + +
    +
    + +

    + Select the folder to copy {{source.name}} to in the tree structure below +

    + + + +
    +
    +
    {{error.errorMsg}}
    +
    {{error.data.message}}
    +
    +
    + +
    +
    + {{source.name}} was copied underneath {{target.name}} +
    + +
    + +
    + +
    + + +
    + +
    +
    +
    + +
    + + +
    +
    diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/changepassword/changepassword.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/changepassword/changepassword.controller.js index ab7f5c66e0..e80aad64f9 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/changepassword/changepassword.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/changepassword/changepassword.controller.js @@ -36,6 +36,9 @@ angular.module("umbraco").controller("Umbraco.PropertyEditors.ChangePasswordCont if (!$scope.model.config || $scope.model.config.minPasswordLength === undefined) { $scope.model.config.minPasswordLength = 0; } + if (!$scope.model.config || $scope.model.config.minNonAlphaNumericChars === undefined) { + $scope.model.config.minNonAlphaNumericChars = 0; + } //set the model defaults if (!Utilities.isObject($scope.model.value)) { diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/datepicker/datepicker.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/datepicker/datepicker.html index 9501a6631b..f5ac69b9b8 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/datepicker/datepicker.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/datepicker/datepicker.html @@ -19,12 +19,12 @@ ng-required="model.validation.mandatory" val-server="value" class="datepickerinput" /> - -
    @@ -32,7 +32,7 @@
    -
    +

    {{mandatoryMessage}}

    {{datePickerForm.datepicker.errorMsg}}

    Invalid date

    diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/email/email.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/email/email.html index 26ec22df8d..3ae03a2d7b 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/email/email.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/email/email.html @@ -1,6 +1,7 @@
    - -
    +

    {{mandatoryMessage}}

    Invalid email

    {{emailFieldForm.textbox.errorMsg}}

    diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js index 716ca405c1..94ea4b8604 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/media.controller.js @@ -1,9 +1,9 @@ angular.module("umbraco") .controller("Umbraco.PropertyEditors.Grid.MediaController", - function ($scope, userService, editorService, localizationService) { - - $scope.thumbnailUrl = getThumbnailUrl(); - + function ($scope, userService, editorService, localizationService) { + + $scope.thumbnailUrl = getThumbnailUrl(); + if (!$scope.model.config.startNodeId) { if ($scope.model.config.ignoreUserStartNodes === true) { $scope.model.config.startNodeId = -1; @@ -29,16 +29,16 @@ angular.module("umbraco") onlyImages: true, dataTypeKey: $scope.model.dataTypeKey, submit: model => { - updateControlValue(model.selection[0]); + updateControlValue(model.selection[0]); editorService.close(); }, - close: () => editorService.close() + close: () => editorService.close() }; editorService.mediaPicker(mediaPicker); }; - $scope.editImage = function() { + $scope.editImage = function() { const mediaCropDetailsConfig = { size: 'small', @@ -47,17 +47,17 @@ angular.module("umbraco") updateControlValue(model.target); editorService.close(); }, - close: () => editorService.close() + close: () => editorService.close() }; localizationService.localize('defaultdialogs_editSelectedMedia').then(value => { mediaCropDetailsConfig.title = value; editorService.mediaCropDetails(mediaCropDetailsConfig); - }); + }); } - + /** - * + * */ function getThumbnailUrl() { @@ -94,19 +94,15 @@ angular.module("umbraco") return url; } - + return null; } /** - * - * @param {object} selectedImage + * + * @param {object} selectedImage */ function updateControlValue(selectedImage) { - - const doGetThumbnail = $scope.control.value.focalPoint !== selectedImage.focalPoint - || $scope.control.value.image !== selectedImage.image; - // we could apply selectedImage directly to $scope.control.value, // but this allows excluding fields in future if needed $scope.control.value = { @@ -118,10 +114,6 @@ angular.module("umbraco") caption: selectedImage.caption, altText: selectedImage.altText }; - - - if (doGetThumbnail) { - $scope.thumbnailUrl = getThumbnailUrl(); - } - } + $scope.thumbnailUrl = getThumbnailUrl(); + } }); diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js index 50146a4c36..15f5ceaa88 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js @@ -141,9 +141,9 @@ angular.module("umbraco") over: function (event, ui) { var area = event.target.getScope_HackForSortable().area; - var allowedEditors = area.allowed; + var allowedEditors = area.$allowedEditors.map(e => e.alias); - if (($.inArray(ui.item[0].getScope_HackForSortable().control.editor.alias, allowedEditors) < 0 && allowedEditors) || + if (($.inArray(ui.item[0].getScope_HackForSortable().control.editor.alias, allowedEditors) < 0) || (startingArea != area && area.maxItems != '' && area.maxItems > 0 && area.maxItems < area.controls.length + 1)) { $scope.$apply(function () { diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/integer/integer.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/integer/integer.html index 5c10790400..24a8c33696 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/integer/integer.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/integer/integer.html @@ -9,10 +9,15 @@ aria-required="{{model.validation.mandatory}}" id="{{model.alias}}" val-server="value" - fix-number min="{{model.config.min}}" max="{{model.config.max}}" step="{{model.config.step}}" /> + min="{{model.config.min}}" + max="{{model.config.max}}" + step="{{model.config.step}}" + ng-step="model.config.step" + fix-number /> - + Not a number + Not a valid numeric step size {{integerFieldForm.integerField.errorMsg}} diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/includeproperties.prevalues.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/includeproperties.prevalues.html index 42597f0c82..7d863f6730 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/includeproperties.prevalues.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/includeproperties.prevalues.html @@ -27,7 +27,7 @@ - +
    @@ -54,7 +54,7 @@ diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts.prevalues.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts.prevalues.html index 4527458d16..295345a827 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts.prevalues.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts.prevalues.html @@ -6,20 +6,20 @@
    - +
    - -
    - -
    -
    @@ -32,12 +32,12 @@
    -
    +
    (blank) -

    Be a part of the community

    The Umbraco community is the best of its kind, be sure to visit, and if you have any questions, we're sure that you can get your answers from the community.

    - our.Umbraco → + our.Umbraco →
    diff --git a/src/Umbraco.Web.UI/web.Template.config b/src/Umbraco.Web.UI/web.Template.config index 03f462fb9e..ae141e5408 100644 --- a/src/Umbraco.Web.UI/web.Template.config +++ b/src/Umbraco.Web.UI/web.Template.config @@ -37,6 +37,7 @@ + diff --git a/src/Umbraco.Web/Dashboards/ContentDashboard.cs b/src/Umbraco.Web/Dashboards/ContentDashboard.cs index 0cd96f738c..260eb8baf9 100644 --- a/src/Umbraco.Web/Dashboards/ContentDashboard.cs +++ b/src/Umbraco.Web/Dashboards/ContentDashboard.cs @@ -1,15 +1,21 @@ -using Umbraco.Core; +using System.Collections.Generic; +using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.Dashboards; +using Umbraco.Core.Services; namespace Umbraco.Web.Dashboards { [Weight(10)] public class ContentDashboard : IDashboard { + private readonly IContentDashboardSettings _dashboardSettings; + private readonly IUserService _userService; + private IAccessRule[] _accessRulesFromConfig; + public string Alias => "contentIntro"; - public string[] Sections => new [] { "content" }; + public string[] Sections => new[] { "content" }; public string View => "views/dashboard/default/startupdashboardintro.html"; @@ -17,13 +23,54 @@ namespace Umbraco.Web.Dashboards { get { - var rules = new IAccessRule[] + var rules = AccessRulesFromConfig; + + if (rules.Length == 0) { - new AccessRule {Type = AccessRuleType.Deny, Value = Constants.Security.TranslatorGroupAlias}, - new AccessRule {Type = AccessRuleType.Grant, Value = Constants.Security.AdminGroupAlias} - }; + rules = new IAccessRule[] + { + new AccessRule {Type = AccessRuleType.Deny, Value = Constants.Security.TranslatorGroupAlias}, + new AccessRule {Type = AccessRuleType.Grant, Value = Constants.Security.AdminGroupAlias} + }; + } + return rules; } } + + private IAccessRule[] AccessRulesFromConfig + { + get + { + if (_accessRulesFromConfig is null) + { + var rules = new List(); + + if (_dashboardSettings.AllowContentDashboardAccessToAllUsers) + { + var allUserGroups = _userService.GetAllUserGroups(); + + foreach (var userGroup in allUserGroups) + { + rules.Add(new AccessRule + { + Type = AccessRuleType.Grant, + Value = userGroup.Alias + }); + } + } + + _accessRulesFromConfig = rules.ToArray(); + } + + return _accessRulesFromConfig; + } + } + + public ContentDashboard(IContentDashboardSettings dashboardSettings, IUserService userService) + { + _dashboardSettings = dashboardSettings; + _userService = userService; + } } } diff --git a/src/Umbraco.Web/Editors/BackOfficeServerVariables.cs b/src/Umbraco.Web/Editors/BackOfficeServerVariables.cs index 42b5186c03..6ec9ac4f90 100644 --- a/src/Umbraco.Web/Editors/BackOfficeServerVariables.cs +++ b/src/Umbraco.Web/Editors/BackOfficeServerVariables.cs @@ -57,7 +57,7 @@ namespace Umbraco.Web.Editors var keepOnlyKeys = new Dictionary { {"umbracoUrls", new[] {"authenticationApiBaseUrl", "serverVarsJs", "externalLoginsUrl", "currentUserApiBaseUrl", "iconApiBaseUrl"}}, - {"umbracoSettings", new[] {"allowPasswordReset", "imageFileTypes", "maxFileSize", "loginBackgroundImage", "loginLogoImage", "canSendRequiredEmail", "usernameIsEmail"}}, + {"umbracoSettings", new[] {"allowPasswordReset", "imageFileTypes", "maxFileSize", "loginBackgroundImage", "loginLogoImage", "canSendRequiredEmail", "usernameIsEmail", "minimumPasswordLength", "minimumPasswordNonAlphaNum"}}, {"application", new[] {"applicationPath", "cacheBuster"}}, {"isDebuggingEnabled", new string[] { }}, {"features", new [] {"disabledFeatures"}} @@ -100,6 +100,8 @@ namespace Umbraco.Web.Editors /// internal Dictionary GetServerVariables() { + var userMembershipProvider = Core.Security.MembershipProviderExtensions.GetUsersMembershipProvider(); + var defaultVals = new Dictionary { { @@ -357,6 +359,8 @@ namespace Umbraco.Web.Editors {"showUserInvite", EmailSender.CanSendRequiredEmail}, {"canSendRequiredEmail", EmailSender.CanSendRequiredEmail}, {"showAllowSegmentationForDocumentTypes", false}, + {"minimumPasswordLength", userMembershipProvider.MinRequiredPasswordLength}, + {"minimumPasswordNonAlphaNum", userMembershipProvider.MinRequiredNonAlphanumericCharacters}, } }, { diff --git a/src/Umbraco.Web/Editors/Binders/ContentModelBinderHelper.cs b/src/Umbraco.Web/Editors/Binders/ContentModelBinderHelper.cs index e0d39b5f65..75060d059a 100644 --- a/src/Umbraco.Web/Editors/Binders/ContentModelBinderHelper.cs +++ b/src/Umbraco.Web/Editors/Binders/ContentModelBinderHelper.cs @@ -1,5 +1,9 @@ -using System.Net; +using System.Collections.Generic; +using System.Linq; +using System.Net; using System.Net.Http; +using System.Text; +using System.Text.RegularExpressions; using System.Web.Http; using System.Web.Http.Controllers; using Umbraco.Core; @@ -17,6 +21,8 @@ namespace Umbraco.Web.Editors.Binders ///
    internal static class ContentModelBinderHelper { + private const char _escapeChar = '\\'; + public static TModelSave BindModelFromMultipartRequest(HttpActionContext actionContext, ModelBindingContext bindingContext) where TModelSave : IHaveUploadedFiles { @@ -30,6 +36,7 @@ namespace Umbraco.Web.Editors.Binders //The name that has been assigned in JS has 2 or more parts. The second part indicates the property id // for which the file belongs, the remaining parts are just metadata that can be used by the property editor. var parts = file.Headers.ContentDisposition.Name.Trim(Constants.CharArrays.DoubleQuote).Split(Constants.CharArrays.Underscore); + if (parts.Length < 2) { var response = actionContext.Request.CreateResponse(HttpStatusCode.BadRequest); diff --git a/src/Umbraco.Web/Editors/CodeFileController.cs b/src/Umbraco.Web/Editors/CodeFileController.cs index 409cded781..a6d142a6ea 100644 --- a/src/Umbraco.Web/Editors/CodeFileController.cs +++ b/src/Umbraco.Web/Editors/CodeFileController.cs @@ -638,7 +638,10 @@ namespace Umbraco.Web.Editors { var path = IOHelper.MapPath(systemDirectory + "/" + virtualPath); var dirInfo = new DirectoryInfo(path); - return dirInfo.Attributes == FileAttributes.Directory; + + // If you turn off indexing in Windows this will have the attribute: + // `FileAttributes.Directory | FileAttributes.NotContentIndexed` + return (dirInfo.Attributes & FileAttributes.Directory) != 0; } // this is an internal class for passing stylesheet data from the client to the controller while editing diff --git a/src/Umbraco.Web/Editors/DashboardController.cs b/src/Umbraco.Web/Editors/DashboardController.cs index da620eb5ac..97db8818f2 100644 --- a/src/Umbraco.Web/Editors/DashboardController.cs +++ b/src/Umbraco.Web/Editors/DashboardController.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Services; using Umbraco.Core.Dashboards; +using Umbraco.Core.Models; using Umbraco.Web.Services; namespace Umbraco.Web.Editors @@ -52,8 +53,9 @@ namespace Umbraco.Web.Editors var allowedSections = string.Join(",", user.AllowedSections); var language = user.Language; var version = UmbracoVersion.SemanticVersion.ToSemanticString(); + var isAdmin = user.IsAdmin(); - var url = string.Format(baseUrl + "{0}?section={0}&allowed={1}&lang={2}&version={3}", section, allowedSections, language, version); + var url = string.Format(baseUrl + "{0}?section={0}&allowed={1}&lang={2}&version={3}&admin={4}", section, allowedSections, language, version, isAdmin); var key = "umbraco-dynamic-dashboard-" + language + allowedSections.Replace(",", "-") + section; var content = AppCaches.RuntimeCache.GetCacheItem(key); diff --git a/src/Umbraco.Web/Editors/ExamineManagementController.cs b/src/Umbraco.Web/Editors/ExamineManagementController.cs index 49599bc8b9..132cc25404 100644 --- a/src/Umbraco.Web/Editors/ExamineManagementController.cs +++ b/src/Umbraco.Web/Editors/ExamineManagementController.cs @@ -25,7 +25,6 @@ namespace Umbraco.Web.Editors private readonly IAppPolicyCache _runtimeCache; private readonly IndexRebuilder _indexRebuilder; - public ExamineManagementController(IExamineManager examineManager, ILogger logger, AppCaches appCaches, IndexRebuilder indexRebuilder) @@ -79,14 +78,11 @@ namespace Umbraco.Web.Editors { Id = x.Id, Score = x.Score, - //order the values by key - Values = new Dictionary(x.Values.OrderBy(y => y.Key).ToDictionary(y => y.Key, y => y.Value)) + Values = x.AllValues.OrderBy(y => y.Key).ToDictionary(y => y.Key, y => y.Value) }) }; } - - /// /// Check if the index has been rebuilt /// @@ -113,7 +109,6 @@ namespace Umbraco.Web.Editors return found != null ? null : CreateModel(index); - } /// @@ -167,8 +162,6 @@ namespace Umbraco.Web.Editors } } - - private ExamineIndexModel CreateModel(IIndex index) { var indexName = index.Name; @@ -182,11 +175,13 @@ namespace Umbraco.Web.Editors } var isHealth = indexDiag.IsHealthy(); + var properties = new Dictionary { [nameof(IIndexDiagnostics.DocumentCount)] = indexDiag.DocumentCount, [nameof(IIndexDiagnostics.FieldCount)] = indexDiag.FieldCount, }; + foreach (var p in indexDiag.Metadata) properties[p.Key] = p.Value; @@ -198,7 +193,6 @@ namespace Umbraco.Web.Editors CanRebuild = _indexRebuilder.CanRebuild(index) }; - return indexerModel; } @@ -211,7 +205,6 @@ namespace Umbraco.Web.Editors return Request.CreateResponse(HttpStatusCode.OK); } - //if we didn't find anything try to find it by an explicitly declared searcher if (_examineManager.TryGetSearcher(searcherName, out searcher)) return Request.CreateResponse(HttpStatusCode.OK); diff --git a/src/Umbraco.Web/Editors/KeepAliveController.cs b/src/Umbraco.Web/Editors/KeepAliveController.cs index 23815e1bbe..f29ee6c60a 100644 --- a/src/Umbraco.Web/Editors/KeepAliveController.cs +++ b/src/Umbraco.Web/Editors/KeepAliveController.cs @@ -1,14 +1,12 @@ using System.Runtime.Serialization; using System.Web.Http; -using Umbraco.Web.Mvc; using Umbraco.Web.WebApi; -using Umbraco.Web.WebApi.Filters; namespace Umbraco.Web.Editors { public class KeepAliveController : UmbracoApiController { - [OnlyLocalRequests] + [HttpHead] [HttpGet] public KeepAlivePingResult Ping() { diff --git a/src/Umbraco.Web/Editors/MemberTypeController.cs b/src/Umbraco.Web/Editors/MemberTypeController.cs index 4bfea76eda..2e665350e7 100644 --- a/src/Umbraco.Web/Editors/MemberTypeController.cs +++ b/src/Umbraco.Web/Editors/MemberTypeController.cs @@ -237,6 +237,18 @@ namespace Umbraco.Web.Editors return display; } + /// + /// Copy the member type + /// + /// + /// + public HttpResponseMessage PostCopy(MoveOrCopy copy) + { + return PerformCopy( + copy, + getContentType: i => Services.MemberTypeService.Get(i), + doCopy: (type, i) => Services.MemberTypeService.Copy(type, i)); + } } } diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs index fd76b9d486..33cf89b2e4 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/ExcessiveHeadersCheck.cs @@ -49,7 +49,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Security { var message = string.Empty; var success = false; - var url = _runtime.ApplicationUrl; + var url = _runtime.ApplicationUrl.GetLeftPart(UriPartial.Authority); // Access the site home page and check for the headers var request = WebRequest.Create(url); @@ -69,7 +69,7 @@ namespace Umbraco.Web.HealthCheck.Checks.Security } catch (Exception ex) { - message = _textService.Localize("healthcheck/httpsCheckInvalidUrl", new[] { url.ToString(), ex.Message }); + message = _textService.Localize("healthcheck/healthCheckInvalidUrl", new[] { url.ToString(), ex.Message }); } var actions = new List(); diff --git a/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs index 98f8a83c1d..83fafb79f8 100644 --- a/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs +++ b/src/Umbraco.Web/HealthCheck/Checks/Security/HttpsCheck.cs @@ -2,9 +2,9 @@ using System.Collections.Generic; using System.Net; using System.Security.Cryptography.X509Certificates; -using System.Web; using Umbraco.Core; using Umbraco.Core.Configuration; +using Umbraco.Core.Configuration.UmbracoSettings; using Umbraco.Core.IO; using Umbraco.Core.Services; using Umbraco.Web.HealthCheck.Checks.Config; @@ -21,14 +21,16 @@ namespace Umbraco.Web.HealthCheck.Checks.Security private readonly ILocalizedTextService _textService; private readonly IRuntimeState _runtime; private readonly IGlobalSettings _globalSettings; + private readonly IContentSection _contentSection; private const string FixHttpsSettingAction = "fixHttpsSetting"; - public HttpsCheck(ILocalizedTextService textService, IRuntimeState runtime, IGlobalSettings globalSettings) + public HttpsCheck(ILocalizedTextService textService, IRuntimeState runtime, IGlobalSettings globalSettings, IContentSection contentSection) { _textService = textService; _runtime = runtime; _globalSettings = globalSettings; + _contentSection = contentSection; } /// @@ -65,12 +67,25 @@ namespace Umbraco.Web.HealthCheck.Checks.Security // Attempt to access the site over HTTPS to see if it HTTPS is supported // and a valid certificate has been configured var url = _runtime.ApplicationUrl.ToString().Replace("http:", "https:"); + var request = (HttpWebRequest) WebRequest.Create(url); - request.Method = "HEAD"; + request.AllowAutoRedirect = false; try { + var response = (HttpWebResponse)request.GetResponse(); + + // Check for 301/302 as a external login provider such as UmbracoID might be in use + if (response.StatusCode == HttpStatusCode.Moved || response.StatusCode == HttpStatusCode.Redirect) + { + // Reset request to use the static login background image + var absoluteLoginBackgroundImage = $"{url}/{_contentSection.LoginBackgroundImage}"; + + request = (HttpWebRequest)WebRequest.Create(absoluteLoginBackgroundImage); + response = (HttpWebResponse)request.GetResponse(); + } + if (response.StatusCode == HttpStatusCode.OK) { // Got a valid response, check now for if certificate expiring within 14 days diff --git a/src/Umbraco.Web/Models/ContentEditing/SearchResult.cs b/src/Umbraco.Web/Models/ContentEditing/SearchResult.cs index 1cdd539165..d33bc3530e 100644 --- a/src/Umbraco.Web/Models/ContentEditing/SearchResult.cs +++ b/src/Umbraco.Web/Models/ContentEditing/SearchResult.cs @@ -16,6 +16,6 @@ namespace Umbraco.Web.Models.ContentEditing public int FieldCount => Values?.Count ?? 0; [DataMember(Name = "values")] - public IReadOnlyDictionary Values { get; set; } + public IReadOnlyDictionary> Values { get; set; } } } diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 862837381a..f9eacd9e73 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -202,8 +202,8 @@ namespace Umbraco.Web.PropertyEditors _richTextPropertyValueEditor.GetReferences(x.Value))) yield return umbracoEntityReference; - foreach (var umbracoEntityReference in mediaValues.SelectMany(x => - _mediaPickerPropertyValueEditor.GetReferences(x.Value["udi"]))) + foreach (var umbracoEntityReference in mediaValues.Where(x => x.Value.HasValues) + .SelectMany(x => _mediaPickerPropertyValueEditor.GetReferences(x.Value["udi"]))) yield return umbracoEntityReference; } } diff --git a/src/Umbraco.Web/PropertyEditors/ListViewConfiguration.cs b/src/Umbraco.Web/PropertyEditors/ListViewConfiguration.cs index de538793a5..2ea7b9e44e 100644 --- a/src/Umbraco.Web/PropertyEditors/ListViewConfiguration.cs +++ b/src/Umbraco.Web/PropertyEditors/ListViewConfiguration.cs @@ -28,7 +28,7 @@ namespace Umbraco.Web.PropertyEditors Layouts = new[] { new Layout { Name = "List", Icon = "icon-list", IsSystem = 1, Selected = true, Path = "views/propertyeditors/listview/layouts/list/list.html" }, - new Layout { Name = "grid", Icon = "icon-thumbnails-small", IsSystem = 1, Selected = true, Path = "views/propertyeditors/listview/layouts/grid/grid.html" } + new Layout { Name = "Grid", Icon = "icon-thumbnails-small", IsSystem = 1, Selected = true, Path = "views/propertyeditors/listview/layouts/grid/grid.html" } }; IncludeProperties = new [] diff --git a/src/Umbraco.Web/PublishedContentExtensions.cs b/src/Umbraco.Web/PublishedContentExtensions.cs index 8aa3b69fb4..c851894149 100644 --- a/src/Umbraco.Web/PublishedContentExtensions.cs +++ b/src/Umbraco.Web/PublishedContentExtensions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Data; using System.Linq; @@ -173,8 +173,8 @@ namespace Umbraco.Web return value; // else... if we have a property, at least let the converter return its own - // vision of 'no value' (could be an empty enumerable) - otherwise, defaultValue - return property == null ? defaultValue : property.Value(culture, segment, defaultValue: defaultValue); + // vision of 'no value' (could be an empty enumerable) - otherwise, default + return property == null ? default : property.Value(culture, segment, fallback, defaultValue); } #endregion @@ -814,6 +814,64 @@ namespace Umbraco.Web #endregion + #region Axes: breadcrumbs + + /// + /// Gets the breadcrumbs (ancestors and self, top to bottom) for the specified . + /// + /// The content. + /// Indicates whether the specified content should be included. + /// + /// The breadcrumbs (ancestors and self, top to bottom) for the specified . + /// + public static IEnumerable Breadcrumbs(this IPublishedContent content, bool andSelf = true) + { + return content.AncestorsOrSelf(andSelf, null).Reverse(); + } + + /// + /// Gets the breadcrumbs (ancestors and self, top to bottom) for the specified at a level higher or equal to . + /// + /// The content. + /// The minimum level. + /// Indicates whether the specified content should be included. + /// + /// The breadcrumbs (ancestors and self, top to bottom) for the specified at a level higher or equal to . + /// + public static IEnumerable Breadcrumbs(this IPublishedContent content, int minLevel, bool andSelf = true) + { + return content.AncestorsOrSelf(andSelf, n => n.Level >= minLevel).Reverse(); + } + + /// + /// Gets the breadcrumbs (ancestors and self, top to bottom) for the specified at a level higher or equal to the specified root content type . + /// + /// The root content type. + /// The content. + /// Indicates whether the specified content should be included. + /// + /// The breadcrumbs (ancestors and self, top to bottom) for the specified at a level higher or equal to the specified root content type . + /// + public static IEnumerable Breadcrumbs(this IPublishedContent content, bool andSelf = true) + where T : class, IPublishedContent + { + static IEnumerable TakeUntil(IEnumerable source, Func predicate) + { + foreach (var item in source) + { + yield return item; + if (predicate(item)) + { + yield break; + } + } + } + + return TakeUntil(content.AncestorsOrSelf(andSelf, null), n => n is T).Reverse(); + } + + #endregion + #region Axes: descendants, descendants-or-self /// @@ -1271,15 +1329,37 @@ namespace Umbraco.Web #region Axes: custom /// - /// Gets the root content for this content. + /// Gets the root content (ancestor or self at level 1) for the specified . /// /// The content. - /// The 'site' content ie AncestorOrSelf(1). + /// + /// The root content (ancestor or self at level 1) for the specified . + /// + /// + /// This is the same as calling with maxLevel set to 1. + /// public static IPublishedContent Root(this IPublishedContent content) { return content.AncestorOrSelf(1); } + /// + /// Gets the root content (ancestor or self at level 1) for the specified if it's of the specified content type . + /// + /// The content type. + /// The content. + /// + /// The root content (ancestor or self at level 1) for the specified of content type . + /// + /// + /// This is the same as calling with maxLevel set to 1. + /// + public static T Root(this IPublishedContent content) + where T : class, IPublishedContent + { + return content.AncestorOrSelf(1); + } + #endregion #region PropertyAliasesAndNames diff --git a/src/Umbraco.Web/PublishedPropertyExtension.cs b/src/Umbraco.Web/PublishedPropertyExtension.cs index 0c3aa57cc2..6e8647db47 100644 --- a/src/Umbraco.Web/PublishedPropertyExtension.cs +++ b/src/Umbraco.Web/PublishedPropertyExtension.cs @@ -1,4 +1,3 @@ -using System.Collections.Generic; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.Models.PublishedContent; @@ -37,13 +36,20 @@ namespace Umbraco.Web // we have a value // try to cast or convert it var value = property.GetValue(culture, segment); - if (value is T valueAsT) return valueAsT; - var valueConverted = value.TryConvertTo(); - if (valueConverted) return valueConverted.Result; + if (value is T valueAsT) + { + return valueAsT; + } - // cannot cast nor convert the value, nothing we can return but 'defaultValue' + var valueConverted = value.TryConvertTo(); + if (valueConverted) + { + return valueConverted.Result; + } + + // cannot cast nor convert the value, nothing we can return but 'default' // note: we don't want to fallback in that case - would make little sense - return defaultValue; + return default; } // we don't have a value, try fallback @@ -57,15 +63,22 @@ namespace Umbraco.Web var noValue = property.GetValue(culture, segment); if (noValue == null) { - return defaultValue; + return default; + } + + if (noValue is T noValueAsT) + { + return noValueAsT; } - if (noValue is T noValueAsT) return noValueAsT; var noValueConverted = noValue.TryConvertTo(); - if (noValueConverted) return noValueConverted.Result; + if (noValueConverted) + { + return noValueConverted.Result; + } - // cannot cast noValue nor convert it, nothing we can return but 'defaultValue' - return defaultValue; + // cannot cast noValue nor convert it, nothing we can return but 'default' + return default; } #endregion diff --git a/src/Umbraco.Web/Routing/PublishedRouter.cs b/src/Umbraco.Web/Routing/PublishedRouter.cs index a02fd5872a..ebf935dcf8 100644 --- a/src/Umbraco.Web/Routing/PublishedRouter.cs +++ b/src/Umbraco.Web/Routing/PublishedRouter.cs @@ -427,7 +427,7 @@ namespace Umbraco.Web.Routing return finder.TryFindContent(request); }); - _profilingLogger.Debug( + _logger.Debug( "Found? {Found} Content: {PublishedContentId}, Template: {TemplateAlias}, Domain: {Domain}, Culture: {Culture}, Is404: {Is404}, StatusCode: {StatusCode}", found, request.HasPublishedContent ? request.PublishedContent.Id : "NULL", @@ -516,55 +516,47 @@ namespace Umbraco.Web.Routing // don't try to find a redirect if the property doesn't exist if (request.PublishedContent.HasProperty(Constants.Conventions.Content.InternalRedirectId) == false) + { return false; + } + + var internalRedirectId = request.PublishedContent.Value(Constants.Conventions.Content.InternalRedirectId)?.ToString(); + + if (internalRedirectId == null) + { + // no value stored, just return, no need to log + return false; + } + + if (int.TryParse(internalRedirectId, out var internalRedirectIdAsInt) && internalRedirectIdAsInt == request.PublishedContent.Id) + { + // redirect to self + _logger.Debug("FollowInternalRedirects: Redirecting to self, ignore"); + return false; + } - var redirect = false; - var valid = false; IPublishedContent internalRedirectNode = null; - var internalRedirectId = request.PublishedContent.Value(Constants.Conventions.Content.InternalRedirectId, defaultValue: -1); - - if (internalRedirectId > 0) + if (internalRedirectIdAsInt > 0) { // try and get the redirect node from a legacy integer ID - valid = true; - internalRedirectNode = request.UmbracoContext.Content.GetById(internalRedirectId); + internalRedirectNode = request.UmbracoContext.Content.GetById(internalRedirectIdAsInt); } - else + else if (GuidUdi.TryParse(internalRedirectId, out var internalRedirectIdAsUdi)) { - var udiInternalRedirectId = request.PublishedContent.Value(Constants.Conventions.Content.InternalRedirectId); - if (udiInternalRedirectId != null) - { - // try and get the redirect node from a UDI Guid - valid = true; - internalRedirectNode = request.UmbracoContext.Content.GetById(udiInternalRedirectId.Guid); - } - } - - if (valid == false) - { - // bad redirect - log and display the current page (legacy behavior) - _logger.Debug("FollowInternalRedirects: Failed to redirect to id={InternalRedirectId}: value is not an int nor a GuidUdi.", - request.PublishedContent.GetProperty(Constants.Conventions.Content.InternalRedirectId).GetSourceValue()); + // try and get the redirect node from a UDI Guid + internalRedirectNode = request.UmbracoContext.Content.GetById(internalRedirectIdAsUdi.Guid); } if (internalRedirectNode == null) { _logger.Debug("FollowInternalRedirects: Failed to redirect to id={InternalRedirectId}: no such published document.", request.PublishedContent.GetProperty(Constants.Conventions.Content.InternalRedirectId).GetSourceValue()); - } - else if (internalRedirectId == request.PublishedContent.Id) - { - // redirect to self - _logger.Debug("FollowInternalRedirects: Redirecting to self, ignore"); - } - else - { - request.SetInternalRedirectPublishedContent(internalRedirectNode); // don't use .PublishedContent here - redirect = true; - _logger.Debug("FollowInternalRedirects: Redirecting to id={InternalRedirectId}", internalRedirectId); + return false; } - return redirect; + request.SetInternalRedirectPublishedContent(internalRedirectNode); // don't use .PublishedContent here + _logger.Debug("FollowInternalRedirects: Redirecting to id={InternalRedirectId}", internalRedirectIdAsInt); + return true; } /// diff --git a/src/Umbraco.Web/Runtime/WebInitialComposer.cs b/src/Umbraco.Web/Runtime/WebInitialComposer.cs index b15641b503..5d97bfe4a2 100644 --- a/src/Umbraco.Web/Runtime/WebInitialComposer.cs +++ b/src/Umbraco.Web/Runtime/WebInitialComposer.cs @@ -40,7 +40,6 @@ using Current = Umbraco.Web.Composing.Current; using Umbraco.Web.PropertyEditors; using Umbraco.Core.Models; using Umbraco.Web.Models; -using Ganss.XSS; namespace Umbraco.Web.Runtime { @@ -140,15 +139,6 @@ namespace Umbraco.Web.Runtime composition.RegisterUnique(); composition.RegisterUnique(); composition.RegisterUnique(); - composition.Register(_ => - { - var sanitizer = new HtmlSanitizer(); - sanitizer.AllowedAttributes.UnionWith(Umbraco.Core.Constants.SvgSanitizer.Attributes); - sanitizer.AllowedCssProperties.UnionWith(Umbraco.Core.Constants.SvgSanitizer.Attributes); - sanitizer.AllowedTags.UnionWith(Umbraco.Core.Constants.SvgSanitizer.Tags); - return sanitizer; - },Lifetime.Singleton); - composition.RegisterUnique(factory => ExamineManager.Instance); // configure the container for web diff --git a/src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs b/src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs index a126592ffc..e8cb592536 100644 --- a/src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs +++ b/src/Umbraco.Web/Scheduling/BackgroundTaskRunner.cs @@ -319,7 +319,10 @@ namespace Umbraco.Web.Scheduling // create a new token source since this is a new process _shutdownTokenSource = new CancellationTokenSource(); _shutdownToken = _shutdownTokenSource.Token; - _runningTask = Task.Run(async () => await Pump().ConfigureAwait(false), _shutdownToken); + using (ExecutionContext.SuppressFlow()) + { + _runningTask = Task.Run(async () => await Pump().ConfigureAwait(false), _shutdownToken); + } _logger.Debug("{LogPrefix} Starting", _logPrefix); } @@ -544,10 +547,14 @@ namespace Umbraco.Web.Scheduling try { if (bgTask.IsAsync) + { // configure await = false since we don't care about the context, we're on a background thread. await bgTask.RunAsync(token).ConfigureAwait(false); + } else + { bgTask.Run(); + } } finally // ensure we disposed - unless latched again ie wants to re-run { @@ -710,14 +717,20 @@ namespace Umbraco.Web.Scheduling // with a single aspnet thread during shutdown and we don't want to delay other calls to IRegisteredObject.Stop. if (!immediate) { - return Task.Run(StopInitial, CancellationToken.None); + using (ExecutionContext.SuppressFlow()) + { + return Task.Run(StopInitial, CancellationToken.None); + } } else { lock (_locker) { if (_terminated) return Task.CompletedTask; - return Task.Run(StopImmediate, CancellationToken.None); + using (ExecutionContext.SuppressFlow()) + { + return Task.Run(StopImmediate, CancellationToken.None); + } } } } diff --git a/src/Umbraco.Web/Services/IconService.cs b/src/Umbraco.Web/Services/IconService.cs index 15e673e6ba..fad53103c0 100644 --- a/src/Umbraco.Web/Services/IconService.cs +++ b/src/Umbraco.Web/Services/IconService.cs @@ -2,7 +2,6 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using Ganss.XSS; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.Configuration; @@ -15,13 +14,11 @@ namespace Umbraco.Web.Services public class IconService : IIconService { private readonly IGlobalSettings _globalSettings; - private readonly IHtmlSanitizer _htmlSanitizer; private readonly IAppPolicyCache _cache; - public IconService(IGlobalSettings globalSettings, IHtmlSanitizer htmlSanitizer, AppCaches appCaches) + public IconService(IGlobalSettings globalSettings, AppCaches appCaches) { _globalSettings = globalSettings; - _htmlSanitizer = htmlSanitizer; _cache = appCaches.RuntimeCache; } @@ -78,12 +75,11 @@ namespace Umbraco.Web.Services try { var svgContent = System.IO.File.ReadAllText(iconPath); - var sanitizedString = _htmlSanitizer.Sanitize(svgContent); var svg = new IconModel { Name = iconName, - SvgString = sanitizedString + SvgString = svgContent }; return svg; diff --git a/src/Umbraco.Web/Trees/MemberTreeController.cs b/src/Umbraco.Web/Trees/MemberTreeController.cs index c0a9d15cfa..37496d1bff 100644 --- a/src/Umbraco.Web/Trees/MemberTreeController.cs +++ b/src/Umbraco.Web/Trees/MemberTreeController.cs @@ -129,10 +129,12 @@ namespace Umbraco.Web.Trees if (_isUmbracoProvider) { - nodes.AddRange(Services.MemberTypeService.GetAll() - .Select(memberType => - CreateTreeNode(memberType.Alias, id, queryStrings, memberType.Name, memberType.Icon.IfNullOrWhiteSpace(Constants.Icons.Member), true, - queryStrings.GetRequiredValue("application") + TreeAlias.EnsureStartsWith('/') + "/list/" + memberType.Alias))); + nodes.AddRange( + Services.MemberTypeService.GetAll() + .OrderBy(x => x.Name) + .Select(memberType => + CreateTreeNode(memberType.Alias, id, queryStrings, memberType.Name, memberType.Icon.IfNullOrWhiteSpace(Constants.Icons.Member), true, + queryStrings.GetRequiredValue("application") + TreeAlias.EnsureStartsWith('/') + "/list/" + memberType.Alias))); } } diff --git a/src/Umbraco.Web/Trees/MemberTypeAndGroupTreeControllerBase.cs b/src/Umbraco.Web/Trees/MemberTypeAndGroupTreeControllerBase.cs index 5e71266bca..61b9b3e063 100644 --- a/src/Umbraco.Web/Trees/MemberTypeAndGroupTreeControllerBase.cs +++ b/src/Umbraco.Web/Trees/MemberTypeAndGroupTreeControllerBase.cs @@ -13,6 +13,10 @@ namespace Umbraco.Web.Trees protected override TreeNodeCollection GetTreeNodes(string id, FormDataCollection queryStrings) { var nodes = new TreeNodeCollection(); + + // if the request is for folders only then just return + if (queryStrings["foldersonly"].IsNullOrWhiteSpace() == false && queryStrings["foldersonly"] == "1") return nodes; + nodes.AddRange(GetTreeNodesFromService(id, queryStrings)); return nodes; } @@ -30,7 +34,13 @@ namespace Umbraco.Web.Trees } else { - //delete member type/group + var memberType = Services.MemberTypeService.Get(int.Parse(id)); + if (memberType != null) + { + menu.Items.Add(Services.TextService, opensDialog: true); + } + + // delete member type/group menu.Items.Add(Services.TextService, opensDialog: true); } diff --git a/src/Umbraco.Web/Trees/MemberTypeTreeController.cs b/src/Umbraco.Web/Trees/MemberTypeTreeController.cs index 5db9088f20..85f61d5fed 100644 --- a/src/Umbraco.Web/Trees/MemberTypeTreeController.cs +++ b/src/Umbraco.Web/Trees/MemberTypeTreeController.cs @@ -29,6 +29,7 @@ namespace Umbraco.Web.Trees root.HasChildren = Services.MemberTypeService.GetAll().Any(); return root; } + protected override IEnumerable GetTreeNodesFromService(string id, FormDataCollection queryStrings) { return Services.MemberTypeService.GetAll() diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index a6096692b4..a0e074c9c1 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -65,9 +65,6 @@ - - 5.0.376 - 2.7.0.100 From 7704d02883306d2b71c82b24f4d424c0ffd062f9 Mon Sep 17 00:00:00 2001 From: Chad Date: Wed, 21 Apr 2021 11:05:37 +1200 Subject: [PATCH 089/147] Merge in #9859 to v8/feature/nucache-perf (#10152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * load only once * Bump version to 8.6.8 * Initial rework of Lock dictionaries * [Issue 5277-146] accessibility - Close 'X' icon next to language drop… (#9264) * [Issue 5277-146] accessibility - Close 'X' icon next to language drop down is identified as "link" - screen reader * add new loacalization key * Fix issue with SqlMainDomLock that cannot use implicit lock timeouts … (#9973) * Fix issue with SqlMainDomLock that cannot use implicit lock timeouts … (#9973) (cherry picked from commit da5351dfcf23daad69fcd73eb74811456ffc34c0) * Adjust unit tests and apply fixes to scope * Add more unit tests, showing current issue * Counting Umbraco.ModelsBuilder and ModelsBuilder.Umbraco namespaces as external providers * Fix dead lock with TypeLoader * Fix errors shown in unit tests * Throw error if all scopes hasn't been disposed * Clean * Fixes and Updates for DB Scope and Ambient Context leaks (#9953) * Adds some scope tests (ported back from netcore) and provides a much better error message, ensure execution context is not flowed to child tasks that shouldn't leak any current ambient context * updates comment * Ensure SqlMainDomLock suppresses execution context too * Since we're awaiting a task in a library method, ConfigureAwait(false) * missing null check Co-authored-by: Elitsa Marinovska * Adds additional error checking and reporting to MainDom/SqlMainDomLock (#9954) Co-authored-by: Elitsa Marinovska * Add copy logic to Media Picker (#9957) * Add copy logic to Media Picker * Add action for copy all * Fix for selectable media item * Wrap calls to map in scopes * Autocomplete scopes * Remove unnecessary aria-hidden attribute from * Remove scope from method that calls another method that has a scope * Fixes #9993 - Cannot save empty image in Grid * Clean * Revert "The Value() method for IPublishedContent was not working with the defaultValue parameter" (#9989) * Use a hashset to keep track of acquired locks This simplifies disposing/checking for locks greatly. * Add images in grid - fixes 9982 (#9987) Co-authored-by: Sebastiaan Janssen * Only create the dicts and hashset when a lock is requested * Clean * Adds a config for configuring the access rules on the content dashboard - by default it granted for all user groups * Adds additional params indicating whether user is admin * Add images in grid - fixes 9982 (#9987) Co-authored-by: Sebastiaan Janssen (cherry picked from commit e2019777fbfc1f9221d040cb9f0b82c57f8552b9) * Bump version to 8.12.2 * #9964 Removed unneeded check for HttpContext * Fix for #9950 - HttpsCheck will now retry using the login background image if inital request returns 301/302. Excessvie Headers check will now check the root url instead of the backoffice * Merge pull request #9994 from umbraco/v8/bugfix/9993 Fixes #9993 - Cannot save empty image in Grid (cherry picked from commit 0ecc933921f2dea9a2a16d6f395b44a039663ec6) * Apply suggestions from review * Fixes #9983 - Getting kicked, if document type has a Umbraco.UserPicker property (#10002) * Fixes #9983 Temporary fix for this issue. using the entityservice like before. * Needed to remove the call to usersResource here as well for displaying the picked items * Don't need usersResource for now * Fixes #9983 - Getting kicked, if document type has a Umbraco.UserPicker property (#10002) * Fixes #9983 Temporary fix for this issue. using the entityservice like before. * Needed to remove the call to usersResource here as well for displaying the picked items * Don't need usersResource for now (cherry picked from commit 45de0a101eaa2b8f16e21a765f32928c7cb968be) * 8539: Allow alias in image cropper (#9266) Co-authored-by: Owain Williams * Wrap dumping dictionaries in a method. * Create method for generating log message And remove forgotten comments. * Fix swedish translation for somethingElse. * Copy member type (#10020) * Add copy dialog for member type * Implement copy action for member type * Create specific localization for content type, media type and member type * Handle "foldersonly" querystring * Add button type attribute * Add a few missing changes of anchor to button element * Null check on scope and options to ensure backward compatibility * Improve performance, readability and handling of FollowInternalRedirects (#9889) * Improve performance, readability and handling of FollowInternalRedirects * Logger didn't like string param Passing string param to _logger.Debug wasn't happy. Changed to pass existing internalRedirectAsInt variable. Co-authored-by: Nathan Woulfe * Update casing of listview layout name * 9097 add contextual password helper (#9256) * update back-office forms * Display tip on reset password page as well * add directive for password tip * integrate directove in login screen * forgot the ng-keyup :-) * adapt tooltip directive to potential different Members and Users password settings * remove watcher Co-authored-by: Nathan Woulfe * Unbind listener Listening for splitViewRequest was only unbound if the split view editor was opened. Not cleaning up the listener caused a memory leak when changing between nodes as the spit view editor was detached but not garbage-collected * Replace icon in date picker with umb-icon component (#10040) * Replace icon in date picker with component * Adjust height of clear button * Update cypress and fix tests * Listview config icons (#10036) * Update icons to use component * Simplify markup and use disabled button * Use move cursor style on sortable handle * Add class for action column * Update setting auto focus * Increase font size of umb-panel-header-icon * Anchor noopener (#10009) * Set rel="noopener" for anchors with target="_blank" * Reverted unwanted changes to Default.cshtml * Align 'Add language' test to netcore * Add new cypress tests * Add indentation * Getting rid of the config file and implementing an appSetting instead * Implementation for IContentDashboardSettings * Cleanup * bool.Try * Taking AllowContentDashboardAccessToAllUsers prop from GlobalSettings to ContentDashboardSettings and saving AccessRulesFromConfig into a backing field * fix support for non run states * Handling multiple values per field in Examine Management * Add Root and Breadcrumbs extension methods for IPublishedContent (#9033) * Fix usage of obsolete CreatorName and WriterName properties * Add generic Root extension method * Add Breadcrumbs extension methods * Orders member type grouping of members alphabetically, matching the listing of member types. * Revert updating deprecated WriterName/CreatorName refs Changing the properties to use the extensions is a good thing (given the props are deprecated), but causes issues deep in tests. I'm reverting that change to fix the tests, and all refs to the deprecated properties should be updated in one sweep, to deal with any other test issues that might crop up. * Handle Invalid format for Upgrade check * Fixes tabbing-mode remains active after closing modal #9790 (#10074) * Allow to pass in boolean to preventEnterSubmit directive (#8639) * Pass in value to preventEnterSubmit directive * Set enabled similar to preventDefault and preventEnterSubmit directives * Update prevent enter submit value * Init value from controller * Use a different default input id prefix for umb-search-filter * Fix typo * Check for truthly value * Revert "Set enabled similar to preventDefault and preventEnterSubmit directives" This reverts commit 536ce855c4545ead82cea77b4013bf9010a8687b. * None pointer events when clicking icon * Use color variable * Fixes tabbing-mode remains active after closing modal #9790 (#10074) (cherry picked from commit c881fa9e7d08c11954e18489827f70cdafceb947) * Null check on scope and options to ensure backward compatibility (cherry picked from commit fe8cd239d2f4c528c1a8a3cf4c50e90bb43cacfc) * Fix validation of step size in integer/numeric field * 9962: Use $allowedEditors instead of allowed (#10086) * 9962: Use $allowedEditors instead of allowed * 9962: Remove redundant statement * fixes #10021 adds ng-form and val-form-manager to the documentation * Improved accessibility of link picker (#10099) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Added control ids for the link picker * Add French translation * Accessibility: Alerts the user how many results have been returned on a tree search (#10100) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Tree search details the number of search items returned * Add French translations * Updated LightInject to v6.4.0 * Remove HtmlSanitizer once more - see #9803 * Also make sure NuGet installs the correct version of the CodePages dependency * Bump version to 8.13 RC * Fixed copy preserving sort order (#10091) * Revert "Updated LightInject to v6.4.0" This reverts commit fc77252ec756cf90bb74e7fbbe6dd6d75cbdacfc. * Revert "Add copy logic to Media Picker (#9957)" This reverts commit f7c032af65cac83182782c758a3ab79c86b92e70. * Reintroduce old constructor to make non-breaking * Update cypress test to make macros in the grid work again * Attributes could be multiple items, test specifically if `Directory` is an attribute * Accessibility: Adding label fors and control ids for the macro picker (#10101) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Added support for label fors for the macro picker and also gave the ,acro search box a title * Now displays a count of the matching macros returned. Please note the language file amends shared with #10100 * Removed src-only class for the display of the count of messages * Updating typo * Removed top-margin from switcher icon * Allow KeepAlive controller Ping method to be requested by non local requests (#10126) * Allow KeepAlive controller Ping method to be requested by non local requests and accept head requests * removed unused references * fix csproj * fix merge * btree serializer optimizations * array pool and nametable optimizations Co-authored-by: Mole Co-authored-by: Sebastiaan Janssen Co-authored-by: Justin Shearer Co-authored-by: Bjarke Berg Co-authored-by: Callum Whyte Co-authored-by: Shannon Co-authored-by: Elitsa Marinovska Co-authored-by: patrickdemooij9 Co-authored-by: Bjarne Fyrstenborg Co-authored-by: Michael Latouche Co-authored-by: Nathan Woulfe Co-authored-by: Markus Johansson Co-authored-by: Jeavon Leopold Co-authored-by: Benjamin Carleski Co-authored-by: Owain Williams Co-authored-by: Jesper Löfgren Co-authored-by: Martin Bentancour Co-authored-by: Ronald Barendse Co-authored-by: Andy Butland Co-authored-by: BeardinaSuit Co-authored-by: Mads Rasmussen Co-authored-by: Rachel Breeze Co-authored-by: Dave de Moel Co-authored-by: ric <60885685+ricbrady@users.noreply.github.com> Co-authored-by: Carole Rennie Logan Co-authored-by: Dennis Öhman --- .../Sync/DatabaseServerMessenger.cs | 90 ++++++++++++------- .../Sync/ISyncBootStateAccessor.cs | 20 +++++ .../Sync/NonRuntimeLevelBootStateAccessor.cs | 19 ++++ src/Umbraco.Core/Sync/SyncBootState.cs | 24 +++++ src/Umbraco.Core/Umbraco.Core.csproj | 3 + .../PublishedContent/NuCacheChildrenTests.cs | 2 + .../PublishedContent/NuCacheTests.cs | 2 + .../Scoping/ScopedNuCacheTests.cs | 1 + .../ContentTypeServiceVariantsTests.cs | 2 + .../TestHelpers/TestSyncBootStateAccessor.cs | 23 +++++ src/Umbraco.Tests/Umbraco.Tests.csproj | 1 + ...aseServerRegistrarAndMessengerComponent.cs | 1 + ...Tree.DictionaryOfPropertyDataSerializer.cs | 8 +- .../JsonContentNestedDataSerializer.cs | 53 ++++++++++- .../PublishedCache/NuCache/NuCacheComposer.cs | 4 + .../NuCache/PublishedSnapshotService.cs | 19 +++- 16 files changed, 233 insertions(+), 39 deletions(-) create mode 100644 src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs create mode 100644 src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs create mode 100644 src/Umbraco.Core/Sync/SyncBootState.cs create mode 100644 src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs mode change 100755 => 100644 src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs index 49b0d23862..ebc77dbdca 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs @@ -28,7 +28,7 @@ namespace Umbraco.Core.Sync // but only processes instructions coming from remote servers, // thus ensuring that instructions run only once // - public class DatabaseServerMessenger : ServerMessengerBase + public class DatabaseServerMessenger : ServerMessengerBase, ISyncBootStateAccessor { private readonly IRuntimeState _runtime; private readonly ManualResetEvent _syncIdle; @@ -172,35 +172,7 @@ namespace Umbraco.Core.Sync lock (_locko) { if (_released) return; - - var coldboot = false; - if (_lastId < 0) // never synced before - { - // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new - // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. - Logger.Warn("No last synced Id found, this generally means this is a new server/install." - + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" - + " the database and maintain cache updates based on that Id."); - - coldboot = true; - } - else - { - //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each - //row so we will sum these numbers to get the actual count. - var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); - if (count > Options.MaxProcessingInstructionCount) - { - //too many instructions, proceed to cold boot - Logger.Warn( - "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." - + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" - + " to the latest found in the database and maintain cache updates based on that Id.", - count, Options.MaxProcessingInstructionCount); - - coldboot = true; - } - } + var coldboot = IsColdBoot(database); if (coldboot) { @@ -223,6 +195,40 @@ namespace Umbraco.Core.Sync } } + private bool IsColdBoot(IUmbracoDatabase database) + { + var coldboot = false; + if (_lastId < 0) // never synced before + { + // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new + // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. + Logger.Warn("No last synced Id found, this generally means this is a new server/install." + + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" + + " the database and maintain cache updates based on that Id."); + + coldboot = true; + } + else + { + //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each + //row so we will sum these numbers to get the actual count. + var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); + if (count > Options.MaxProcessingInstructionCount) + { + //too many instructions, proceed to cold boot + Logger.Warn( + "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." + + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" + + " to the latest found in the database and maintain cache updates based on that Id.", + count, Options.MaxProcessingInstructionCount); + + coldboot = true; + } + } + + return coldboot; + } + /// /// Synchronize the server (throttled). /// @@ -548,6 +554,30 @@ namespace Umbraco.Core.Sync #endregion + public SyncBootState GetSyncBootState() + { + try + { + ReadLastSynced(); // get _lastId + using (var scope = ScopeProvider.CreateScope()) + { + EnsureInstructions(scope.Database); + bool isColdBoot = IsColdBoot(scope.Database); + + if (isColdBoot) + { + return SyncBootState.ColdBoot; + } + return SyncBootState.HasSyncState; + } + } + catch(Exception ex) + { + Logger.Warn("Error determining Sync Boot State", ex); + return SyncBootState.Unknown; + } + } + #region Notify refreshers private static ICacheRefresher GetRefresher(Guid id) diff --git a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs new file mode 100644 index 0000000000..4b8500f2d9 --- /dev/null +++ b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + /// + /// Retrieve the state of the sync service + /// + public interface ISyncBootStateAccessor + { + /// + /// Get the boot state + /// + /// + SyncBootState GetSyncBootState(); + } +} diff --git a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs new file mode 100644 index 0000000000..70cec6cc96 --- /dev/null +++ b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + /// + /// Boot state implementation for when umbraco is not in the run state + /// + public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor + { + public SyncBootState GetSyncBootState() + { + return SyncBootState.Unknown; + } + } +} diff --git a/src/Umbraco.Core/Sync/SyncBootState.cs b/src/Umbraco.Core/Sync/SyncBootState.cs new file mode 100644 index 0000000000..4abc53abba --- /dev/null +++ b/src/Umbraco.Core/Sync/SyncBootState.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + public enum SyncBootState + { + /// + /// Unknown state. Treat as HasSyncState + /// + Unknown = 0, + /// + /// Cold boot. No Sync state + /// + ColdBoot = 1, + /// + /// Warm boot. Sync state present + /// + HasSyncState = 2 + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 1160881304..6b4725c48c 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -190,6 +190,9 @@ + + + diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index afba2dcc4f..75a20ade6f 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -158,6 +159,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index eee3500495..9feb0d703b 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -204,6 +205,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index be10db3a9d..ad372c00b9 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -101,6 +101,7 @@ namespace Umbraco.Tests.Scoping Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index aaad60f7e9..b252738fee 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; using Umbraco.Core.Strings; using Umbraco.Core.Sync; +using Umbraco.Tests.TestHelpers; using Umbraco.Tests.TestHelpers.Entities; using Umbraco.Tests.Testing; using Umbraco.Web.PublishedCache; @@ -73,6 +74,7 @@ namespace Umbraco.Tests.Services Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs new file mode 100644 index 0000000000..e5f6989381 --- /dev/null +++ b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Sync; + +namespace Umbraco.Tests.TestHelpers +{ + class TestSyncBootStateAccessor : ISyncBootStateAccessor + { + private readonly SyncBootState _syncBootState; + + public TestSyncBootStateAccessor(SyncBootState syncBootState) + { + _syncBootState = syncBootState; + } + public SyncBootState GetSyncBootState() + { + return _syncBootState; + } + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 46d2216e82..4920bcda2a 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -184,6 +184,7 @@ + diff --git a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs index 2fa9d80779..26ba0db324 100644 --- a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs @@ -72,6 +72,7 @@ namespace Umbraco.Web.Compose composition.SetDatabaseServerMessengerOptions(GetDefaultOptions); composition.SetServerMessenger(); + composition.Register(factory=> factory.GetInstance() as BatchedDatabaseServerMessenger, Lifetime.Singleton); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index 0b15c0ba4b..1b96538dd0 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -13,11 +13,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public IDictionary ReadFrom(Stream stream) { - var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); // read properties count var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + var dict = new Dictionary(pcount,StringComparer.InvariantCultureIgnoreCase); // read each property for (var i = 0; i < pcount; i++) { @@ -28,13 +28,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); // create pdata and add to the dictionary - var pdatas = new List(); + var pdatas = new PropertyData[vcount]; // for each value, read and add to pdata for (var j = 0; j < vcount; j++) { var pdata = new PropertyData(); - pdatas.Add(pdata); + pdatas[j] =pdata; // everything that can be null is read/written as object // even though - culture and segment should never be null here, as 'null' represents @@ -46,7 +46,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Value = ReadObject(stream); } - dict[key] = pdatas.ToArray(); + dict[key] = pdatas; } return dict; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index c4d40f721f..21cd0bf763 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,6 +1,8 @@ using Newtonsoft.Json; using System; +using System.Buffers; using System.Collections.Generic; +using System.IO; using Umbraco.Core.Models; using Umbraco.Core.Serialization; @@ -21,13 +23,20 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateTimeZoneHandling = DateTimeZoneHandling.Utc, DateFormatString = "o" }; - + private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); - return JsonConvert.DeserializeObject(stringData, _jsonSerializerSettings); + JsonSerializer serializer = JsonSerializer.Create(_jsonSerializerSettings); + using (JsonTextReader reader = new JsonTextReader(new StringReader(stringData))) + { + // reader will get buffer from array pool + reader.ArrayPool = JsonArrayPool.Instance; + reader.PropertyNameTable = _propertyNameTable; + return serializer.Deserialize(reader); + } } public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) @@ -39,4 +48,44 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return new ContentCacheDataSerializationResult(json, null); } } + public class JsonArrayPool : IArrayPool + { + public static readonly JsonArrayPool Instance = new JsonArrayPool(); + + public char[] Rent(int minimumLength) + { + // get char array from System.Buffers shared pool + return ArrayPool.Shared.Rent(minimumLength); + } + + public void Return(char[] array) + { + // return char array to System.Buffers shared pool + ArrayPool.Shared.Return(array); + } + } + public class AutomaticJsonNameTable : DefaultJsonNameTable + { + int nAutoAdded = 0; + int maxToAutoAdd; + + public AutomaticJsonNameTable(int maxToAdd) + { + this.maxToAutoAdd = maxToAdd; + } + + public override string Get(char[] key, int start, int length) + { + var s = base.Get(key, start, length); + + if (s == null && nAutoAdded < maxToAutoAdd) + { + s = new string(key, start, length); + Add(s); + nAutoAdded++; + } + + return s; + } + } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 98d8b91386..6dac3b9afb 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,6 +1,7 @@ using System.Configuration; using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Core.Sync; using Umbraco.Core.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -27,6 +28,9 @@ namespace Umbraco.Web.PublishedCache.NuCache composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + //Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer + composition.Register(Lifetime.Singleton); + // register the NuCache database data source composition.RegisterUnique(); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs old mode 100755 new mode 100644 index 3a055223a5..5b3980ad06 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -21,6 +21,7 @@ using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Services.Implement; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Web.Install; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -62,6 +63,8 @@ namespace Umbraco.Web.PublishedCache.NuCache private bool _localContentDbExists; private bool _localMediaDbExists; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + // define constant - determines whether to use cache when previewing // to store eg routes, property converted values, anything - caching // means faster execution, but uses memory - not sure if we want it @@ -80,7 +83,10 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) + UrlSegmentProviderCollection urlSegmentProviders, + ISyncBootStateAccessor syncBootStateAccessor, + IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, + ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -100,6 +106,8 @@ namespace Umbraco.Web.PublishedCache.NuCache _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; _contentDataSerializer = contentDataSerializer; + _syncBootStateAccessor = syncBootStateAccessor; + // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member _entitySerializer = entitySerializer; @@ -218,7 +226,12 @@ namespace Umbraco.Web.PublishedCache.NuCache { var okContent = false; var okMedia = false; - + if (_syncBootStateAccessor.GetSyncBootState() == SyncBootState.ColdBoot) + { + _logger.Warn("Sync Service is in a Cold Boot state. Skip LoadCachesOnStartup as the Sync Service will trigger a full reload"); + _isReady = true; + return; + } try { if (_localContentDbExists) @@ -234,7 +247,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (!okMedia) _logger.Warn("Loading media from local db raised warnings, will reload from database."); } - + if (!okContent) LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true)); From e418bc56c64d53be847d2a827745f011da4302d0 Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 23 Apr 2021 13:40:29 +1000 Subject: [PATCH 090/147] fix merge issue and add correct assembly binding attribute --- src/Umbraco.Web.UI/web.Template.Debug.config | 7 +++++++ src/Umbraco.Web.UI/web.Template.config | 5 +++++ src/Umbraco.Web/Umbraco.Web.csproj | 7 +++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Web.UI/web.Template.Debug.config b/src/Umbraco.Web.UI/web.Template.Debug.config index d4c39e15dd..4e0ac86862 100644 --- a/src/Umbraco.Web.UI/web.Template.Debug.config +++ b/src/Umbraco.Web.UI/web.Template.Debug.config @@ -87,6 +87,13 @@ + + + + + + diff --git a/src/Umbraco.Web.UI/web.Template.config b/src/Umbraco.Web.UI/web.Template.config index ae141e5408..1de20bd835 100644 --- a/src/Umbraco.Web.UI/web.Template.config +++ b/src/Umbraco.Web.UI/web.Template.config @@ -252,6 +252,11 @@ + + + + + diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 9a567ad20f..d116163dc3 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -77,7 +77,7 @@ - 2.1.152 + 2.2.85 @@ -1314,8 +1314,7 @@ - - + - diff --git a/src/Umbraco.Web.UI.Client/src/views/components/umb-media-grid.html b/src/Umbraco.Web.UI.Client/src/views/components/umb-media-grid.html index 9754056267..5a5116225d 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/umb-media-grid.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/umb-media-grid.html @@ -59,8 +59,7 @@
    @@ -68,8 +67,7 @@
    @@ -79,13 +77,12 @@
    -
    - - + + +
    - {{item.name}}
    diff --git a/src/Umbraco.Web.UI.Client/src/views/components/umb-table.html b/src/Umbraco.Web.UI.Client/src/views/components/umb-table.html index 3b6d82f73c..0c88dedee3 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/umb-table.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/umb-table.html @@ -11,11 +11,9 @@
    - @@ -24,11 +22,10 @@ Status
    - @@ -39,15 +36,16 @@
    - - + +
    - diff --git a/src/Umbraco.Web.UI.Client/src/views/content/create.html b/src/Umbraco.Web.UI.Client/src/views/content/create.html index 1d4b646e05..afeb398574 100644 --- a/src/Umbraco.Web.UI.Client/src/views/content/create.html +++ b/src/Umbraco.Web.UI.Client/src/views/content/create.html @@ -14,7 +14,7 @@

    -
    @@ -28,7 +28,7 @@

    -
    @@ -37,8 +37,8 @@
    • - -
    • -
    -
    diff --git a/src/Umbraco.Web.UI.Client/src/views/media/create.html b/src/Umbraco.Web.UI.Client/src/views/media/create.html index 57c8f48eb4..7599c4cf7b 100644 --- a/src/Umbraco.Web.UI.Client/src/views/media/create.html +++ b/src/Umbraco.Web.UI.Client/src/views/media/create.html @@ -15,36 +15,22 @@
      -
    • -
      • -
    -
    diff --git a/src/Umbraco.Web.UI.Client/src/views/member/create.html b/src/Umbraco.Web.UI.Client/src/views/member/create.html index 5f4ad77f04..1762308a2c 100644 --- a/src/Umbraco.Web.UI.Client/src/views/member/create.html +++ b/src/Umbraco.Web.UI.Client/src/views/member/create.html @@ -6,8 +6,8 @@
    • - diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.controller.js index 5e66684ac5..486bdab044 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.controller.js @@ -2,6 +2,8 @@ angular.module("umbraco") .controller("Umbraco.PropertyEditors.Grid.MacroController", function ($scope, $timeout, editorService, macroResource, macroService, localizationService, $routeParams) { + $scope.control.icon = $scope.control.icon || 'icon-settings-alt'; + localizationService.localize("grid_clickToInsertMacro").then(function(label) { $scope.title = label; }); diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.html index c07d29d89c..300ec91bcc 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/editors/macro.html @@ -2,9 +2,9 @@
    diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js index 983644767d..21f6354c62 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/grid/grid.controller.js @@ -318,7 +318,6 @@ angular.module("umbraco") // Add items overlay menu // ********************************************* $scope.openEditorOverlay = function (event, area, index, key) { - const dialog = { view: "itempicker", filter: area.$allowedEditors.length > 15, From 57cc3d5da2d9f732d79d9a897ed5e7e452a09867 Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Mon, 31 May 2021 19:09:20 +0200 Subject: [PATCH 099/147] Use custom SVG icon in user group filter --- .../src/views/users/views/users/users.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/users/views/users/users.html b/src/Umbraco.Web.UI.Client/src/views/users/views/users/users.html index 57c87807fc..57c1128151 100644 --- a/src/Umbraco.Web.UI.Client/src/views/users/views/users/users.html +++ b/src/Umbraco.Web.UI.Client/src/views/users/views/users/users.html @@ -150,7 +150,7 @@ From 340273a6045e6466f41d5e14f335d6e990da9b75 Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Mon, 7 Jun 2021 02:40:14 +0200 Subject: [PATCH 100/147] Support custom SVG icon in Nested Content (#10368) * Use custom SVG icon in Nested Content * Get icon once * Assign value instead of compare --- .../nestedcontent/nestedcontent.propertyeditor.html | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.propertyeditor.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.propertyeditor.html index aaebb5d07e..22897e3ca2 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.propertyeditor.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.propertyeditor.html @@ -6,9 +6,9 @@
    -
    +
    - +
    @@ -18,7 +18,13 @@ ng-hide="vm.singleMode" umb-auto-focus="{{vm.focusOnNode && vm.currentNode.key === node.key ? 'true' : 'false'}}"> -
    +
    + + + +
    -
    From 31c0faa54e657978727e599a6d149c53de86563b Mon Sep 17 00:00:00 2001 From: patrickdemooij9 Date: Sat, 5 Jun 2021 22:44:13 +0200 Subject: [PATCH 101/147] Updated umbtable.directive.js for a working example I noticed that the example on https://our.umbraco.com/apidocs/v8/ui/#/api/umbraco.directives.directive:umbTable is actually not working. items should be replaced with vm.items. --- .../src/common/directives/components/umbtable.directive.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/directives/components/umbtable.directive.js b/src/Umbraco.Web.UI.Client/src/common/directives/components/umbtable.directive.js index 1554c136b6..c6f4c79ea2 100644 --- a/src/Umbraco.Web.UI.Client/src/common/directives/components/umbtable.directive.js +++ b/src/Umbraco.Web.UI.Client/src/common/directives/components/umbtable.directive.js @@ -12,7 +12,7 @@
    Date: Mon, 7 Jun 2021 03:32:01 +0200 Subject: [PATCH 102/147] Update connect color in noUiSlider (#10349) Co-authored-by: Nathan Woulfe --- .../src/less/components/umb-range-slider.less | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Umbraco.Web.UI.Client/src/less/components/umb-range-slider.less b/src/Umbraco.Web.UI.Client/src/less/components/umb-range-slider.less index 6c1980a6e4..42a13c7dda 100644 --- a/src/Umbraco.Web.UI.Client/src/less/components/umb-range-slider.less +++ b/src/Umbraco.Web.UI.Client/src/less/components/umb-range-slider.less @@ -15,6 +15,11 @@ height: 20px; top: -6px; } +.umb-range-slider .noUi-connect { + background-color: @purple-washed; + border: 1px solid @purple-l3; +} + .umb-range-slider .noUi-tooltip { padding: 2px 6px; } From 1e374105785d63ce4d4f21507b6533e7fa150150 Mon Sep 17 00:00:00 2001 From: inetzo Date: Mon, 7 Jun 2021 12:46:49 +0200 Subject: [PATCH 103/147] Convert date to datetime Datetime values where always converted to date(102) so sorting by time does not work. By converting to 120 (yyyy-mm-dd hh:mi:ss (24h)) the time is respected. --- src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs index 6f13afb24c..4d6b2eeea1 100644 --- a/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs +++ b/src/Umbraco.Core/Persistence/SqlSyntax/SqlSyntaxProviderBase.cs @@ -571,7 +571,7 @@ namespace Umbraco.Core.Persistence.SqlSyntax public virtual string CreateDefaultConstraint => "ALTER TABLE {0} ADD CONSTRAINT {1} DEFAULT ({2}) FOR {3}"; public virtual string ConvertIntegerToOrderableString => "REPLACE(STR({0}, 8), SPACE(1), '0')"; - public virtual string ConvertDateToOrderableString => "CONVERT(nvarchar, {0}, 102)"; + public virtual string ConvertDateToOrderableString => "CONVERT(nvarchar, {0}, 120)"; public virtual string ConvertDecimalToOrderableString => "REPLACE(STR({0}, 20, 9), SPACE(1), '0')"; } } From e9c3c9fea6234bbe753b75108b8339e21bcc59b0 Mon Sep 17 00:00:00 2001 From: Bjarne Fyrstenborg Date: Tue, 8 Jun 2021 10:19:26 +0200 Subject: [PATCH 104/147] Fix styling issues with listview sorting in overlay. (#10351) Remove margin bottom at input fields --- .../components/umb-list-view-settings.less | 7 +- .../src/less/listview.less | 104 +++++++++--------- 2 files changed, 56 insertions(+), 55 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/less/components/umb-list-view-settings.less b/src/Umbraco.Web.UI.Client/src/less/components/umb-list-view-settings.less index ba46c68a57..9eb00d4437 100644 --- a/src/Umbraco.Web.UI.Client/src/less/components/umb-list-view-settings.less +++ b/src/Umbraco.Web.UI.Client/src/less/components/umb-list-view-settings.less @@ -66,11 +66,8 @@ padding-left: 15px; } - .ui-sortable-handle { - min-height: 37px; - display: flex; - width:0; - align-items: center; + input[type="text"] { + margin-bottom: 0; } } } diff --git a/src/Umbraco.Web.UI.Client/src/less/listview.less b/src/Umbraco.Web.UI.Client/src/less/listview.less index 582da12804..9321577c15 100644 --- a/src/Umbraco.Web.UI.Client/src/less/listview.less +++ b/src/Umbraco.Web.UI.Client/src/less/listview.less @@ -222,63 +222,67 @@ /* ---------- LAYOUTS ---------- */ .list-view-layout { - display: flex; - align-items: center; - padding: 10px 15px; - background: @gray-10; - margin-bottom: 1px; -} + display: flex; + align-items: center; + padding: 10px 15px; + background: @gray-10; + margin-bottom: 1px; -.list-view-layout__sort-handle { - font-size: 14px; - color: @gray-8; - margin-right: 15px; -} + &__sort-handle { + font-size: 14px; + color: @gray-8; + margin-right: 15px; + } -.list-view-layout__name { - flex: 5; - font-weight: bold; - margin-right: 15px; - display: flex; - align-content: center; - flex-wrap: wrap; - line-height: 1.2em; -} + &__name { + flex: 5; + font-weight: bold; + margin-right: 15px; + display: flex; + align-content: center; + flex-wrap: wrap; + line-height: 1.2em; + } -.list-view-layout__name-text { - margin-right: 3px; -} - -.list-view-layout__system { - font-size: 10px; - font-weight: normal; -} + &__name-text { + margin-right: 3px; + } -.list-view-layout__path { - flex: 10; - margin-right: 15px; -} + &__system { + font-size: 10px; + font-weight: normal; + } -.list-view-layout__icon-wrapper { - margin-right: 10px; -} + &__path { + flex: 10; + margin-right: 15px; + } -.list-view-layout__icon { - font-size: 18px; - vertical-align: middle; - border: 1px solid @gray-8; - background: @white; - padding: 0; - display: flex; - align-items: center; - justify-content: center; - width: 30px; - height: 30px; -} + &__icon-wrapper { + margin-right: 10px; + } -.list-view-layout__remove { - position: relative; - cursor: pointer; + &__icon { + font-size: 18px; + vertical-align: middle; + border: 1px solid @gray-8; + background: @white; + padding: 0; + display: flex; + align-items: center; + justify-content: center; + width: 30px; + height: 30px; + } + + &__remove { + position: relative; + cursor: pointer; + } + + input[type="text"] { + margin-bottom: 0; + } } .list-view-add-layout { From b06f902865812120174ddc9a0851edcc07088e62 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 8 Jun 2021 12:45:21 +0200 Subject: [PATCH 105/147] Update config.yml --- .github/ISSUE_TEMPLATE/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d5418ad270..edee3cc9d9 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,8 @@ blank_issues_enabled: false contact_links: + - name: 📮 Features and ideas + url: https://github.com/umbraco/Umbraco-CMS/discussions/new?categories=features-and-ideas + about: Start a new discussion when you have ideas or feature requests, eventually discussions can turn into plans - name: ⁉️ Support Question url: https://our.umbraco.com about: This issue tracker is NOT meant for support questions. If you have a question, please join us on the forum. From ef20264df6c04123ee14536c68587c38d42a8d43 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 8 Jun 2021 12:47:44 +0200 Subject: [PATCH 106/147] Delete 02_feature_request.yml --- .github/ISSUE_TEMPLATE/02_feature_request.yml | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/02_feature_request.yml diff --git a/.github/ISSUE_TEMPLATE/02_feature_request.yml b/.github/ISSUE_TEMPLATE/02_feature_request.yml deleted file mode 100644 index 5d53b2f12e..0000000000 --- a/.github/ISSUE_TEMPLATE/02_feature_request.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: 📮 Feature Request -description: Open a feature request, if you want to propose a new feature. -labels: type/feature -body: -- type: dropdown - id: version - attributes: - label: Umbraco version - description: Which major Umbraco version are you proposing a feature for? - options: - - v8 - - v9 - validations: - required: true -- type: textarea - id: summary - attributes: - label: Description - description: Write a brief desciption of your proposed new feature. - validations: - required: true -- type: textarea - attributes: - label: How can you help? - id: help - description: Umbraco''s core team has limited available time, but maybe you can help? - placeholder: > - If we can not work on your suggestion, please don't take it personally. Most likely, it's either: - - - We think your idea is valid, but we can't find the time to work on it. - - - Your idea might be better suited as a package, if it's not suitable for the majority of users. From b63ee6e7bf7baf7fef2770acbbf31fbc81781f6b Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 8 Jun 2021 13:00:40 +0200 Subject: [PATCH 107/147] Update config.yml --- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index edee3cc9d9..ecf10b8854 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: false contact_links: - - name: 📮 Features and ideas - url: https://github.com/umbraco/Umbraco-CMS/discussions/new?categories=features-and-ideas + - name: 💡 Features and ideas + url: https://github.com/umbraco/Umbraco-CMS/discussions/new?category=features-and-ideas about: Start a new discussion when you have ideas or feature requests, eventually discussions can turn into plans - name: ⁉️ Support Question url: https://our.umbraco.com From 4e60e7dec17c308ddfdec042b459da6bb7572fa5 Mon Sep 17 00:00:00 2001 From: Nikcio Date: Fri, 28 May 2021 00:34:00 +0200 Subject: [PATCH 108/147] Translation added Added some danish translations and fixed a translation problem in the copy document. --- .../src/views/content/copy.html | 2 +- src/Umbraco.Web.UI/Umbraco/config/lang/da.xml | 179 ++++++++++++++++-- 2 files changed, 159 insertions(+), 22 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/content/copy.html b/src/Umbraco.Web.UI.Client/src/views/content/copy.html index 111dacd7cb..399e0b8bf0 100644 --- a/src/Umbraco.Web.UI.Client/src/views/content/copy.html +++ b/src/Umbraco.Web.UI.Client/src/views/content/copy.html @@ -11,7 +11,7 @@
    - {{source.name}} was copied to + {{source.name}} was copied to {{target.name}}
    diff --git a/src/Umbraco.Web.UI/Umbraco/config/lang/da.xml b/src/Umbraco.Web.UI/Umbraco/config/lang/da.xml index b2b11fdef5..b23d674072 100644 --- a/src/Umbraco.Web.UI/Umbraco/config/lang/da.xml +++ b/src/Umbraco.Web.UI/Umbraco/config/lang/da.xml @@ -31,6 +31,7 @@ Udgiv Afpublicér Genindlæs elementer + Fjern Genudgiv hele sitet Omdøb Gendan @@ -55,6 +56,7 @@ Sæt rettigheder Lås op Opret indholdsskabelon + Gensend Invitation Standard værdi @@ -141,18 +143,21 @@ Gem og send til udgivelse Gem listevisning Planlæg + Se side Forhåndsvisning Forhåndsvisning er deaktiveret fordi der ikke er nogen skabelon tildelt Vælg formattering Vis koder Indsæt tabel + Generer modeller og luk Gem og generer modeller Fortryd Genskab + Rul tilbage Slet tag Fortryd Bekræft - Flere publiseringsmuligheder + Flere publiseringsmuligheder Indsæt Indsæt og luk @@ -160,6 +165,7 @@ For Brugeren har slettet indholdet Brugeren har afpubliceret indholdet + Brugeren har afpubliceret indholdet for sprogene: %0% Brugeren har gemt og udgivet indholdet Brugeren har gemt og udgivet indholdet for sprogene: %0% Brugeren har gemt indholdet @@ -168,8 +174,10 @@ Brugeren har kopieret indholdet Brugeren har tilbagerullet indholdet til en tidligere tilstand Brugeren har sendt indholdet til udgivelse + Brugeren har sendt indholdet til udgivelse for sprogene: %0% Brugeren har sendt indholdet til oversættelse Brugeren har sorteret de underliggende sider + %0% Kopieret Udgivet Udgivet @@ -178,10 +186,13 @@ Gemt Slettet Afpubliceret + Afpubliceret Indhold tilbagerullet Sendt til udgivelse + Sendt til udgivelse Sendt til oversættelse Sorteret + Brugerdefineret Historik (alle sprog) @@ -241,6 +252,7 @@ Ingen dato valgt Sidetitel Dette medie har ikke noget link + Intet indhold kan tilføjes for dette element Egenskaber Dette dokument er udgivet, men ikke synligt da den overliggende side '%0%' ikke er udgivet! Dette sprog er udgivet, men ikke synligt, da den overliggende side '%0%' ikke er udgivet! @@ -263,6 +275,7 @@ Statistik Titel (valgfri) Alternativ tekst (valgfri) + Overskrift (valgfri) Type Hvilke varianter vil du udgive? Vælg hvilke varianter, der skal gemmes. @@ -272,6 +285,8 @@ Sidst redigeret Tidspunkt for seneste redigering Fjern fil + Klik her for at fjerne billedet fra medie filen + Klik her for at fjerne filen fra medie filen Link til dokument Medlem af grupper(ne) Ikke medlem af grupper(ne) @@ -283,6 +298,9 @@ Er du sikker på, at du vil slette alle elementer? Egenskaben %0% anvender editoren %1% som ikke er understøttet af Nested Content. Der er ikke konfigureret nogen indholdstyper for denne egenskab. + Tilføj element type + Vælg element type + Vælg gruppen, hvis værdier skal vises. Hvis dette er efterladt blankt vil den første gruppe på element typen bruges. %0% fra %1% Tilføj en ny tekstboks Fjern denne tekstboks @@ -301,6 +319,14 @@ Ikke-udgivne sprog Uændrede sprog Disse sprog er ikke blevet oprettet + Alle nye varianter vil blive gemt. + Hvilke varianter skal udgives? + Vælg, hvilke varianter skal gemmes. + Vælg varianter som skal sendes til gennemgang. + Sæt udgivnings tidspunkt... + Vælg varianterne som skal afpubliceres. Afpublicering af et krævet sprog vil afpublicere alle varianter. + De følgende varianter er krævet for at en udgivelse kan finde sted: + Vi er ikke klar til at udgive Klar til at udgive? Klar til at gemme? Send til godkendelse @@ -326,10 +352,12 @@ Maks filstørrelse er Medie rod Flytning af mediet fejlede + Overordnet og destinations mappe kan ikke være den samme Kopiering af mediet fejlede Oprettelse af mappen under parent med id %0% fejlede Omdøbning af mappen med id %0% fejlede Træk dine filer ind i dropzonen for, at uploade dem til mediebiblioteket. + Upload er ikke tiladt på denne lokation Opret et nyt medlem @@ -337,16 +365,16 @@ Medlemgrupper har ingen yderligere egenskaber til redigering. - Kopiering af indholdstypen fejlede - Flytning af indholdstypen fejlede + Kopiering af indholdstypen fejlede + Flytning af indholdstypen fejlede - Kopiering af medietypen fejlede - Flytning af medietypen fejlede - Auto vælg + Kopiering af medietypen fejlede + Flytning af medietypen fejlede + Auto vælg - Kopiering af medlemstypen fejlede + Kopiering af medlemstypen fejlede Hvor ønsker du at oprette den nye %0% @@ -359,6 +387,7 @@ Den valgte side i træet tillader ikke at sider oprettes under den. Rediger tilladelser for denne dokumenttype. Opret en ny dokumenttype + Dokumenttyper inde i Indstillinger sektionen, ved at ændre Tillad på rodniveau indestillingen under Permissions.]]> "media typer"    .]]> Det valgte medie i træet tillader ikke at medier oprettes under det. Rediger tilladelser for denne medietype. @@ -435,6 +464,9 @@ Luk denne dialog Er du sikker på at du vil slette Er du sikker på du vil deaktivere + Er du sikker på at du vil fjerne + %0%]]> + %0%]]> Er du sikker på at du vil forlade Umbraco? Er du sikker? Klip @@ -449,6 +481,7 @@ Indsæt makro Indsæt tabel Dette vil slette sproget + Ændring af kulturen for et sprog kan forsage en krævende opration og vil resultere i indholds cache og indeksering vil blive genlavet Sidst redigeret Link Internt link: @@ -513,6 +546,9 @@ Vælg konfiguration Vælg snippet Dette vil slette noden og alle dets sprog. Hvis du kun vil slette et sprog, så afpublicér det i stedet. + %0%]]> + %0% fra gruppen]]> + Ja, fjern Der er ingen ordbogselementer. @@ -574,6 +610,9 @@ #value eller ?key=value Indtast alias... Genererer alias... + Opret element + Rediger + Navn Opret brugerdefineret listevisning @@ -638,6 +677,7 @@ Denne egenskab er ugyldig + Valgmuligheder Om Handling Muligheder @@ -655,6 +695,7 @@ Ryd Luk Luk vindue + Luk vindue Kommentar Bekræft Proportioner @@ -663,6 +704,7 @@ Fortsæt Kopiér Opret + Beskær sektion Database Dato Standard @@ -788,6 +830,8 @@ Andet Artikler Videoer + installere + Avatar til Blå @@ -805,7 +849,7 @@ Vis genveje Brug listevisning Tillad på rodniveau - Lommentér/Udkommentér linjer + Kommentér/Udkommentér linjer Slet linje Kopiér linjer op Kopiér linjer ned @@ -1034,6 +1078,7 @@ Mange hilsner fra Umbraco robotten Bemærk: at dokumenter og medier som afhænger af denne pakke vil muligvis holde op med at virke, så vær forsigtig. Hvis i tvivl, kontakt personen som har udviklet pakken.]]> Pakke version + Opgraderer fra version Pakke allerede installeret Denne pakke kan ikke installeres, den kræver en minimum Umbraco version af Afinstallerer... @@ -1071,8 +1116,13 @@ Mange hilsner fra Umbraco robotten Hvis du ønsker at give adgang til enkelte medlemmer + Utilstrækkelige bruger adgang til a udgive alle under dokumenter Udgivelsen kunne ikke udgives da publiceringsdato er sat - + + Sortering udført Træk de forskellige sider op eller ned for at indstille hvordan de skal arrangeres, eller klik på kolonnehovederne for at sortere hele rækken af sider + Denne node har ingen under noder at sortere Validering Valideringsfejl skal rettes før elementet kan gemmes Fejlet Gemt + Gemt. For at se ændringerne skal du genindlæse din browser Utilstrækkelige brugerrettigheder, kunne ikke fuldføre handlingen Annulleret Handlingen blev annulleret af et 3. part tilføjelsesprogram @@ -1214,10 +1266,16 @@ Mange hilsner fra Umbraco robotten Udgivelse fejlede da overliggende side ikke er udgivet Indhold publiceret og nu synligt for besøgende + %0% dokumenter udgivet og synlige på hjemmesiden + %0% udgivet og synligt på hjemmesiden + %0% dokumenter udgivet for sprogene %1% og synlige på hjemmesiden Indhold gemt Husk at publicere for at gøre det synligt for besøgende + En planlægning for udgivelse er blevet opdateret + %0% gemt Send til Godkendelse Rettelser er blevet sendt til godkendelse + %0% rettelser er blevet sendt til godkendelse Medie gemt Medie gemt uden problemer Medlem gemt @@ -1244,6 +1302,8 @@ Mange hilsner fra Umbraco robotten Skabelon gemt Skabelon gemt uden fejl! Indhold fjernet fra udgivelse + Indhold variation %0% afpubliceret + Det krævet sprog '%0%' var afpubliceret. Alle sprog for dette indholds element er nu afpubliceret. Partial view gemt Partial view gemt uden fejl! Partial view ikke gemt @@ -1258,11 +1318,20 @@ Mange hilsner fra Umbraco robotten Brugergrupper er blevet indstillet Låste %0% brugere op %0% er nu låst op + Medlem blev exportet til fil + Der skete en fejl under exporteringen af medlemmet Brugeren %0% blev slettet Invitér bruger Invitationen blev gensendt til %0% + Kan ikke udgive dokumentet da det krævet '%0%' ikke er udgivet + Validering fejlede for sproget '%0%' Dokumenttypen blev eksporteret til en fil Der skete en fejl under eksport af en dokumenttype + Udgivelses datoen kan ikke ligge i fortiden + Kan ikke planlægge dokumentes udgivelse da det krævet '%0%' ikke er udgivet + Kan ikke planlægge dokumentes udgivelse da det krævet '%0%' har en senere udgivelses dato end et ikke krævet sprog + Afpubliceringsdatoen kan ikke ligge i fortiden + Afpubliceringsdatoen kan ikke være før udgivelsesdatoen Tilføj style @@ -1333,6 +1402,7 @@ Mange hilsner fra Umbraco robotten ]]> Query builder sider returneret, på + Kopier til udkilpsholder Returner alt indhold indhold af typen "%0%" @@ -1381,10 +1451,12 @@ Mange hilsner fra Umbraco robotten Grid layout Et layout er det overordnede arbejdsområde til dit grid - du vil typisk kun behøve ét eller to Tilføj grid layout + Rediger grid layout Juster dit layout ved at justere kolonnebredder og tilføj yderligere sektioner Rækkekonfigurationer Rækker er foruddefinerede celler, der arrangeres vandret Tilføj rækkekonfiguration + Rediger rækkekonfiguration Juster rækken ved at indstille cellebredder og tilføje yderligere celler Ingen yderligere konfiguration tilgængelig Kolonner @@ -1399,6 +1471,9 @@ Mange hilsner fra Umbraco robotten Vælg ekstra Vælg standard er tilføjet + Advarsel + Du sletter en rækkekonfiguration + Sletning af et rækkekonfigurations navn vil resultere i et tab af data for alle eksiterende indhold som bruger dens konfiguration. Maksimalt emner Efterlad blank eller sæt til 0 for ubegrænset @@ -1466,6 +1541,7 @@ Mange hilsner fra Umbraco robotten Element-type Er en Element-type En Element-type er tiltænkt brug i f.eks. Nested Content, ikke i indholdstræet. + En Dokumenttype kan ikke ændres til en Element-type efter den er blevet brugt til at oprette en eller flere indholds elementer. Dette benyttes ikke for en Element-type Du har lavet ændringer til denne egenskab. Er du sikker på at du vil kassere dem? Visning @@ -1509,21 +1585,27 @@ Mange hilsner fra Umbraco robotten Casing Kodning Felt som skal indsættes - Konvertér linieskift - Erstatter et linieskift med html-tag'et &lt;br&gt; + Konvertér linjeskift + Ja, konverter linjeskift + Erstatter et linjeskift med html-tag'et &lt;br&gt; Custom felter Ja, kun dato Format og kodning Formatér som dato + Formater værdien som en dato eller en dato med tid, i forhold til den aktive kultur HTML indkod Vil erstatte specielle karakterer med deres HTML jævnbyrdige. Denne tekst vil blive sat ind lige efter værdien af feltet Denne tekst vil blive sat ind lige før værdien af feltet Lowercase + Ændre udskrift Ingen + Udskrift eksempel Indsæt efter felt Indsæt før felt Rekursivt + Ja, lav det rekursivt + Separator Fjern paragraf-tags Fjerner eventuelle &lt;P&gt; omkring teksten Standard felter @@ -1609,6 +1691,8 @@ Mange hilsner fra Umbraco robotten Skift dit kodeord Skift billede Nyt kodeord + Minium %0% karakterer tilbage! + Der skal som minium være %0% specielle karakterer. er ikke blevet låst ude Kodeordet er ikke blevet ændret Gentag dit nye kodeord @@ -1643,8 +1727,10 @@ Mange hilsner fra Umbraco robotten Adgangskode Nulstil kodeord Dit kodeord er blevet ændret! + Kodeord ændret Bekræft venligst dit nye kodeord Indtast dit nye kodeord + Dit nye kodeord kan ikke være blankt! Nuværende kodeord ugyldig nuværende kodeord Dit nye kodeord må ikke være tomt! @@ -1664,6 +1750,7 @@ Mange hilsner fra Umbraco robotten Vælg brugergrupper Ingen startnode valgt Ingen startnoder valgt + Indhold startnode Begræns indholdstræet til en bestemt startnode Indhold startnoder Begræns indholdstræet til bestemte startnoder @@ -1677,7 +1764,7 @@ Mange hilsner fra Umbraco robotten er blevet inviteret En invitation er blevet sendt til den nye bruger med oplysninger om, hvordan man logger ind i Umbraco. Hej og velkommen til Umbraco! På bare 1 minut vil du være klar til at komme i gang, vi skal bare have dig til at oprette en adgangskode og tilføje et billede til din avatar. - Velkommen til Umbraco! Desværre er din invitation udløbet. Kontakt din administrator og bed om at gensende invitationen. + Velkommen til Umbraco! Desværre er din invitation udløbet. Kontakt din administrator og bed om at gensende invitationen. Hvis du uploader et billede af dig selv, gør du det nemt for andre brugere at genkende dig. Klik på cirklen ovenfor for at uploade et billede. Forfatter Skift @@ -1690,6 +1777,10 @@ Mange hilsner fra Umbraco robotten Tilbage til brugere Umbraco: Invitation

    Hej %0%, du er blevet inviteret af %1% til Umbraco backoffice.

    Besked fra %1%: %2%

    Klik på dette link for acceptere invitationen

    Hvis du ikke kan klikke på linket, så kopier og indsæt denne URL i dit browservindue

    %3%

    ]]>     + Inviter + Gensender invitation... + Slet bruger + Er du sikker på du ønsker at slette denne brugers konto? Alle Aktiv Deaktiveret @@ -1701,6 +1792,7 @@ Mange hilsner fra Umbraco robotten Nyeste Ældste Sidst logget ind + Ingen brugere er blevet tilføjet Validering @@ -1709,7 +1801,9 @@ Mange hilsner fra Umbraco robotten Valider som URL ...eller indtast din egen validering Feltet er påkrævet + Indtast en selvvalgt validerings fejlbesked (valgfrit) Indtast et regulært udtryk + Indtast en selvvalgt validerings fejlbesked (valgfrit) Du skal tilføje mindst Du kan kun have Tilføj op til @@ -1724,14 +1818,18 @@ Mange hilsner fra Umbraco robotten Værdien kan ikke være tom Værdien kan ikke være tom Værdien er ugyldig, som ikke matcher det korrekte format + Selvvalgt validering %1% mere.]]> %1% for mange.]]> Slå URL tracker fra Slå URL tracker til + Kultur Original URL Viderestillet til + Viderestil URL håndtering + De følgende URLs viderestiller til dette indholds element Der er ikke lavet nogen viderestillinger Når en udgivet side bliver omdøbt eller flyttet, vil en viderestilling automatisk blive lavet til den nye side. Er du sikker på at du vil fjerne viderestillingen fra '%0%' til '%1%'? @@ -1806,12 +1904,34 @@ Mange hilsner fra Umbraco robotten Opsæt rettigheder på %0% Juster soterings rækkefølgen for %0% Opret indholds skabelon baseret på %0% + Åben kontext menu for Aktivt sprog Skift sprog til Opret ny mappe + Delvist View + Delvist View Macro + Medlem + Data type + Søg i viderestillings dashboardet + Søg i brugergruppe sektionen + Søg i bruger sektionen Opret element + Opret Rediger Navn + Tilføj ny række + Vis flere muligheder + Søg i Umbraco backoffice + Søg efter indholdsnoder, medienoder osv. i backoffice + Når autoudfyldnings resultaterne er klar, tryk op og ned pilene, eller benyt tab knappen og brug enter knappen til at vælge. + Vej + Fundet i + Har oversættelse + Mangler oversættelse + Ordbogs elementer + Udfør handling %0% på %1% noden + Tilføj billede overskrift + Søg i indholdstræet Referencer @@ -1823,12 +1943,17 @@ Mange hilsner fra Umbraco robotten Brugt i Medlems Typer Ingen referencer til Medlems Typer. Brugt af + Brugt i Dokumenter + Brugt i Medlemmer + Brugt i Medier Slet gemte søgning Log type + Gemte søgninger Gem søgning Indtast et navn for din søgebetingelse + Filter søgning Samlet resultat Dato Type @@ -1853,6 +1978,17 @@ Mange hilsner fra Umbraco robotten Find logs med Namespace Find logs med maskin navn Åben + Henter + Hver 2 sekunder + Hver 5 sekunder + Hver 10 sekunder + Hver 20 sekunder + Hver 30 sekunder + Henter hver 2s + Henter hver 5s + Henter hver 10s + Henter hver 20s + Henter hver 30s Kopier %0% @@ -1863,6 +1999,7 @@ Mange hilsner fra Umbraco robotten Åben egenskabshandlinger + Luk egenskabshandlinger Vælg elementtype @@ -1908,12 +2045,12 @@ Mange hilsner fra Umbraco robotten Tilføj %0% Feltet %0% bruger editor %1% som ikke er supporteret for blokke. - - Hvad er Indholdsskabeloner? - Indholdsskabeloner er foruddefineret indhold der kan vælges når der oprettes nye indholdselementer. - Hvordan opretter jeg en Indholdsskabelon? - - + Hvad er Indholdsskabeloner? + Indholdsskabeloner er foruddefineret indhold der kan vælges når der oprettes nye indholdselementer. + Hvordan opretter jeg en Indholdsskabelon? + + Der er to måder at oprette Indholdsskabeloner på:

    • Højreklik på en indholdsnode og vælg "Opret indholdsskabelon" for at oprette en ny Indholdsskabelon.
      • @@ -1922,9 +2059,9 @@ Mange hilsner fra Umbraco robotten

      Når indholdsskabelonen har fået et navn, kan redaktører begynde at bruge indholdsskabelonen som udgangspunkt for deres nye side.

      ]]> - Hvordan vedligeholder jeg Indholdsskabeloner? - Du kan redigere og slette Indholdsskabeloner fra "Indholdsskabeloner" i sektionen Indstillinger. Fold dokumenttypen som Indholdsskabelonen er baseret på ud og klik på den for at redigere eller slette den. - + Hvordan vedligeholder jeg Indholdsskabeloner? + Du kan redigere og slette Indholdsskabeloner fra "Indholdsskabeloner" i sektionen Indstillinger. Fold dokumenttypen som Indholdsskabelonen er baseret på ud og klik på den for at redigere eller slette den. + Afslut Afslut forhåndsvisning From 40c4c875c4673da125e8d04e4fc9ba617e130fdd Mon Sep 17 00:00:00 2001 From: Jan Skovgaard <1932158+BatJan@users.noreply.github.com> Date: Wed, 9 Jun 2021 01:02:37 +0200 Subject: [PATCH 109/147] Link picker: Make link input full width if anchor option is disabled (#10313) * Add missing focus styling * Add width modifier to link input * same result without adding new classes Co-authored-by: BatJan Co-authored-by: Jan Skovgaard Olsen Co-authored-by: Nathan Woulfe --- src/Umbraco.Web.UI.Client/src/less/property-editors.less | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Umbraco.Web.UI.Client/src/less/property-editors.less b/src/Umbraco.Web.UI.Client/src/less/property-editors.less index 328ba2229b..11d11c7e3a 100644 --- a/src/Umbraco.Web.UI.Client/src/less/property-editors.less +++ b/src/Umbraco.Web.UI.Client/src/less/property-editors.less @@ -950,6 +950,12 @@ .umb-linkpicker__url { width: 50%; padding-right: 5px; + + // when the anchor input is hidden by config + // the URL input should be full-width + &:only-child { + width: 100%; + } } .umb-linkpicker__anchor { From b97f6dcb2fad9d57f26295d2dc79e4c3bd621f57 Mon Sep 17 00:00:00 2001 From: Lucas Michaelsen <69140700+lucasmichaelsengorm@users.noreply.github.com> Date: Tue, 15 Jun 2021 01:09:26 +0200 Subject: [PATCH 110/147] HTML Symantic error UL > LI >LI changes to UL > LI > UL > LI (#10455) * HTML Symantic error UL > LI >LI changes to UL > LI > UL > LI - ScreenReader reads propperly * ensurce render if condition is meet. If any child render a subset of unorder list withs is list-items --- .../src/views/components/tree/umb-tree.html | 71 ++++++++++--------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree.html b/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree.html index 59d67ee2e1..e50707f2d4 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/tree/umb-tree.html @@ -13,26 +13,28 @@ - +
    - - + +
      + + +
    @@ -51,26 +53,27 @@ - +
    - - +
      + + +
    From efeda47e30e395635e5219a0ca85fc7a14d9cfdb Mon Sep 17 00:00:00 2001 From: patrickdemooij9 Date: Tue, 15 Jun 2021 01:22:18 +0200 Subject: [PATCH 111/147] 10430: Fix typo (#10440) * 10430: Fix typo * 10430: ng-disabled instead of disabled --- .../mediapicker3/umb-media-picker3-property-editor.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umb-media-picker3-property-editor.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umb-media-picker3-property-editor.html index aa9f50b7df..cb6d9e5e26 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umb-media-picker3-property-editor.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/mediapicker3/umb-media-picker3-property-editor.html @@ -42,7 +42,7 @@ id="{{vm.model.alias}}" type="button" class="btn-reset umb-media-card-grid__create-button umb-outline" - disbled="!vm.allowAdd" + ng-disabled="!vm.allowAdd" ng-click="vm.addMediaAt(vm.model.value.length, $event)">
    From f303fa0cc3c0209695e7b718f95274386591e8be Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 09:51:23 +0200 Subject: [PATCH 112/147] Make sure we can build in VS2022 --- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index f3652a1273..d3e8a355c9 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -319,7 +319,7 @@ $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v12.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v14.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v15.0 - $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v16.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v17.0 @@ -329,8 +329,7 @@ $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v14.0\Web\Microsoft.Web.Publishing.Tasks.dll $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v15.0\Web\Microsoft.Web.Publishing.Tasks.dll $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v16.0\Web\Microsoft.Web.Publishing.Tasks.dll - - $(ProgramFiles32)\Microsoft Visual Studio\2019\Preview\MSBuild\Microsoft\VisualStudio\v16.0\Web\Microsoft.Web.Publishing.Tasks.dll + $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v17.0\Web\Microsoft.Web.Publishing.Tasks.dll @@ -436,4 +435,4 @@ - \ No newline at end of file + From 2c7413693b9edd1bd34475179f033c356c0fd666 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 11:03:21 +0200 Subject: [PATCH 113/147] Prefix the new media types to stop them from clashing when people make document types with that alias (#10483) --- src/Umbraco.Core/Constants-Conventions.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Core/Constants-Conventions.cs b/src/Umbraco.Core/Constants-Conventions.cs index c8233c8d34..da2d2cd0ec 100644 --- a/src/Umbraco.Core/Constants-Conventions.cs +++ b/src/Umbraco.Core/Constants-Conventions.cs @@ -121,22 +121,22 @@ namespace Umbraco.Core /// /// MediaType alias for a video. /// - public const string Video = "Video"; + public const string Video = "umbracoMediaVideo"; /// /// MediaType alias for an audio. /// - public const string Audio = "Audio"; + public const string Audio = "umbracoMediaAudio"; /// /// MediaType alias for an article. /// - public const string Article = "Article"; + public const string Article = "umbracoMediaArticle"; /// /// MediaType alias for vector graphics. /// - public const string VectorGraphics = "VectorGraphics"; + public const string VectorGraphics = "umbracoMediaVectorGraphics"; /// /// MediaType alias indicating allowing auto-selection. From 64ebbae5b8bfe37113d95231ec85685cf56cbb22 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 11:37:32 +0200 Subject: [PATCH 114/147] Keep the old constants as they were and add new "Name" constants, in case people were already using the old constants to query by alias --- src/Umbraco.Core/Constants-Conventions.cs | 20 +++++++++++++++++++ .../Migrations/Install/DatabaseDataCreator.cs | 8 ++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Core/Constants-Conventions.cs b/src/Umbraco.Core/Constants-Conventions.cs index da2d2cd0ec..98df39202c 100644 --- a/src/Umbraco.Core/Constants-Conventions.cs +++ b/src/Umbraco.Core/Constants-Conventions.cs @@ -118,6 +118,26 @@ namespace Umbraco.Core /// public const string Image = "Image"; + /// + /// MediaType name for a video. + /// + public const string VideoName = "Video"; + + /// + /// MediaType name for an audio. + /// + public const string AudioName = "Audio"; + + /// + /// MediaType name for an article. + /// + public const string ArticleName = "Article"; + + /// + /// MediaType name for vector graphics. + /// + public const string VectorGraphicsName = "VectorGraphics"; + /// /// MediaType alias for a video. /// diff --git a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs index 6a56141491..a8d089dd05 100644 --- a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs +++ b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs @@ -130,10 +130,10 @@ namespace Umbraco.Core.Migrations.Install _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1031, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1031", SortOrder = 2, UniqueId = new Guid("f38bd2d7-65d0-48e6-95dc-87ce06ec2d3d"), Text = Constants.Conventions.MediaTypes.Folder, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1032, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1032", SortOrder = 2, UniqueId = new Guid("cc07b313-0843-4aa8-bbda-871c8da728c8"), Text = Constants.Conventions.MediaTypes.Image, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1033, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1033", SortOrder = 2, UniqueId = new Guid("4c52d8ab-54e6-40cd-999c-7a5f24903e4d"), Text = Constants.Conventions.MediaTypes.File, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1034, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1034", SortOrder = 2, UniqueId = new Guid("f6c515bb-653c-4bdc-821c-987729ebe327"), Text = Constants.Conventions.MediaTypes.Video, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1035, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1035", SortOrder = 2, UniqueId = new Guid("a5ddeee0-8fd8-4cee-a658-6f1fcdb00de3"), Text = Constants.Conventions.MediaTypes.Audio, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1036, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1036", SortOrder = 2, UniqueId = new Guid("a43e3414-9599-4230-a7d3-943a21b20122"), Text = Constants.Conventions.MediaTypes.Article, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1037, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1037", SortOrder = 2, UniqueId = new Guid("c4b1efcf-a9d5-41c4-9621-e9d273b52a9c"), Text = "Vector Graphics (SVG)", NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1034, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1034", SortOrder = 2, UniqueId = new Guid("f6c515bb-653c-4bdc-821c-987729ebe327"), Text = Constants.Conventions.MediaTypes.VideoName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1035, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1035", SortOrder = 2, UniqueId = new Guid("a5ddeee0-8fd8-4cee-a658-6f1fcdb00de3"), Text = Constants.Conventions.MediaTypes.AudioName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1036, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1036", SortOrder = 2, UniqueId = new Guid("a43e3414-9599-4230-a7d3-943a21b20122"), Text = Constants.Conventions.MediaTypes.ArticleName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1037, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1037", SortOrder = 2, UniqueId = new Guid("c4b1efcf-a9d5-41c4-9621-e9d273b52a9c"), Text = Constants.Conventions.MediaTypes.VectorGraphicsName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = Constants.DataTypes.Tags, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = $"-1,{Constants.DataTypes.Tags}", SortOrder = 2, UniqueId = new Guid("b6b73142-b9c1-4bf8-a16d-e1c23320b549"), Text = "Tags", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = Constants.DataTypes.ImageCropper, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = $"-1,{Constants.DataTypes.ImageCropper}", SortOrder = 2, UniqueId = new Guid("1df9f033-e6d4-451f-b8d2-e0cbc50a836f"), Text = "Image Cropper", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1044, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1044", SortOrder = 0, UniqueId = new Guid("d59be02f-1df9-4228-aa1e-01917d806cda"), Text = Constants.Conventions.MemberTypes.DefaultAlias, NodeObjectType = Constants.ObjectTypes.MemberType, CreateDate = DateTime.Now }); From 70590ce9934d821292979da6f22efb8e0160ec81 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 11:42:09 +0200 Subject: [PATCH 115/147] Invert the previous change so querying with these constants will give the same results as in 8.14.0 --- src/Umbraco.Core/Constants-Conventions.cs | 16 ++++++++-------- .../Migrations/Install/DatabaseDataCreator.cs | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Umbraco.Core/Constants-Conventions.cs b/src/Umbraco.Core/Constants-Conventions.cs index 98df39202c..37267a5e22 100644 --- a/src/Umbraco.Core/Constants-Conventions.cs +++ b/src/Umbraco.Core/Constants-Conventions.cs @@ -121,42 +121,42 @@ namespace Umbraco.Core /// /// MediaType name for a video. /// - public const string VideoName = "Video"; + public const string Video = "Video"; /// /// MediaType name for an audio. /// - public const string AudioName = "Audio"; + public const string Audio = "Audio"; /// /// MediaType name for an article. /// - public const string ArticleName = "Article"; + public const string Article = "Article"; /// /// MediaType name for vector graphics. /// - public const string VectorGraphicsName = "VectorGraphics"; + public const string VectorGraphics = "VectorGraphics"; /// /// MediaType alias for a video. /// - public const string Video = "umbracoMediaVideo"; + public const string VideoAlias = "umbracoMediaVideo"; /// /// MediaType alias for an audio. /// - public const string Audio = "umbracoMediaAudio"; + public const string AudioAlias = "umbracoMediaAudio"; /// /// MediaType alias for an article. /// - public const string Article = "umbracoMediaArticle"; + public const string ArticleAlias = "umbracoMediaArticle"; /// /// MediaType alias for vector graphics. /// - public const string VectorGraphics = "umbracoMediaVectorGraphics"; + public const string VectorGraphicsAlias = "umbracoMediaVectorGraphics"; /// /// MediaType alias indicating allowing auto-selection. diff --git a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs index a8d089dd05..55ac3a4ec9 100644 --- a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs +++ b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs @@ -130,10 +130,10 @@ namespace Umbraco.Core.Migrations.Install _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1031, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1031", SortOrder = 2, UniqueId = new Guid("f38bd2d7-65d0-48e6-95dc-87ce06ec2d3d"), Text = Constants.Conventions.MediaTypes.Folder, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1032, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1032", SortOrder = 2, UniqueId = new Guid("cc07b313-0843-4aa8-bbda-871c8da728c8"), Text = Constants.Conventions.MediaTypes.Image, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1033, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1033", SortOrder = 2, UniqueId = new Guid("4c52d8ab-54e6-40cd-999c-7a5f24903e4d"), Text = Constants.Conventions.MediaTypes.File, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1034, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1034", SortOrder = 2, UniqueId = new Guid("f6c515bb-653c-4bdc-821c-987729ebe327"), Text = Constants.Conventions.MediaTypes.VideoName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1035, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1035", SortOrder = 2, UniqueId = new Guid("a5ddeee0-8fd8-4cee-a658-6f1fcdb00de3"), Text = Constants.Conventions.MediaTypes.AudioName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1036, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1036", SortOrder = 2, UniqueId = new Guid("a43e3414-9599-4230-a7d3-943a21b20122"), Text = Constants.Conventions.MediaTypes.ArticleName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1037, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1037", SortOrder = 2, UniqueId = new Guid("c4b1efcf-a9d5-41c4-9621-e9d273b52a9c"), Text = Constants.Conventions.MediaTypes.VectorGraphicsName, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1034, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1034", SortOrder = 2, UniqueId = new Guid("f6c515bb-653c-4bdc-821c-987729ebe327"), Text = Constants.Conventions.MediaTypes.Video, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1035, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1035", SortOrder = 2, UniqueId = new Guid("a5ddeee0-8fd8-4cee-a658-6f1fcdb00de3"), Text = Constants.Conventions.MediaTypes.Audio, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1036, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1036", SortOrder = 2, UniqueId = new Guid("a43e3414-9599-4230-a7d3-943a21b20122"), Text = Constants.Conventions.MediaTypes.Article, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1037, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1037", SortOrder = 2, UniqueId = new Guid("c4b1efcf-a9d5-41c4-9621-e9d273b52a9c"), Text = Constants.Conventions.MediaTypes.VectorGraphics, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = Constants.DataTypes.Tags, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = $"-1,{Constants.DataTypes.Tags}", SortOrder = 2, UniqueId = new Guid("b6b73142-b9c1-4bf8-a16d-e1c23320b549"), Text = "Tags", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = Constants.DataTypes.ImageCropper, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = $"-1,{Constants.DataTypes.ImageCropper}", SortOrder = 2, UniqueId = new Guid("1df9f033-e6d4-451f-b8d2-e0cbc50a836f"), Text = "Image Cropper", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1044, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1044", SortOrder = 0, UniqueId = new Guid("d59be02f-1df9-4228-aa1e-01917d806cda"), Text = Constants.Conventions.MemberTypes.DefaultAlias, NodeObjectType = Constants.ObjectTypes.MemberType, CreateDate = DateTime.Now }); @@ -174,10 +174,10 @@ namespace Umbraco.Core.Migrations.Install _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 532, NodeId = 1031, Alias = Constants.Conventions.MediaTypes.Folder, Icon = Constants.Icons.MediaFolder, Thumbnail = Constants.Icons.MediaFolder, IsContainer = false, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 533, NodeId = 1032, Alias = Constants.Conventions.MediaTypes.Image, Icon = Constants.Icons.MediaImage, Thumbnail = Constants.Icons.MediaImage, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 534, NodeId = 1033, Alias = Constants.Conventions.MediaTypes.File, Icon = Constants.Icons.MediaFile, Thumbnail = Constants.Icons.MediaFile, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); - _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 540, NodeId = 1034, Alias = Constants.Conventions.MediaTypes.Video, Icon = Constants.Icons.MediaVideo, Thumbnail = Constants.Icons.MediaVideo, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); - _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 541, NodeId = 1035, Alias = Constants.Conventions.MediaTypes.Audio, Icon = Constants.Icons.MediaAudio, Thumbnail = Constants.Icons.MediaAudio, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); - _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 542, NodeId = 1036, Alias = Constants.Conventions.MediaTypes.Article, Icon = Constants.Icons.MediaArticle, Thumbnail = Constants.Icons.MediaArticle, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); - _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 543, NodeId = 1037, Alias = Constants.Conventions.MediaTypes.VectorGraphics, Icon = Constants.Icons.MediaVectorGraphics, Thumbnail = Constants.Icons.MediaVectorGraphics, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); + _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 540, NodeId = 1034, Alias = Constants.Conventions.MediaTypes.VideoAlias, Icon = Constants.Icons.MediaVideo, Thumbnail = Constants.Icons.MediaVideo, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); + _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 541, NodeId = 1035, Alias = Constants.Conventions.MediaTypes.AudioAlias, Icon = Constants.Icons.MediaAudio, Thumbnail = Constants.Icons.MediaAudio, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); + _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 542, NodeId = 1036, Alias = Constants.Conventions.MediaTypes.ArticleAlias, Icon = Constants.Icons.MediaArticle, Thumbnail = Constants.Icons.MediaArticle, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); + _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 543, NodeId = 1037, Alias = Constants.Conventions.MediaTypes.VectorGraphicsAlias, Icon = Constants.Icons.MediaVectorGraphics, Thumbnail = Constants.Icons.MediaVectorGraphics, AllowAtRoot = true, Variations = (byte) ContentVariation.Nothing }); _database.Insert(Constants.DatabaseSchema.Tables.ContentType, "pk", false, new ContentTypeDto { PrimaryKey = 531, NodeId = 1044, Alias = Constants.Conventions.MemberTypes.DefaultAlias, Icon = Constants.Icons.Member, Thumbnail = Constants.Icons.Member, Variations = (byte) ContentVariation.Nothing }); } From e95b57ca1f992665eb951e8327125c9bbc3abd09 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 11:44:26 +0200 Subject: [PATCH 116/147] Fixed friendly name of Vector graphics --- src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs index 55ac3a4ec9..9e6884efe0 100644 --- a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs +++ b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs @@ -133,7 +133,7 @@ namespace Umbraco.Core.Migrations.Install _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1034, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1034", SortOrder = 2, UniqueId = new Guid("f6c515bb-653c-4bdc-821c-987729ebe327"), Text = Constants.Conventions.MediaTypes.Video, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1035, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1035", SortOrder = 2, UniqueId = new Guid("a5ddeee0-8fd8-4cee-a658-6f1fcdb00de3"), Text = Constants.Conventions.MediaTypes.Audio, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1036, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1036", SortOrder = 2, UniqueId = new Guid("a43e3414-9599-4230-a7d3-943a21b20122"), Text = Constants.Conventions.MediaTypes.Article, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1037, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1037", SortOrder = 2, UniqueId = new Guid("c4b1efcf-a9d5-41c4-9621-e9d273b52a9c"), Text = Constants.Conventions.MediaTypes.VectorGraphics, NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1037, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1037", SortOrder = 2, UniqueId = new Guid("c4b1efcf-a9d5-41c4-9621-e9d273b52a9c"), Text = "Vector Graphics (SVG)", NodeObjectType = Constants.ObjectTypes.MediaType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = Constants.DataTypes.Tags, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = $"-1,{Constants.DataTypes.Tags}", SortOrder = 2, UniqueId = new Guid("b6b73142-b9c1-4bf8-a16d-e1c23320b549"), Text = "Tags", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = Constants.DataTypes.ImageCropper, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = $"-1,{Constants.DataTypes.ImageCropper}", SortOrder = 2, UniqueId = new Guid("1df9f033-e6d4-451f-b8d2-e0cbc50a836f"), Text = "Image Cropper", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1044, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1044", SortOrder = 0, UniqueId = new Guid("d59be02f-1df9-4228-aa1e-01917d806cda"), Text = Constants.Conventions.MemberTypes.DefaultAlias, NodeObjectType = Constants.ObjectTypes.MemberType, CreateDate = DateTime.Now }); From 499d0ff5c113b0841e2514d255dcc8b7f47b1745 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 12:32:01 +0200 Subject: [PATCH 117/147] Fix hardcoded aliases --- src/Umbraco.Tests/Services/MediaServiceTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Tests/Services/MediaServiceTests.cs b/src/Umbraco.Tests/Services/MediaServiceTests.cs index d5cec11211..a0d74c0144 100644 --- a/src/Umbraco.Tests/Services/MediaServiceTests.cs +++ b/src/Umbraco.Tests/Services/MediaServiceTests.cs @@ -165,7 +165,7 @@ namespace Umbraco.Tests.Services var mediaService = ServiceContext.MediaService; var mediaType = MockedContentTypes.CreateNewMediaType(); ServiceContext.MediaTypeService.Save(mediaType); - var media = mediaService.CreateMedia(string.Empty, -1, "video"); + var media = mediaService.CreateMedia(string.Empty, -1, Constants.Conventions.MediaTypes.VideoAlias); // Act & Assert Assert.Throws(() => mediaService.Save(media)); @@ -177,7 +177,7 @@ namespace Umbraco.Tests.Services var mediaService = ServiceContext.MediaService; var mediaType = MockedContentTypes.CreateNewMediaType(); ServiceContext.MediaTypeService.Save(mediaType); - var media = mediaService.CreateMedia("Test", -1, "video"); + var media = mediaService.CreateMedia("Test", -1, Constants.Conventions.MediaTypes.VideoAlias); mediaService.Save(media); From 3645539fe296c424f1d8afdcb7085261e868d2d7 Mon Sep 17 00:00:00 2001 From: Warren Buckley Date: Mon, 14 Jun 2021 12:20:06 +0100 Subject: [PATCH 118/147] Run `npm update caniuse-lite` (cherry picked from commit e70c3628513ca720c4b4098f60a0a3a43df399f2) --- src/Umbraco.Web.UI.Client/package-lock.json | 205 +++++--------------- src/Umbraco.Web.UI.Client/package.json | 4 +- 2 files changed, 55 insertions(+), 154 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/package-lock.json b/src/Umbraco.Web.UI.Client/package-lock.json index 3f53638fc6..b2d811df89 100644 --- a/src/Umbraco.Web.UI.Client/package-lock.json +++ b/src/Umbraco.Web.UI.Client/package-lock.json @@ -1836,8 +1836,7 @@ "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", - "dev": true, - "optional": true + "dev": true }, "base64id": { "version": "1.0.0", @@ -2084,7 +2083,6 @@ "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz", "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==", "dev": true, - "optional": true, "requires": { "p-finally": "^1.0.0" } @@ -2126,7 +2124,6 @@ "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", "integrity": "sha1-oWCRFxcQPAdBDO9j71Gzl8Alr5w=", "dev": true, - "optional": true, "requires": { "readable-stream": "^2.3.5", "safe-buffer": "^5.1.1" @@ -2136,15 +2133,13 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true, - "optional": true + "dev": true }, "readable-stream": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=", "dev": true, - "optional": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -2160,7 +2155,6 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", "dev": true, - "optional": true, "requires": { "safe-buffer": "~5.1.0" } @@ -2301,7 +2295,6 @@ "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.4.3.tgz", "integrity": "sha512-zvj65TkFeIt3i6aj5bIvJDzjjQQGs4o/sNoezg1F1kYap9Nu2jcUdpwzRSJTHMMzG0H7bZkn4rNQpImhuxWX2A==", "dev": true, - "optional": true, "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4" @@ -2327,8 +2320,7 @@ "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", - "dev": true, - "optional": true + "dev": true }, "buffer-equal": { "version": "1.0.0", @@ -2503,9 +2495,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001168", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001168.tgz", - "integrity": "sha512-P2zmX7swIXKu+GMMR01TWa4csIKELTNnZKc+f1CjebmZJQtTAEXmpQSoKVJVVcvPGAA0TEYTOUp3VehavZSFPQ==", + "version": "1.0.30001237", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001237.tgz", + "integrity": "sha512-pDHgRndit6p1NR2GhzMbQ6CkRrp4VKuSsqbcLeOQppYPKOYkKT/6ZvZDvKJUqcmtyWIAHuZq3SVS2vc1egCZzw==", "dev": true }, "caseless": { @@ -2519,7 +2511,6 @@ "resolved": "https://registry.npmjs.org/caw/-/caw-2.0.1.tgz", "integrity": "sha512-Cg8/ZSBEa8ZVY9HspcGUYaK63d/bN7rqS3CYCzEGUxuYv6UlmcjzDUz2fCFFHyTvUW5Pk0I+3hkA3iXlIj6guA==", "dev": true, - "optional": true, "requires": { "get-proxy": "^2.0.0", "isurl": "^1.0.0-alpha5", @@ -2953,7 +2944,6 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=", "dev": true, - "optional": true, "requires": { "graceful-readlink": ">= 1.0.0" } @@ -3048,7 +3038,6 @@ "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.12.tgz", "integrity": "sha512-a1eOIcu8+7lUInge4Rpf/n4Krkf3Dd9lqhljRzII1/Zno/kRtUWnznPO3jOKBmTEktkt3fkxisUcivoj0ebzoA==", "dev": true, - "optional": true, "requires": { "ini": "^1.3.4", "proto-list": "~1.2.1" @@ -3104,7 +3093,6 @@ "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz", "integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==", "dev": true, - "optional": true, "requires": { "safe-buffer": "5.1.2" } @@ -3548,7 +3536,6 @@ "resolved": "https://registry.npmjs.org/decompress/-/decompress-4.2.0.tgz", "integrity": "sha1-eu3YVCflqS2s/lVnSnxQXpbQH50=", "dev": true, - "optional": true, "requires": { "decompress-tar": "^4.0.0", "decompress-tarbz2": "^4.0.0", @@ -3565,7 +3552,6 @@ "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", "dev": true, - "optional": true, "requires": { "pify": "^3.0.0" }, @@ -3574,8 +3560,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", - "dev": true, - "optional": true + "dev": true } } } @@ -3586,7 +3571,6 @@ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", "dev": true, - "optional": true, "requires": { "mimic-response": "^1.0.0" } @@ -3596,7 +3580,6 @@ "resolved": "https://registry.npmjs.org/decompress-tar/-/decompress-tar-4.1.1.tgz", "integrity": "sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==", "dev": true, - "optional": true, "requires": { "file-type": "^5.2.0", "is-stream": "^1.1.0", @@ -3607,8 +3590,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz", "integrity": "sha1-LdvqfHP/42No365J3DOMBYwritY=", - "dev": true, - "optional": true + "dev": true } } }, @@ -3617,7 +3599,6 @@ "resolved": "https://registry.npmjs.org/decompress-tarbz2/-/decompress-tarbz2-4.1.1.tgz", "integrity": "sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==", "dev": true, - "optional": true, "requires": { "decompress-tar": "^4.1.0", "file-type": "^6.1.0", @@ -3630,8 +3611,7 @@ "version": "6.2.0", "resolved": "https://registry.npmjs.org/file-type/-/file-type-6.2.0.tgz", "integrity": "sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==", - "dev": true, - "optional": true + "dev": true } } }, @@ -3640,7 +3620,6 @@ "resolved": "https://registry.npmjs.org/decompress-targz/-/decompress-targz-4.1.1.tgz", "integrity": "sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==", "dev": true, - "optional": true, "requires": { "decompress-tar": "^4.1.1", "file-type": "^5.2.0", @@ -3651,8 +3630,7 @@ "version": "5.2.0", "resolved": "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz", "integrity": "sha1-LdvqfHP/42No365J3DOMBYwritY=", - "dev": true, - "optional": true + "dev": true } } }, @@ -3661,7 +3639,6 @@ "resolved": "https://registry.npmjs.org/decompress-unzip/-/decompress-unzip-4.0.1.tgz", "integrity": "sha1-3qrM39FK6vhVePczroIQ+bSEj2k=", "dev": true, - "optional": true, "requires": { "file-type": "^3.8.0", "get-stream": "^2.2.0", @@ -3673,15 +3650,13 @@ "version": "3.9.0", "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=", - "dev": true, - "optional": true + "dev": true }, "get-stream": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", "integrity": "sha1-Xzj5PzRgCWZu4BUKBUFn+Rvdld4=", "dev": true, - "optional": true, "requires": { "object-assign": "^4.0.1", "pinkie-promise": "^2.0.0" @@ -3691,8 +3666,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "dev": true, - "optional": true + "dev": true } } }, @@ -3974,8 +3948,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", - "dev": true, - "optional": true + "dev": true } } }, @@ -3992,8 +3965,7 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", - "dev": true, - "optional": true + "dev": true }, "duplexify": { "version": "3.7.1", @@ -4596,7 +4568,6 @@ "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", "dev": true, - "optional": true, "requires": { "cross-spawn": "^5.0.1", "get-stream": "^3.0.0", @@ -4612,7 +4583,6 @@ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", "dev": true, - "optional": true, "requires": { "lru-cache": "^4.0.1", "shebang-command": "^1.2.0", @@ -4752,7 +4722,6 @@ "resolved": "https://registry.npmjs.org/ext-list/-/ext-list-2.2.2.tgz", "integrity": "sha512-u+SQgsubraE6zItfVA0tBuCBhfU9ogSRnsvygI7wht9TS510oLkBRXBsqopeUG/GBOIQyKZO9wjTqIu/sf5zFA==", "dev": true, - "optional": true, "requires": { "mime-db": "^1.28.0" } @@ -4762,7 +4731,6 @@ "resolved": "https://registry.npmjs.org/ext-name/-/ext-name-5.0.0.tgz", "integrity": "sha512-yblEwXAbGv1VQDmow7s38W77hzAgJAO50ztBLMcUyUBfxv1HC+LGwtiEN+Co6LtlqT/5uwVOxsD4TNIilWhwdQ==", "dev": true, - "optional": true, "requires": { "ext-list": "^2.0.0", "sort-keys-length": "^1.0.0" @@ -4997,7 +4965,6 @@ "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", "dev": true, - "optional": true, "requires": { "pend": "~1.2.0" } @@ -5036,15 +5003,13 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz", "integrity": "sha1-q/c9+rc10EVECr/qLZHzieu/oik=", - "dev": true, - "optional": true + "dev": true }, "filenamify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/filenamify/-/filenamify-2.1.0.tgz", "integrity": "sha512-ICw7NTT6RsDp2rnYKVd8Fu4cr6ITzGy3+u4vUujPkabyaz+03F24NWEX7fs5fp+kBonlaqPH8fAO2NM+SXt/JA==", "dev": true, - "optional": true, "requires": { "filename-reserved-regex": "^2.0.0", "strip-outer": "^1.0.0", @@ -5405,8 +5370,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha1-a+Dem+mYzhavivwkSXue6bfM2a0=", - "dev": true, - "optional": true + "dev": true }, "fs-mkdirp-stream": { "version": "1.0.0", @@ -5453,8 +5417,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5475,14 +5438,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5497,20 +5458,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5627,8 +5585,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5640,7 +5597,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5655,7 +5611,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5663,14 +5618,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5689,7 +5642,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5770,8 +5722,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5783,7 +5734,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5869,8 +5819,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5906,7 +5855,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5926,7 +5874,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5970,14 +5917,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -6004,7 +5949,6 @@ "resolved": "https://registry.npmjs.org/get-proxy/-/get-proxy-2.1.0.tgz", "integrity": "sha512-zmZIaQTWnNQb4R4fJUEp/FC51eZsc6EkErspy3xtIYStaq8EB/hDIWipxsal+E8rz0qD7f2sL/NA9Xee4RInJw==", "dev": true, - "optional": true, "requires": { "npm-conf": "^1.1.0" } @@ -6013,15 +5957,13 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", - "dev": true, - "optional": true + "dev": true }, "get-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", - "dev": true, - "optional": true + "dev": true }, "get-value": { "version": "2.0.6", @@ -6336,8 +6278,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=", - "dev": true, - "optional": true + "dev": true }, "growly": { "version": "1.3.0", @@ -7108,8 +7049,7 @@ "version": "1.4.2", "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz", "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw==", - "dev": true, - "optional": true + "dev": true }, "has-symbols": { "version": "1.0.0", @@ -7122,7 +7062,6 @@ "resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz", "integrity": "sha512-vdbKfmw+3LoOYVr+mtxHaX5a96+0f3DljYd8JOqvOLsf5mw2Otda2qCDT9qRqLAhrjyQ0h7ual5nOiASpsGNFw==", "dev": true, - "optional": true, "requires": { "has-symbol-support-x": "^1.4.1" } @@ -7322,8 +7261,7 @@ "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", - "dev": true, - "optional": true + "dev": true }, "ignore": { "version": "4.0.6", @@ -7453,7 +7391,6 @@ "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", "dev": true, - "optional": true, "requires": { "repeating": "^2.0.0" } @@ -7781,7 +7718,6 @@ "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz", "integrity": "sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko=", "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -7834,8 +7770,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/is-natural-number/-/is-natural-number-4.0.1.tgz", "integrity": "sha1-q5124dtM7VHjXeDHLr7PCfc0zeg=", - "dev": true, - "optional": true + "dev": true }, "is-negated-glob": { "version": "1.0.0", @@ -7873,15 +7808,13 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz", "integrity": "sha1-iVJojF7C/9awPsyF52ngKQMINHA=", - "dev": true, - "optional": true + "dev": true }, "is-plain-obj": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", - "dev": true, - "optional": true + "dev": true }, "is-plain-object": { "version": "2.0.4", @@ -7951,15 +7884,13 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", - "dev": true, - "optional": true + "dev": true }, "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", - "dev": true, - "optional": true + "dev": true }, "is-svg": { "version": "3.0.0", @@ -8056,7 +7987,6 @@ "resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz", "integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==", "dev": true, - "optional": true, "requires": { "has-to-string-tag-x": "^1.2.0", "is-object": "^1.0.1" @@ -8969,8 +8899,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", "integrity": "sha1-b54wtHCE2XGnyCD/FabFFnt0wm8=", - "dev": true, - "optional": true + "dev": true }, "lpad-align": { "version": "1.1.2", @@ -9040,8 +8969,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", - "dev": true, - "optional": true + "dev": true }, "map-visit": { "version": "1.0.0", @@ -9211,8 +9139,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "dev": true, - "optional": true + "dev": true }, "minimatch": { "version": "3.0.4", @@ -9435,9 +9362,9 @@ "dev": true }, "nouislider": { - "version": "14.6.3", - "resolved": "https://registry.npmjs.org/nouislider/-/nouislider-14.6.3.tgz", - "integrity": "sha512-/3tAqsWY2JYW9vd7bC14bFRA1P9A+pRHOtKmoMsyfnB0fQcd1UFx2pdY1Ey5wAUzTnXTesmYaEo/ecLVETijIQ==" + "version": "14.6.4", + "resolved": "https://registry.npmjs.org/nouislider/-/nouislider-14.6.4.tgz", + "integrity": "sha512-PVCGYl+aC7/nVEbW61ypJWfuW3UCpvctz/luxpt4byxxli1FFyjBX9NIiy4Yak9AaO6a5BkPGfFYMCW4eg3eeQ==" }, "now-and-later": { "version": "2.0.1", @@ -12547,7 +12474,6 @@ "resolved": "https://registry.npmjs.org/npm-conf/-/npm-conf-1.1.3.tgz", "integrity": "sha512-Yic4bZHJOt9RCFbRP3GgpqhScOY4HH3V2P8yBj6CeYq118Qr+BLXqT2JvpJ00mryLESpgOxf5XlFv4ZjXxLScw==", "dev": true, - "optional": true, "requires": { "config-chain": "^1.1.11", "pify": "^3.0.0" @@ -12557,8 +12483,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", - "dev": true, - "optional": true + "dev": true } } }, @@ -12567,7 +12492,6 @@ "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", "dev": true, - "optional": true, "requires": { "path-key": "^2.0.0" } @@ -12929,8 +12853,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", - "dev": true, - "optional": true + "dev": true }, "p-is-promise": { "version": "1.1.0", @@ -12967,7 +12890,6 @@ "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-1.2.1.tgz", "integrity": "sha1-XrOzU7f86Z8QGhA4iAuwVOu+o4Y=", "dev": true, - "optional": true, "requires": { "p-finally": "^1.0.0" } @@ -13158,8 +13080,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=", - "dev": true, - "optional": true + "dev": true }, "performance-now": { "version": "2.1.0", @@ -13667,8 +13588,7 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", "integrity": "sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk=", - "dev": true, - "optional": true + "dev": true }, "prr": { "version": "1.0.1", @@ -14024,7 +13944,6 @@ "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz", "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=", "dev": true, - "optional": true, "requires": { "is-finite": "^1.0.0" } @@ -14386,7 +14305,6 @@ "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.5.tgz", "integrity": "sha1-z+kXyz0nS8/6x5J1ivUxc+sfq9w=", "dev": true, - "optional": true, "requires": { "commander": "~2.8.1" } @@ -14789,7 +14707,6 @@ "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", "integrity": "sha1-RBttTTRnmPG05J6JIK37oOVD+a0=", "dev": true, - "optional": true, "requires": { "is-plain-obj": "^1.0.0" } @@ -14799,7 +14716,6 @@ "resolved": "https://registry.npmjs.org/sort-keys-length/-/sort-keys-length-1.0.1.tgz", "integrity": "sha1-nLb09OnkgVWmqgZx7dM2/xR5oYg=", "dev": true, - "optional": true, "requires": { "sort-keys": "^1.0.0" } @@ -15130,7 +15046,6 @@ "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-2.1.0.tgz", "integrity": "sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==", "dev": true, - "optional": true, "requires": { "is-natural-number": "^4.0.1" } @@ -15139,8 +15054,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", - "dev": true, - "optional": true + "dev": true }, "strip-indent": { "version": "1.0.1", @@ -15163,7 +15077,6 @@ "resolved": "https://registry.npmjs.org/strip-outer/-/strip-outer-1.0.1.tgz", "integrity": "sha1-sv0qv2YEudHmATBXGV34Nrip1jE=", "dev": true, - "optional": true, "requires": { "escape-string-regexp": "^1.0.2" } @@ -15289,7 +15202,6 @@ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", "integrity": "sha1-jqVdqzeXIlPZqa+Q/c1VmuQ1xVU=", "dev": true, - "optional": true, "requires": { "bl": "^1.0.0", "buffer-alloc": "^1.2.0", @@ -15304,15 +15216,13 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true, - "optional": true + "dev": true }, "readable-stream": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=", "dev": true, - "optional": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -15328,7 +15238,6 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", "dev": true, - "optional": true, "requires": { "safe-buffer": "~5.1.0" } @@ -15339,15 +15248,13 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz", "integrity": "sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0=", - "dev": true, - "optional": true + "dev": true }, "tempfile": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-2.0.0.tgz", "integrity": "sha1-awRGhWqbERTRhW/8vlCczLCXcmU=", "dev": true, - "optional": true, "requires": { "temp-dir": "^1.0.0", "uuid": "^3.0.1" @@ -15442,8 +15349,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=", - "dev": true, - "optional": true + "dev": true }, "timers-ext": { "version": "0.1.7", @@ -15500,8 +15406,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", "integrity": "sha1-STvUj2LXxD/N7TE6A9ytsuEhOoA=", - "dev": true, - "optional": true + "dev": true }, "to-fast-properties": { "version": "2.0.0", @@ -15605,7 +15510,6 @@ "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-1.0.0.tgz", "integrity": "sha1-42RqLqTokTEr9+rObPsFOAvAHCE=", "dev": true, - "optional": true, "requires": { "escape-string-regexp": "^1.0.2" } @@ -15742,7 +15646,6 @@ "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", "integrity": "sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==", "dev": true, - "optional": true, "requires": { "buffer": "^5.2.1", "through": "^2.3.8" @@ -15943,8 +15846,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz", "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k=", - "dev": true, - "optional": true + "dev": true }, "use": { "version": "3.1.1", @@ -16440,7 +16342,6 @@ "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", "dev": true, - "optional": true, "requires": { "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" diff --git a/src/Umbraco.Web.UI.Client/package.json b/src/Umbraco.Web.UI.Client/package.json index 045f788929..45411fad26 100644 --- a/src/Umbraco.Web.UI.Client/package.json +++ b/src/Umbraco.Web.UI.Client/package.json @@ -55,7 +55,7 @@ "@babel/preset-env": "7.6.3", "@babel/plugin-proposal-object-rest-spread": "7.13.8", "autoprefixer": "9.6.5", - "caniuse-lite": "^1.0.30001037", + "caniuse-lite": "^1.0.30001237", "cssnano": "4.1.10", "fs": "0.0.2", "gulp": "4.0.2", @@ -78,8 +78,8 @@ "jasmine-core": "3.5.0", "jsdom": "16.4.0", "karma": "4.4.1", - "karma-jsdom-launcher": "^8.0.2", "karma-jasmine": "2.0.1", + "karma-jsdom-launcher": "^8.0.2", "karma-junit-reporter": "2.0.1", "karma-spec-reporter": "0.0.32", "less": "3.10.3", From 0bc2ff17d77c76a985788979c3017d5d57369812 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 16:14:01 +0200 Subject: [PATCH 119/147] =?UTF-8?q?Yeah=20okay,=20#h5is=20I=20broke=20the?= =?UTF-8?q?=20build=20=F0=9F=A4=A6=E2=80=8D=E2=99=82=EF=B8=8F=20-=20fixed?= =?UTF-8?q?=20here=20for=20this=20PR,=20will=20also=20fix=20in=20the=20con?= =?UTF-8?q?trib=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index d3e8a355c9..180837bc80 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -319,6 +319,7 @@ $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v12.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v14.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v15.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v16.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v17.0 From 3265fbb4734bd65059aa7f49cfa64b29db04513c Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 18 Jun 2021 16:15:32 +0200 Subject: [PATCH 120/147] Fix the build --- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index d3e8a355c9..180837bc80 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -319,6 +319,7 @@ $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v12.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v14.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v15.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v16.0 $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v17.0 From 32a1870d80fceff655feae9868107ddde91e407b Mon Sep 17 00:00:00 2001 From: Christian Tricarico Date: Sun, 20 Jun 2021 02:32:10 +0200 Subject: [PATCH 121/147] Backoffice Italian localization improvements (#10371) * Italian translations in localization xml. * Italian translations in localization xml. * restore previous package-lock version --- src/Umbraco.Web.UI/Umbraco/config/lang/it.xml | 1674 +++++++++-------- 1 file changed, 886 insertions(+), 788 deletions(-) diff --git a/src/Umbraco.Web.UI/Umbraco/config/lang/it.xml b/src/Umbraco.Web.UI/Umbraco/config/lang/it.xml index 2f89060ac6..63b7a2fff8 100644 --- a/src/Umbraco.Web.UI/Umbraco/config/lang/it.xml +++ b/src/Umbraco.Web.UI/Umbraco/config/lang/it.xml @@ -1,401 +1,437 @@ - - The Umbraco community - https://our.umbraco.com/documentation/Extending-Umbraco/Language-Files - - - Gestisci hostnames - Audit Trail - Sfoglia - Copia - Crea - Crea pacchetto - Cancella - Disabilita - Svuota il cestino - Esporta il tipo di documento - Importa il tipo di documento - Importa il pacchetto - Modifica in Area di Lavoro - Uscita - Sposta - Notifiche - Accesso pubblico - Pubblica - Aggiorna nodi - Ripubblica intero sito - Permessi - Annulla ultima modifica - Invia per la pubblicazione - Invia per la traduzione - Ordina - Invia la pubblicazione - Traduci - Annulla pubblicazione - Aggiorna - - - Aggiungi nuovo dominio - Dominio - - - - - Hostname non valido - Modifica il dominio corrente - - - Visualizzazione per - - - Grassetto - Cancella rientro paragrafo - Inserisci dal file - Inserisci intestazione grafica - Modifica Html - Inserisci rientro paragrafo - Corsivo - Centra - Allinea testo a sinistra - Allinea testo a destra - Inserisci Link - Inserisci local link (ancora) - Elenco puntato - Elenco numerato - Inserisci macro - Inserisci immagine - Modifica relazioni - Salva - Salva e pubblica - Salva e invia per approvazione - Anteprima - - Scegli lo stile - Mostra gli stili - Inserisci tabella - - - Informazioni su questa pagina - Link alternativo - - Links alternativi - Clicca per modificare questo elemento - Creato da - Creato il - Tipo di documento - Modifica - Attivo fino al - - - Ultima pubblicazione - Link ai media - Tipo di media - Gruppo di membri - Ruolo - Tipologia Membro - - Titolo della Pagina - - - Pubblicato - Stato della pubblicazione - Pubblicato il - Rimuovi data - Ordinamento dei nodi aggiornato - - Statistiche - Titolo (opzionale) - Tipo - Non pubblicare - Ultima modifica - Rimuovi il file - Link al documento - - - - Crea al - Scegli il tipo ed il titolo - - - - - - hai aperto una nuova finestra - Riavvia - Visita - Benvenuto - - - Rimani - Scarta le modifiche - Hai delle modifiche non salvate - Sei sicuro di voler lasciare questa pagina? - hai delle modifiche non salvate - - - Fatto - Elimianto %0% elemento - Elimianto %0% elementi - Eliminato %0% su %1% elemento - Eliminato %0% su %1% elementi - Pubblicato %0% elemento - Pubblicato %0% elementi - Pubblicato %0% su %1% elemento - Pubblicato %0% su %1% elementi - %0% elemento non pubblicato - %0% elementi non pubblicati - Elementi non pubblicati - %0% su %1% - Elementi non pubblicati - %0% su %1% - Spostato %0% elemento - Spsotato %0% elementi - Spostato %0% su %1% elemento - Spostato %0% su %1% elementi - Copiato %0% elemento - Copiato %0% elementi - Copiato %0% su %1% elemento - Copiato %0% su %1% elementi - - - Titolo del Link - Link - Nome - Gestione alias Hostnames - Chiudi questa finestra - - - - - Taglia - Modifica elemento Dictionary - Modifica il linguaggio - Inserisci il link locale - Inserisci carattere - - - Inserisci link - Inserisci macro - Inserisci tabella - Ultima modifica - Link - - - - - Incolla - Modifica il Permesso per - - - - regexlib.com ha attualmente qualche problema, di cui non abbiamo il controllo. Siamo spiacevoli dell'inconveniente.]]> - - Elimina Macro - Campo obbligatorio - - - - Numero di colonne - Numero di righe - - Seleziona elemento - Visualizza gli elementi in cache - - - - - - - - - - - - Rendering controllo - Bottoni - Abilita impostazioni avanzate per - Abilita menu contestuale - Dimensione massima delle immagini inserite - Fogli di stile collegati - Visualizza etichetta - Larghezza e altezza - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Info - Azione - Aggiungi - Alias - - Bordo - o - Annulla - - Scegli - Chiudi - Chiudi la finestra - Commento - Conferma - Blocca le proporzioni - Continua - Copia - Crea - Base di dati - Data - Default - Elimina - Eliminato - Elimina... - Design - Dimensioni - - Scarica - Modifica - Modificato - Elementi - Email - Errore - Trova - Cartella - Altezza - Guida - Icona - Importa - - Inserisci - Installa - Giustificato - Lingua - Layout - Caricamento - Bloccato - Login - Log off - Logout - Macro - Sposta - Nome - Nuovo - Successivo - No - di - Ok - Apri - o - Password - Percorso - - Precedente - - - Cestino - Rimangono - Rinomina - Rinnova - Riprova - Permessi - Cerca - Server - Mostra - Mostra la pagina inviata - Dimensione - Ordina - Invia - Tipo - - Su - Aggiorna - Aggiornamento - Carica - URL - Utente - - Valore - Vedi - Benvenuto... - Larghezza - Si - Riordina - Ho finito di ordinare - - - Colore di sfondo - Grassetto - Colore del testo - Carattere - Testo - - - Pagina - - - - - - - installa per installare il database Umbraco %0% ]]> - Avanti per proseguire.]]> - Database non trovato! Perfavore, controlla che le informazioni della stringa di connessione nel file "web.config" siano corrette.

    -

    Per procedere, edita il file "web.config" (utilizzando Visual Studio o l'editor di testo che preferisci), scorri in basso, aggiungi la stringa di connessione per il database chiamato "umbracoDbDSN" e salva il file.

    Clicca il tasto riprova quando hai finito.
    Maggiori dettagli per la modifica del file web.config qui.

    ]]>     - - Premi il tasto aggiorna per aggiornare il database ad Umbraco %0%

    Non preoccuparti, il contenuto non verrà perso e tutto continuerà a funzionare dopo l'aggiornamento!

    ]]>     - Premi il tasto Avanti per continuare.]]> - Avanti per continuare la configurazione.]]> - La password predefinita per l'utente di default deve essere cambiata!]]> - L'utente di default è stato disabilitato o non ha accesso ad Umbraco!

    Non è necessario eseguire altre operazioni. Clicca il tasto Avanti per continuare.]]> - La password è stata modificata con successo

    Non è necessario eseguire altre operazioni. Clicca il tasto Avanti per continuare.]]> - - - - - - - - Le impostazioni dei permessi sono perfette!

    Puoi eseguire Umbraco senza problemi, ma potresti non poter installare i pacchetti che sono consigliati per sfruttare tutti i vantaggi offerti da Umbraco.]]>     - - - video tutorial su come impostare i permessi delle cartelle per Umbraco o leggi la versione testuale.]]> - Le impostazioni dei permessi potrebbero avere dei problemi!

    Puoi eseguire Umbraco, ma potresti non essere in grado di creare cartelle o installare pacchetti che sono raccomandati per sfruttare tutti i vantaggi di Umbraco.]]>     - Le impostazioni dei permessi non sono corrette per Umbraco!

    Per eseguire Umbraco, devi aggiornare le impostazioni dei permessi.]]>     - La configurazione dei permessi è perfetta!

    Sei pronto per avviare Umbraco e installare i pacchetti!]]>     - - - - - - Guarda come) Puoi anche installare eventuali Runway in un secondo momento. Vai nella sezione Developer e scegli Pacchetti.]]> - - Runway è installato - + The Umbraco community + https://our.umbraco.com/documentation/Extending-Umbraco/Language-Files + + + Gestisci hostnames + Audit Trail + Sfoglia + Copia + Crea + Crea pacchetto + Cancella + Disabilita + Svuota il cestino + Esporta il tipo di documento + Importa il tipo di documento + Importa il pacchetto + Modifica in Area di Lavoro + Uscita + Sposta + Notifiche + Accesso pubblico + Pubblica + Aggiorna nodi + Ripubblica intero sito + Permessi + Annulla ultima modifica + Invia per la pubblicazione + Invia per la traduzione + Ordina + Invia la pubblicazione + Traduci + Annulla pubblicazione + Aggiorna + Rimuovi + Ripristina + Crea Content Template + Crea gruppo + + + Aggiungi nuovo dominio + Dominio + + + + + Hostname non valido + Modifica il dominio corrente + + + Visualizzazione per + Contenuto pubblicato + Contenuto salvato + + + Grassetto + Cancella rientro paragrafo + Inserisci dal file + Inserisci intestazione grafica + Modifica Html + Inserisci rientro paragrafo + Corsivo + Centra + Allinea testo a sinistra + Allinea testo a destra + Inserisci Link + Inserisci local link (ancora) + Elenco puntato + Elenco numerato + Inserisci macro + Inserisci immagine + Modifica relazioni + Salva + Salva e pubblica + Salva e invia per approvazione + Anteprima + + Scegli lo stile + Mostra gli stili + Inserisci tabella + Altre azioni + Pubblica con i discendenti + Pianifica + Seleziona + Annulla selezione + + + Informazioni su questa pagina + Link alternativo + + Links alternativi + Clicca per modificare questo elemento + Creato da + Creato il + Tipo di documento + Modifica + Attivo fino al + + + Ultima pubblicazione + Link ai media + Tipo di media + Gruppo di membri + Ruolo + Tipologia Membro + + Titolo della Pagina + + + Pubblicato + Stato della pubblicazione + Pubblicato il + Rimuovi data + Ordinamento dei nodi aggiornato + + Statistiche + Titolo (opzionale) + Tipo + Non pubblicare + Ultima modifica + Rimuovi il file + Link al documento + Elementi + Pubblicato + Seleziona da data e l'ora in cui pubblicare/depubblicare il contenuto. + Imposta data + Depubblicato il + + + + Crea un elemento sotto + Scegli il tipo ed il titolo + Cartella + + + + + + hai aperto una nuova finestra + Riavvia + Visita + Benvenuto + + + Rimani + Scarta le modifiche + Hai delle modifiche non salvate + Sei sicuro di voler lasciare questa pagina? - hai delle modifiche non salvate + + + Fatto + Elimianto %0% elemento + Elimianto %0% elementi + Eliminato %0% su %1% elemento + Eliminato %0% su %1% elementi + Pubblicato %0% elemento + Pubblicato %0% elementi + Pubblicato %0% su %1% elemento + Pubblicato %0% su %1% elementi + %0% elemento non pubblicato + %0% elementi non pubblicati + Elementi non pubblicati - %0% su %1% + Elementi non pubblicati - %0% su %1% + Spostato %0% elemento + Spsotato %0% elementi + Spostato %0% su %1% elemento + Spostato %0% su %1% elementi + Copiato %0% elemento + Copiato %0% elementi + Copiato %0% su %1% elemento + Copiato %0% su %1% elementi + + + Titolo del Link + Link + Nome + Gestione alias Hostnames + Chiudi questa finestra + + + + + Taglia + Modifica elemento Dictionary + Modifica il linguaggio + Inserisci il link locale + Inserisci carattere + + + Inserisci link + Inserisci macro + Inserisci tabella + Ultima modifica + Link + + + + + Incolla + Modifica il Permesso per + + + + regexlib.com ha attualmente qualche problema, di cui non abbiamo il controllo. Siamo spiacevoli dell'inconveniente.]]> + + Elimina Macro + Campo obbligatorio + + + + Numero di colonne + Numero di righe + + Seleziona elemento + Visualizza gli elementi in cache + Seleziona contenuto + + + + + + + + + + + + Rendering controllo + Bottoni + Abilita impostazioni avanzate per + Abilita menu contestuale + Dimensione massima delle immagini inserite + Fogli di stile collegati + Visualizza etichetta + Larghezza e altezza + + + + + + + + + + + + + + + + + + + + + + + + + + + Questa proprietà non è valida + + + Info + Azione + Aggiungi + Alias + + Bordo + o + Annulla + + Scegli + Chiudi + Chiudi la finestra + Commento + Conferma + Blocca le proporzioni + Continua + Copia + Crea + Base di dati + Data + Default + Elimina + Eliminato + Elimina... + Design + Dimensioni + + Scarica + Modifica + Modificato + Elementi + Email + Errore + Trova + Cartella + Altezza + Guida + Icona + Importa + + Inserisci + Installa + Giustificato + Lingua + Layout + Caricamento + Bloccato + Login + Log off + Logout + Macro + Sposta + Nome + Nuovo + Successivo + No + di + Ok + Apri + o + Password + Percorso + + Precedente + + + Cestino + Rimangono + Rinomina + Rinnova + Riprova + Permessi + Cerca + Server + Mostra + Mostra la pagina inviata + Dimensione + Ordina + Conferma + Tipo + Digita per cercare... + Su + Aggiorna + Aggiornamento + Carica + URL + Utente + + Valore + Vedi + Benvenuto... + Larghezza + Si + Riordina + Ho finito di ordinare + Richiesto + Contenuti + Azioni + Cerca solo in questa cartella + Pianifica pubblicazione + selezionato + Annulla + Cambia password + Cronologia + Generale + Rimuovi + Gruppi + + + Colore di sfondo + Grassetto + Colore del testo + Carattere + Testo + + + Pagina + + + + + + + installa per installare il database Umbraco %0% ]]> + Avanti per proseguire.]]> + + Database non trovato! Perfavore, controlla che le informazioni della stringa di connessione nel file "web.config" siano corrette.

    +

    Per procedere, edita il file "web.config" (utilizzando Visual Studio o l'editor di testo che preferisci), scorri in basso, aggiungi la stringa di connessione per il database chiamato "umbracoDbDSN" e salva il file.

    Clicca il tasto riprova quando hai finito.
    Maggiori dettagli per la modifica del file web.config qui.

    ]]> +     + + Premi il tasto aggiorna per aggiornare il database ad Umbraco %0%

    Non preoccuparti, il contenuto non verrà perso e tutto continuerà a funzionare dopo l'aggiornamento!

    ]]>     + Premi il tasto Avanti per continuare.]]> + Avanti per continuare la configurazione.]]> + La password predefinita per l'utente di default deve essere cambiata!]]> + L'utente di default è stato disabilitato o non ha accesso ad Umbraco!

    Non è necessario eseguire altre operazioni. Clicca il tasto Avanti per continuare.]]> + La password è stata modificata con successo

    Non è necessario eseguire altre operazioni. Clicca il tasto Avanti per continuare.]]> + + + + + + + + Le impostazioni dei permessi sono perfette!

    Puoi eseguire Umbraco senza problemi, ma potresti non poter installare i pacchetti che sono consigliati per sfruttare tutti i vantaggi offerti da Umbraco.]]>     + + + video tutorial su come impostare i permessi delle cartelle per Umbraco o leggi la versione testuale.]]> + Le impostazioni dei permessi potrebbero avere dei problemi!

    Puoi eseguire Umbraco, ma potresti non essere in grado di creare cartelle o installare pacchetti che sono raccomandati per sfruttare tutti i vantaggi di Umbraco.]]>     + Le impostazioni dei permessi non sono corrette per Umbraco!

    Per eseguire Umbraco, devi aggiornare le impostazioni dei permessi.]]>     + La configurazione dei permessi è perfetta!

    Sei pronto per avviare Umbraco e installare i pacchetti!]]>     + + + + + + Guarda come) Puoi anche installare eventuali Runway in un secondo momento. Vai nella sezione Developer e scegli Pacchetti.]]> + + Runway è installato + + Questa è la lista dei nostri moduli raccomandati, seleziona quali vorresti installare, o vedi l'intera lista di moduli - ]]> - Raccommandato solo per utenti esperti - Vorrei iniziare da un sito semplice - + + Raccommandato solo per utenti esperti + Vorrei iniziare da un sito semplice + + "Runway" è un semplice sito web contenente alcuni tipi di documento e alcuni templates di base. L'installer configurerà Runway per te automaticamente, ma tu potrai facilmente modificarlo, estenderlo o eliminarlo. Non è necessario installarlo e potrai usare Umbraco anche senza di esso, ma @@ -406,63 +442,85 @@ Inclusi in Runway: Home page, pagina Guida introduttiva, pagina Installazione moduli
    Moduli opzionali: Top Navigation, Sitemap, Contatti, Gallery.     - ]]> - Cosa è Runway - Passo 1/5 Accettazione licenza - Passo 2/5: Configurazione database - Passo 3/5: Controllo permessi dei file - Passo 4/5: Controllo impostazioni sicurezza - Passo 5/5: Umbraco è pronto per iniziare - Grazie per aver scelto Umbraco - Naviga per il tuo nuovo sito -Hai installato Runway, quindi perché non dare uno sguardo al vostro nuovo sito web.]]> - Ulteriori informazioni e assistenza -Fatti aiutare dalla nostra community, consulta la documentazione o guarda alcuni video gratuiti su come costruire un semplice sito web, come usare i pacchetti e una guida rapida alla terminologia Umbraco]]> - - /web.config e aggiornare la chiave AppSetting UmbracoConfigurationStatus impostando il valore '%0%'.]]> - iniziare immediatamente cliccando sul bottone "Avvia Umbraco".
    Se sei nuovo a Umbraco, si possono trovare un sacco di risorse sulle nostre pagine Getting Started.]]>     - Avvia Umbraco -Per gestire il tuo sito web, è sufficiente aprire il backoffice di Umbraco e iniziare ad aggiungere i contenuti, aggiornando i modelli e i fogli di stile o aggiungere nuove funzionalità]]> - Connessione al database non riuscita. - Umbraco Versione 3 - Umbraco Versione 4 - Guarda - Umbraco %0% per una nuova installazione o per l'aggiornamento dalla versione 3.0. + ]]> + + Cosa è Runway + Passo 1/5 Accettazione licenza + Passo 2/5: Configurazione database + Passo 3/5: Controllo permessi dei file + Passo 4/5: Controllo impostazioni sicurezza + Passo 5/5: Umbraco è pronto per iniziare + Grazie per aver scelto Umbraco + + Naviga per il tuo nuovo sito +Hai installato Runway, quindi perché non dare uno sguardo al vostro nuovo sito web.]]> + + + Ulteriori informazioni e assistenza +Fatti aiutare dalla nostra community, consulta la documentazione o guarda alcuni video gratuiti su come costruire un semplice sito web, come usare i pacchetti e una guida rapida alla terminologia Umbraco]]> + + + /web.config e aggiornare la chiave AppSetting UmbracoConfigurationStatus impostando il valore '%0%'.]]> + iniziare immediatamente cliccando sul bottone "Avvia Umbraco".
    Se sei nuovo a Umbraco, si possono trovare un sacco di risorse sulle nostre pagine Getting Started.]]>     + + Avvia Umbraco +Per gestire il tuo sito web, è sufficiente aprire il backoffice di Umbraco e iniziare ad aggiungere i contenuti, aggiornando i modelli e i fogli di stile o aggiungere nuove funzionalità]]> + + Connessione al database non riuscita. + Umbraco Versione 3 + Umbraco Versione 4 + Guarda + + Umbraco %0% per una nuova installazione o per l'aggiornamento dalla versione 3.0.

    - Clicca "avanti" per avviare la procedura.]]>     - - - Codice cultura - Nome cultura - - - - Riconnetti adesso per salvare il tuo lavoro - - - © 2001 - %0%
    umbraco.com

    ]]> - - - Dashboard - Sezioni - Contenuto - - - Scegli la pagina sopra... - - Seleziona dove il documento %0% deve essere copiato - - Seleziona dove il documento %0% deve essere spostato - - - - - - - - - - "avanti" per avviare la procedura.]]> + + + + Codice cultura + Nome cultura + + + + Riconnetti adesso per salvare il tuo lavoro + + + © 2001 - %0%
    umbraco.com

    ]]>     + Buona domenica + Buon lunedì + Buon martedì + Buon mercoledì + Buon giovedì + Buon venerdì + Buon sabato + Mostra password + Nascondi password + Password dimenticata? + Una email verrà inviata all'indirizzo specificato con un link per il reset della password + Ritorna alla finestra di login + + + Dashboard + Sezioni + Contenuto + + + Scegli la pagina sopra... + + Seleziona dove il documento %0% deve essere copiato + + Seleziona dove il documento %0% deve essere spostato + + + + + + + + + + + - Salve %0%

    + ]]> +     + + Salve %0%

    Questa è un'email automatica per informare che l'azione '%1%' è stata eseguita sulla pagina '%2%' @@ -501,257 +561,266 @@ Per gestire il tuo sito web, è sufficiente aprire il backoffice di Umbraco e in

    Buona giornata!

    Grazie da Umbraco -

    ]]>     - [%0%] Notifica per %1% eseguita su %2% - Notifiche - - - ]]> + + [%0%] Notifica per %1% eseguita su %2% + Notifiche + + + + e selezionando il pacchetto. I pacchetti Umbraco generalmente hanno l'estensione ".umb" o ".zip". - ]]> - Autore - Documentazione - Meta dati pacchetto - Nome del pacchetto - Il pacchetto non contiene tutti gli elementi -
    - E' possibile rimuovere questo pacchetto dal sistema cliccando "rimuovi pacchetto" in basso.]]>     - Opzioni pacchetto - Pacchetto leggimi - Pacchetto repository - Conferma eliminazione - - - Disinstalla pacchetto - + ]]> + + Autore + Documentazione + Meta dati pacchetto + Nome del pacchetto + Il pacchetto non contiene tutti gli elementi + +
    + E' possibile rimuovere questo pacchetto dal sistema cliccando "rimuovi pacchetto" in basso.]]> +     + Opzioni pacchetto + Pacchetto leggimi + Pacchetto repository + Conferma eliminazione + + + Disinstalla pacchetto + + Avviso: tutti i documenti, i media, etc a seconda degli elementi che rimuoverai, smetteranno di funzionare, e potrebbero portare a un'instabilità del sistema, - perciò disinstalla con cautela. In caso di dubbio contattare l'autore del pacchetto.]]> - Versione del pacchetto - - - - - - - - - - usando i gruppi di membri di Umbraco.]]> - Devi creare un gruppo di membri prima di utilizzare l'autenticazione basata sui ruoli - - - - - - - - - - - - - - - - - - - - - - - - ok per pubblicare %0% e rendere questo contenuto accessibile al pubblico.

    Puoi pubblicare questa pagina e tutte le sue sottopagine selezionando pubblica tutti i figli qui sotto.]]>     - - - - - - - - - - - - - - - - Il testo in rosso non verrà mostrato nella versione selezionata, quello in verde verrà aggiunto]]> - - - - - - - - - - - Concierge - Contenuto - Courier - Sviluppo - Configurazione guidata Umbraco - Media - Membri - Newsletters - Impostazioni - Statistiche - Traduzione - Utenti - - - Tipo di contenuto master abilitato - Questo tipo di contenuto usa - - - - - Tipo - Foglio di stile - Tab - Titolo tab - Tabs - - - Ordinamento - Data creazione - - - - - - - - - Tipo di dati: %1%]]> - - Tipo di documento salvato - Tab creata - Tab eliminata - Tab con id: %0% eliminata - Contenuto non pubblicare - - - - Tipo di dato salvato - - - - - - - - - - - - - - - Tipo utente salvato - - - - - - Partial view salvata - Partial view salvata senza errori! - Partial view non salvata - Errore durante il salvataggio del file. - - - - - - - - - - - Anteprima - Stili - - - - - - - - - Master template - - Template - - - Image - Macro - Seleziona il tipo di contenuto - Seleziona un layout - Aggiungi una riga - Aggiungi contenuto - Elimina contenuto - Impostazioni applicati - Questo contenuto non è consentito qui - Questo contenuto è consentito qui - Clicca per incorporare - Clicca per inserire l'immagine - Didascalia dell'immagine... - Scrivi qui... - I Grid Layout - I layout sono l'area globale di lavoro per il grid editor, di solito ti serve solo uno o due layout differenti - Aggiungi un Grid Layout - Sistema il layout impostando la larghezza della colonna ed aggiungendo ulteriori sezioni - Configurazioni della riga - Le righe sono le colonne predefinite disposte orizzontalmente - Aggiungi configurazione della riga - Sistema la riga impostando la larghezza della colonna ed aggiungendo ulteriori colonne - Colonne - Totale combinazioni delle colonne nel grid layout - Impostazioni - Configura le impostazioni che possono essere cambiate dai editori - Stili - Configura i stili che possono essere cambiati dai editori - Permetti tutti i editor - Permetti tutte le configurazioni della riga - - - - - - Scegli il campo - Converte le interruzioni di linea - - Campi Personalizzati - - - - - - - Minuscolo - Nessuno - - - Ricorsivo - - - Campi Standard - Maiuscolo - - - - - - - - Dettagli - Scarica xml DTD - Campi - Includi le sottopagine - + + Versione del pacchetto + + + + + + + + + + usando i gruppi di membri di Umbraco.]]> + Devi creare un gruppo di membri prima di utilizzare l'autenticazione basata sui ruoli + + + + + + + + + + + + + + + + + + + + + + + + ok per pubblicare %0% e rendere questo contenuto accessibile al pubblico.

    Puoi pubblicare questa pagina e tutte le sue sottopagine selezionando pubblica tutti i figli qui sotto.]]>     + + + + + + + + + + + + + + + + Il testo in rosso non verrà mostrato nella versione selezionata, quello in verde verrà aggiunto]]> + + + + + + + + + + + Concierge + Contenuto + Courier + Sviluppo + Configurazione guidata Umbraco + Media + Membri + Newsletters + Impostazioni + Statistiche + Traduzione + Utenti + + + Tipo di contenuto master abilitato + Questo tipo di contenuto usa + + + + + Tipo + Foglio di stile + Tab + Titolo tab + Tabs + + + Ordinamento + Data creazione + + + + + + + + + Tipo di dati: %1%]]> + + Tipo di documento salvato + Tab creata + Tab eliminata + Tab con id: %0% eliminata + Contenuto non pubblicare + + + + Tipo di dato salvato + + + + + + + + + + + + + + + Tipo utente salvato + + + + + + Partial view salvata + Partial view salvata senza errori! + Partial view non salvata + Errore durante il salvataggio del file. + + + + + + + + + + + Anteprima + Stili + + + + + + + + + Master template + + Template + Data creazione + + + Immagine + Macro + Seleziona il tipo di contenuto + Seleziona un layout + Aggiungi una riga + Aggiungi contenuto + Elimina contenuto + Impostazioni applicati + Questo contenuto non è consentito qui + Questo contenuto è consentito qui + Clicca per incorporare + Clicca per inserire l'immagine + Didascalia dell'immagine... + Scrivi qui... + I Grid Layout + I layout sono l'area globale di lavoro per il grid editor, di solito ti serve solo uno o due layout differenti + Aggiungi un Grid Layout + Sistema il layout impostando la larghezza della colonna ed aggiungendo ulteriori sezioni + Configurazioni della riga + Le righe sono le colonne predefinite disposte orizzontalmente + Aggiungi configurazione della riga + Sistema la riga impostando la larghezza della colonna ed aggiungendo ulteriori colonne + Colonne + Totale combinazioni delle colonne nel grid layout + Impostazioni + Configura le impostazioni che possono essere cambiate dai editori + Stili + Configura i stili che possono essere cambiati dai editori + Permetti tutti i editor + Permetti tutte le configurazioni della riga + + + + + + Scegli il campo + Converte le interruzioni di linea + + Campi Personalizzati + + + + + + + Minuscolo + Nessuno + + + Ricorsivo + + + Campi Standard + Maiuscolo + + + + + + + + Dettagli + Scarica xml DTD + Campi + Includi le sottopagine + + - - - - - - - - - - Traduttore - - - - Cache Browser - Cestino - Pacchetti creati - Tipi di dato - Dizionario - Pacchetti installati - Installare skin - Installare starter kit - Lingue - Installa un pacchetto locale - Macros - Tipi di media - Membri - Gruppi di Membri - Ruoli - Tipologia Membri - Tipi di documento - Pacchetti - Pacchetti - Installa dal repository - Installa Runway - Moduli Runway - Files di scripting - Scripts - Fogli di stile - Templates - Permessi Utente - Tipi di Utente - Utenti - - - - - - - - - Amministratore - Campo Categoria - Cambia la tua password - - Conferma la nuova password - Contenuto del canale - Campo Descrizione - Disabilita l'utente - Tipo di Documento - Editor - Campo Eccezione - Lingua - Login - - Sezioni - Modifica la tua password - - Password - - - Password attuale - - - - - - - - - - - Username - - - - Autore - + ]]> + + + + + + + + + + + Traduttore + + + + Cache Browser + Cestino + Pacchetti creati + Tipi di dato + Dizionario + Pacchetti installati + Installare skin + Installare starter kit + Lingue + Installa un pacchetto locale + Macros + Tipi di media + Membri + Gruppi di Membri + Ruoli + Tipologia Membri + Tipi di documento + Pacchetti + Pacchetti + Installa dal repository + Installa Runway + Moduli Runway + Files di scripting + Scripts + Fogli di stile + Templates + Permessi Utente + Tipi di Utente + Utenti + Contenuti + + + + + + + + + Amministratore + Campo Categoria + Cambia la tua password + + Conferma la nuova password + Contenuto del canale + Campo Descrizione + Disabilita l'utente + Tipo di Documento + Editor + Campo Eccezione + Lingua + Login + + Sezioni + Modifica la tua password + + Password + + + Password attuale + + + + + + + + + + + Username + + + + Autore + Il tuo profilo + La tua storia recente + Crea utente + Crea nuovi utenti e dai loro accesso ad Umbraco. Quando un nuovo utente viene creato viene generata una password che potrai condividere con l'utente. + Aggiungi gruppi per assegnare accessi e permessi + Torna agli utenti + Gestione utenti + + + Devi aggiungere almeno + elementi + + + Digita per cercare... + Inserisci la tua email + Inserisci la tua password + Inserisci un nome... + Inserisci una email... + + + o clicca qui per scegliere i file + Trascina i tuoi file all'interno di quest'area + + + Contenuti + Info + Elementi + From 809bb3b5d3462ebb43dd8c8929de497442b22b73 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Mon, 21 Jun 2021 14:01:50 +0200 Subject: [PATCH 122/147] Fixes #10423 Don't obsolete the v2 pickers, renamed them and they're able to be created again, without config changes --- src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs | 4 ++-- .../views/prevalueeditors/obsoletemediapickernotice.html | 2 +- .../PropertyEditors/MediaPickerPropertyEditor.cs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs index 6a56141491..d78e0a90d7 100644 --- a/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs +++ b/src/Umbraco.Core/Migrations/Install/DatabaseDataCreator.cs @@ -141,8 +141,8 @@ namespace Umbraco.Core.Migrations.Install //New UDI pickers with newer Ids _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1046, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1046", SortOrder = 2, UniqueId = new Guid("FD1E0DA5-5606-4862-B679-5D0CF3A52A59"), Text = "Content Picker", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1047, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1047", SortOrder = 2, UniqueId = new Guid("1EA2E01F-EBD8-4CE1-8D71-6B1149E63548"), Text = "Member Picker", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1048, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1048", SortOrder = 2, UniqueId = new Guid("135D60E0-64D9-49ED-AB08-893C9BA44AE5"), Text = "(Obsolete) Media Picker", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); - _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1049, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1049", SortOrder = 2, UniqueId = new Guid("9DBBCBBB-2327-434A-B355-AF1B84E5010A"), Text = "(Obsolete) Multiple Media Picker", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1048, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1048", SortOrder = 2, UniqueId = new Guid("135D60E0-64D9-49ED-AB08-893C9BA44AE5"), Text = "Media Picker (legacy)", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); + _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1049, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1049", SortOrder = 2, UniqueId = new Guid("9DBBCBBB-2327-434A-B355-AF1B84E5010A"), Text = "Multiple Media Picker (legacy)", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1050, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1050", SortOrder = 2, UniqueId = new Guid("B4E3535A-1753-47E2-8568-602CF8CFEE6F"), Text = "Multi URL Picker", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); _database.Insert(Constants.DatabaseSchema.Tables.Node, "id", false, new NodeDto { NodeId = 1051, Trashed = false, ParentId = -1, UserId = -1, Level = 1, Path = "-1,1051", SortOrder = 2, UniqueId = Constants.DataTypes.Guids.MediaPicker3Guid, Text = "Media Picker", NodeObjectType = Constants.ObjectTypes.DataType, CreateDate = DateTime.Now }); diff --git a/src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html b/src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html index cc861dcb4b..617ffd80d6 100644 --- a/src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html +++ b/src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html @@ -1 +1 @@ -

    Important: switching from the (Obsolete) Media Picker to Media Picker will mean all data (references to previously selected media items) will be deleted and no longer available.

    \ No newline at end of file +

    Important: switching from "Media Picker (legacy)" to "Media Picker" is not supported and doing so will mean all data (references to previously selected media items) will no longer be available on existing content items.

    diff --git a/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs index 499737f3b7..1784b5fd0d 100644 --- a/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs @@ -8,17 +8,17 @@ namespace Umbraco.Web.PropertyEditors { /// /// Represents a media picker property editor. - /// Marked as Deprecated as best to use the NEW Media Picker aka MediaPicker3 + /// Nameed "(legacy)" as it's best to use the NEW Media Picker aka MediaPicker3 /// [DataEditor( Constants.PropertyEditors.Aliases.MediaPicker, EditorType.PropertyValue | EditorType.MacroParameter, - "(Obsolete) Media Picker", + "Media Picker (legacy)", "mediapicker", ValueType = ValueTypes.Text, Group = Constants.PropertyEditors.Groups.Media, Icon = Constants.Icons.MediaImage, - IsDeprecated = true)] + IsDeprecated = false)] public class MediaPickerPropertyEditor : DataEditor { From 7d53659fac2c9544e3dd7999a92b852fea840c04 Mon Sep 17 00:00:00 2001 From: Warren Buckley Date: Mon, 21 Jun 2021 17:27:34 +0100 Subject: [PATCH 123/147] Update src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs --- src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs index 1784b5fd0d..a33bed89fa 100644 --- a/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/MediaPickerPropertyEditor.cs @@ -8,7 +8,7 @@ namespace Umbraco.Web.PropertyEditors { /// /// Represents a media picker property editor. - /// Nameed "(legacy)" as it's best to use the NEW Media Picker aka MediaPicker3 + /// Named "(legacy)" as it's best to use the NEW Media Picker aka MediaPicker3 /// [DataEditor( Constants.PropertyEditors.Aliases.MediaPicker, From bb79544e2a88af0e93d44ca0eb4bc62959a7b4f6 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 22 Jun 2021 09:35:41 +0200 Subject: [PATCH 124/147] Update 01_bug_report.yml --- .github/ISSUE_TEMPLATE/01_bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/01_bug_report.yml b/.github/ISSUE_TEMPLATE/01_bug_report.yml index 04d1a0e04c..800ac53e68 100644 --- a/.github/ISSUE_TEMPLATE/01_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/01_bug_report.yml @@ -6,7 +6,7 @@ body: - type: input id: "version" attributes: - label: "Which Umbraco version are you using?" + label: "Which *exact* Umbraco version are you using? For example: 8.13.1 - don't just write v8" description: "Use the help icon in the Umbraco backoffice to find the version you're using" validations: required: true From 8a810e698c1b99affc9536abcd06f0cb3f40e581 Mon Sep 17 00:00:00 2001 From: Patrick de Mooij Date: Fri, 11 Jun 2021 22:09:08 +0200 Subject: [PATCH 125/147] 10429: Make sure to set filtered variable on media item --- .../infiniteeditors/mediapicker/mediapicker.controller.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/mediapicker/mediapicker.controller.js b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/mediapicker/mediapicker.controller.js index 029dedf214..0b9c59f2da 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/mediapicker/mediapicker.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/mediapicker/mediapicker.controller.js @@ -477,9 +477,15 @@ angular.module("umbraco") vm.loading = true; entityResource.getPagedDescendants($scope.filterOptions.excludeSubFolders ? $scope.currentFolder.id : $scope.startNodeId, "Media", vm.searchOptions) .then(function (data) { + // update image data to work with image grid if (data.items) { - data.items.forEach(mediaItem => setMediaMetaData(mediaItem)); + var allowedTypes = dialogOptions.filter ? dialogOptions.filter.split(",") : null; + + data.items.forEach(function(mediaItem) { + setMediaMetaData(mediaItem); + mediaItem.filtered = allowedTypes && allowedTypes.indexOf(mediaItem.metaData.ContentTypeAlias) < 0; + }); } // update images From 94d525d88f713b36419f28bfda4d82ee68637d83 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 22 Jun 2021 15:12:07 -0600 Subject: [PATCH 126/147] small changes based on review. --- .../Migrations/Upgrade/UmbracoPlan.cs | 4 +- .../AddCmsContentNuByteColumn.cs | 2 +- .../UpgradedIncludeIndexes.cs | 2 +- src/Umbraco.Core/Umbraco.Core.csproj | 4 +- src/Umbraco.Tests/Umbraco.Tests.csproj | 7 +- .../MsgPackContentNestedDataSerializer.cs | 87 +------------------ 6 files changed, 10 insertions(+), 96 deletions(-) rename src/Umbraco.Core/Migrations/Upgrade/{V_8_12_0 => V_8_15_0}/AddCmsContentNuByteColumn.cs (90%) rename src/Umbraco.Core/Migrations/Upgrade/{V_8_12_0 => V_8_15_0}/UpgradedIncludeIndexes.cs (98%) diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index 2a24c800b5..d9a71a1cc2 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -9,7 +9,7 @@ using Umbraco.Core.Migrations.Upgrade.V_8_1_0; using Umbraco.Core.Migrations.Upgrade.V_8_6_0; using Umbraco.Core.Migrations.Upgrade.V_8_9_0; using Umbraco.Core.Migrations.Upgrade.V_8_10_0; -using Umbraco.Core.Migrations.Upgrade.V_8_12_0; +using Umbraco.Core.Migrations.Upgrade.V_8_15_0; namespace Umbraco.Core.Migrations.Upgrade { @@ -203,7 +203,7 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.10.0 To("{D6A8D863-38EC-44FB-91EC-ACD6A668BD18}"); - // to 8.12.0... + // to 8.15.0... To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); To("{4695D0C9-0729-4976-985B-048D503665D8}"); diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs similarity index 90% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/AddCmsContentNuByteColumn.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs index 7c793688ec..3eab1a812e 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/AddCmsContentNuByteColumn.cs @@ -1,7 +1,7 @@ using System.Linq; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_12_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 { public class AddCmsContentNuByteColumn : MigrationBase { diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpgradedIncludeIndexes.cs similarity index 98% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/UpgradedIncludeIndexes.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpgradedIncludeIndexes.cs index d88abdef75..d51f99fb44 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_12_0/UpgradedIncludeIndexes.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_15_0/UpgradedIncludeIndexes.cs @@ -3,7 +3,7 @@ using Umbraco.Core.Migrations.Expressions.Execute.Expressions; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_12_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_15_0 { public class UpgradedIncludeIndexes : MigrationBase { diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 956ea7602e..514bd8d798 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -139,8 +139,8 @@ - - + + diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 2d6ce89a7b..815db985ce 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -79,8 +79,6 @@ - - 1.8.14 @@ -102,10 +100,7 @@ - - 4.14.5 - - + diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 944d93107d..6ae872ef69 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -28,8 +28,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // TODO: We want to be able to intern the strings for aliases when deserializing like we do for Newtonsoft but I'm unsure exactly how // to do that but it would seem to be with a custom message pack resolver but I haven't quite figured out based on the docs how // to do that since that is part of the int key -> string mapping operation, might have to see the source code to figure that one out. - - // resolver custom types first + // There are docs here on how to build one of these: https://github.com/neuecc/MessagePack-CSharp/blob/master/README.md#low-level-api-imessagepackformattert + // and there are a couple examples if you search on google for them but this will need to be a separate project. + // NOTE: resolver custom types first // new ContentNestedDataResolver(), // finally use standard resolver @@ -121,87 +122,5 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } } - - - - //private class ContentNestedDataResolver : IFormatterResolver - //{ - // // GetFormatter's get cost should be minimized so use type cache. - // public IMessagePackFormatter GetFormatter() => FormatterCache.Formatter; - - // private static class FormatterCache - // { - // public static readonly IMessagePackFormatter Formatter; - - // // generic's static constructor should be minimized for reduce type generation size! - // // use outer helper method. - // static FormatterCache() - // { - // Formatter = (IMessagePackFormatter)SampleCustomResolverGetFormatterHelper.GetFormatter(typeof(T)); - // } - // } - //} - - //internal static class SampleCustomResolverGetFormatterHelper - //{ - // // If type is concrete type, use type-formatter map - // static readonly Dictionary _formatterMap = new Dictionary() - // { - // {typeof(ContentNestedData), new ContentNestedDataFormatter()} - // // add more your own custom serializers. - // }; - - // internal static object GetFormatter(Type t) - // { - // object formatter; - // if (_formatterMap.TryGetValue(t, out formatter)) - // { - // return formatter; - // } - - // // If target type is generics, use MakeGenericType. - // if (t.IsGenericParameter && t.GetGenericTypeDefinition() == typeof(ValueTuple<,>)) - // { - // return Activator.CreateInstance(typeof(ValueTupleFormatter<,>).MakeGenericType(t.GenericTypeArguments)); - // } - - // // If type can not get, must return null for fallback mechanism. - // return null; - // } - //} - - //public class ContentNestedDataFormatter : IMessagePackFormatter - //{ - // public void Serialize(ref MessagePackWriter writer, ContentNestedData value, MessagePackSerializerOptions options) - // { - // if (value == null) - // { - // writer.WriteNil(); - // return; - // } - - // writer.WriteArrayHeader(3); - // writer.WriteString(value.UrlSegment); - // writer.WriteString(value.FullName); - // writer.WriteString(value.Age); - - // writer.WriteString(value.FullName); - // } - - // public ContentNestedData Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) - // { - // if (reader.TryReadNil()) - // { - // return null; - // } - - // options.Security.DepthStep(ref reader); - - // var path = reader.ReadString(); - - // reader.Depth--; - // return new FileInfo(path); - // } - //} } } From ec65a6c99b1887c2abe03e5210d8898cc8185ba9 Mon Sep 17 00:00:00 2001 From: Russell Date: Wed, 23 Jun 2021 09:15:05 +1200 Subject: [PATCH 127/147] Add lang to extended valid elements for span Resolution to https://github.com/umbraco/Umbraco-CMS/issues/10505 --- .../src/common/services/tinymce.service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/services/tinymce.service.js b/src/Umbraco.Web.UI.Client/src/common/services/tinymce.service.js index fbe342d44f..20e7ad665d 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/tinymce.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/tinymce.service.js @@ -11,7 +11,7 @@ function tinyMceService($rootScope, $q, imageHelper, $locale, $http, $timeout, s //These are absolutely required in order for the macros to render inline //we put these as extended elements because they get merged on top of the normal allowed elements by tiny mce - var extendedValidElements = "@[id|class|style],-div[id|dir|class|align|style],ins[datetime|cite],-ul[class|style],-li[class|style],-h1[id|dir|class|align|style],-h2[id|dir|class|align|style],-h3[id|dir|class|align|style],-h4[id|dir|class|align|style],-h5[id|dir|class|align|style],-h6[id|style|dir|class|align],span[id|class|style]"; + var extendedValidElements = "@[id|class|style],-div[id|dir|class|align|style],ins[datetime|cite],-ul[class|style],-li[class|style],-h1[id|dir|class|align|style],-h2[id|dir|class|align|style],-h3[id|dir|class|align|style],-h4[id|dir|class|align|style],-h5[id|dir|class|align|style],-h6[id|style|dir|class|align],span[id|class|style|lang]"; var fallbackStyles = [{ title: "Page header", block: "h2" }, { title: "Section header", block: "h3" }, { title: "Paragraph header", block: "h4" }, { title: "Normal", block: "p" }, { title: "Quote", block: "blockquote" }, { title: "Code", block: "code" }]; // these languages are available for localization var availableLanguages = [ From 6bd2f7f696bd6d7c04379c090ae096bbea00a526 Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Wed, 23 Jun 2021 09:10:40 +0200 Subject: [PATCH 128/147] Fixes the acceptance tests "Content with contentpicker" --- .../cypress/integration/Content/content.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts index 0cec374c5d..25d0b02e1d 100644 --- a/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts +++ b/src/Umbraco.Tests.AcceptanceTest/cypress/integration/Content/content.ts @@ -605,7 +605,7 @@ context('Content', () => { cy.umbracoEditorHeaderName('ContentPickerContent'); cy.get('.umb-node-preview-add').click(); // Should really try and find a better way to do this, but umbracoTreeItem tries to click the content pane in the background - cy.get('[ng-if="vm.treeReady"] > .umb-tree > [ng-if="!tree.root.containsGroups"] > .umb-animated > .umb-tree-item__inner').click(); + cy.get('[ng-if="vm.treeReady"] > .umb-tree .umb-tree-item__inner').click(); // We have to wait for the picked content to show up or it wont be added. cy.get('.umb-node-preview__description').should('be.visible'); //save and publish From 9382f6c32d430a4bafe0bfdbc3098c02f1f00925 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Wed, 23 Jun 2021 10:10:28 +0200 Subject: [PATCH 129/147] Add the ability to convert Media Picker v2 to v3 progressively (#10517) Co-authored-by: Ronald Barendse --- .../obsoletemediapickernotice.html | 1 - .../MediaPicker3PropertyEditor.cs | 99 ++++++++++++---- .../MediaPickerConfiguration.cs | 3 - .../MediaPickerWithCropsValueConverter.cs | 109 ++++++------------ 4 files changed, 116 insertions(+), 96 deletions(-) delete mode 100644 src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html diff --git a/src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html b/src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html deleted file mode 100644 index 617ffd80d6..0000000000 --- a/src/Umbraco.Web.UI.Client/src/views/prevalueeditors/obsoletemediapickernotice.html +++ /dev/null @@ -1 +0,0 @@ -

    Important: switching from "Media Picker (legacy)" to "Media Picker" is not supported and doing so will mean all data (references to previously selected media items) will no longer be available on existing content items.

    diff --git a/src/Umbraco.Web/PropertyEditors/MediaPicker3PropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/MediaPicker3PropertyEditor.cs index 4ce376f543..43d190e173 100644 --- a/src/Umbraco.Web/PropertyEditors/MediaPicker3PropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/MediaPicker3PropertyEditor.cs @@ -1,10 +1,15 @@ -using System.Collections.Generic; -using Newtonsoft.Json; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization; using Umbraco.Core; using Umbraco.Core.Logging; +using Umbraco.Core.Models; using Umbraco.Core.Models.Editors; using Umbraco.Core.PropertyEditors; -using Umbraco.Web.PropertyEditors.ValueConverters; +using Umbraco.Core.PropertyEditors.ValueConverters; +using Umbraco.Core.Services; namespace Umbraco.Web.PropertyEditors { @@ -22,43 +27,97 @@ namespace Umbraco.Web.PropertyEditors public class MediaPicker3PropertyEditor : DataEditor { /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// + /// The logger. public MediaPicker3PropertyEditor(ILogger logger) : base(logger) - { - } + { } /// protected override IConfigurationEditor CreateConfigurationEditor() => new MediaPicker3ConfigurationEditor(); + /// protected override IDataValueEditor CreateValueEditor() => new MediaPicker3PropertyValueEditor(Attribute); internal class MediaPicker3PropertyValueEditor : DataValueEditor, IDataValueReference { - /// - /// Note: no FromEditor() and ToEditor() methods - /// We do not want to transform the way the data is stored in the DB and would like to keep a raw JSON string - /// - public MediaPicker3PropertyValueEditor(DataEditorAttribute attribute) : base(attribute) + public MediaPicker3PropertyValueEditor(DataEditorAttribute attribute) + : base(attribute) + { } + + public override object ToEditor(Property property, IDataTypeService dataTypeService, string culture = null, string segment = null) { + var value = property.GetValue(culture, segment); + + return Deserialize(value); } public IEnumerable GetReferences(object value) { - var rawJson = value == null ? string.Empty : value is string str ? str : value.ToString(); - - if (rawJson.IsNullOrWhiteSpace()) - yield break; - - var mediaWithCropsDtos = JsonConvert.DeserializeObject(rawJson); - - foreach (var mediaWithCropsDto in mediaWithCropsDtos) + foreach (var dto in Deserialize(value)) { - yield return new UmbracoEntityReference(GuidUdi.Create(Constants.UdiEntityType.Media, mediaWithCropsDto.MediaKey)); + yield return new UmbracoEntityReference(Udi.Create(Constants.UdiEntityType.Media, dto.MediaKey)); } } + internal static IEnumerable Deserialize(object value) + { + var rawJson = value is string str ? str : value?.ToString(); + if (string.IsNullOrWhiteSpace(rawJson)) + { + yield break; + } + + if (!rawJson.DetectIsJson()) + { + // Old comma seperated UDI format + foreach (var udiStr in rawJson.Split(Constants.CharArrays.Comma)) + { + if (GuidUdi.TryParse(udiStr, out var udi)) + { + yield return new MediaWithCropsDto + { + Key = Guid.NewGuid(), + MediaKey = udi.Guid, + Crops = Enumerable.Empty(), + FocalPoint = new ImageCropperValue.ImageCropperFocalPoint + { + Left = 0.5m, + Top = 0.5m + } + }; + } + } + } + else + { + // New JSON format + foreach (var dto in JsonConvert.DeserializeObject>(rawJson)) + { + yield return dto; + } + } + } + + /// + /// Model/DTO that represents the JSON that the MediaPicker3 stores. + /// + [DataContract] + internal class MediaWithCropsDto + { + [DataMember(Name = "key")] + public Guid Key { get; set; } + + [DataMember(Name = "mediaKey")] + public Guid MediaKey { get; set; } + + [DataMember(Name = "crops")] + public IEnumerable Crops { get; set; } + + [DataMember(Name = "focalPoint")] + public ImageCropperValue.ImageCropperFocalPoint FocalPoint { get; set; } + } } } } diff --git a/src/Umbraco.Web/PropertyEditors/MediaPickerConfiguration.cs b/src/Umbraco.Web/PropertyEditors/MediaPickerConfiguration.cs index 7266be9c26..b8b9476184 100644 --- a/src/Umbraco.Web/PropertyEditors/MediaPickerConfiguration.cs +++ b/src/Umbraco.Web/PropertyEditors/MediaPickerConfiguration.cs @@ -8,9 +8,6 @@ namespace Umbraco.Web.PropertyEditors ///     public class MediaPickerConfiguration : IIgnoreUserStartNodesConfig { - [ConfigurationField("notice", "You can NOT change the property editor", "obsoletemediapickernotice")] - public bool Notice { get; set; } - [ConfigurationField("multiPicker", "Pick multiple items", "boolean")] public bool Multiple { get; set; } diff --git a/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs b/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs index f9b2ad75e1..f2f055d698 100644 --- a/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs +++ b/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs @@ -1,8 +1,6 @@ -using Newtonsoft.Json; -using System; -using System.Collections; +using System; using System.Collections.Generic; -using System.Runtime.Serialization; +using System.Linq; using Umbraco.Core.Models; using Umbraco.Core.Models.PublishedContent; using Umbraco.Core.PropertyEditors; @@ -14,7 +12,6 @@ namespace Umbraco.Web.PropertyEditors.ValueConverters [DefaultPropertyValueConverter] public class MediaPickerWithCropsValueConverter : PropertyValueConverterBase { - private readonly IPublishedSnapshotAccessor _publishedSnapshotAccessor; public MediaPickerWithCropsValueConverter(IPublishedSnapshotAccessor publishedSnapshotAccessor) @@ -22,98 +19,66 @@ namespace Umbraco.Web.PropertyEditors.ValueConverters _publishedSnapshotAccessor = publishedSnapshotAccessor ?? throw new ArgumentNullException(nameof(publishedSnapshotAccessor)); } - public override PropertyCacheLevel GetPropertyCacheLevel(IPublishedPropertyType propertyType) => PropertyCacheLevel.Snapshot; - - /// - /// Enusre this property value convertor is for the New Media Picker with Crops aka MediaPicker 3 - /// public override bool IsConverter(IPublishedPropertyType propertyType) => propertyType.EditorAlias.Equals(Core.Constants.PropertyEditors.Aliases.MediaPicker3); - /// - /// Check if the raw JSON value is not an empty array - /// - public override bool? IsValue(object value, PropertyValueLevel level) => value?.ToString() != "[]"; - - /// - /// What C# model type does the raw JSON return for Models & Views - /// - public override Type GetPropertyValueType(IPublishedPropertyType propertyType) + public override bool? IsValue(object value, PropertyValueLevel level) { - // Check do we want to return IPublishedContent collection still or a NEW model ? - var isMultiple = IsMultipleDataType(propertyType.DataType); - return isMultiple - ? typeof(IEnumerable) - : typeof(MediaWithCrops); + var isValue = base.IsValue(value, level); + if (isValue != false && level == PropertyValueLevel.Source) + { + // Empty JSON array is not a value + isValue = value?.ToString() != "[]"; + } + + return isValue; } - public override object ConvertSourceToIntermediate(IPublishedElement owner, IPublishedPropertyType propertyType, object source, bool preview) => source?.ToString(); + public override Type GetPropertyValueType(IPublishedPropertyType propertyType) + => IsMultipleDataType(propertyType.DataType) + ? typeof(IEnumerable) + : typeof(MediaWithCrops); + + public override PropertyCacheLevel GetPropertyCacheLevel(IPublishedPropertyType propertyType) => PropertyCacheLevel.Snapshot; public override object ConvertIntermediateToObject(IPublishedElement owner, IPublishedPropertyType propertyType, PropertyCacheLevel referenceCacheLevel, object inter, bool preview) { - var mediaItems = new List(); var isMultiple = IsMultipleDataType(propertyType.DataType); - if (inter == null) + if (string.IsNullOrEmpty(inter?.ToString())) { - return isMultiple ? mediaItems: null; + // Short-circuit on empty value + return isMultiple ? Enumerable.Empty() : null; } - var dtos = JsonConvert.DeserializeObject>(inter.ToString()); + var mediaItems = new List(); + var dtos = MediaPicker3PropertyEditor.MediaPicker3PropertyValueEditor.Deserialize(inter); - foreach(var media in dtos) + foreach (var dto in dtos) { - var item = _publishedSnapshotAccessor.PublishedSnapshot.Media.GetById(media.MediaKey); - if (item != null) + var mediaItem = _publishedSnapshotAccessor.PublishedSnapshot.Media.GetById(preview, dto.MediaKey); + if (mediaItem != null) { mediaItems.Add(new MediaWithCrops { - MediaItem = item, + MediaItem = mediaItem, LocalCrops = new ImageCropperValue { - Crops = media.Crops, - FocalPoint = media.FocalPoint, - Src = item.Url() + Crops = dto.Crops, + FocalPoint = dto.FocalPoint, + Src = mediaItem.Url() } }); + + if (!isMultiple) + { + // Short-circuit on single item + break; + } } } - return isMultiple ? mediaItems : FirstOrDefault(mediaItems); + return isMultiple ? mediaItems : mediaItems.FirstOrDefault(); } - /// - /// Is the media picker configured to pick multiple media items - /// - /// - /// - private bool IsMultipleDataType(PublishedDataType dataType) - { - var config = dataType.ConfigurationAs(); - return config.Multiple; - } - - private object FirstOrDefault(IList mediaItems) - { - return mediaItems.Count == 0 ? null : mediaItems[0]; - } - - - /// - /// Model/DTO that represents the JSON that the MediaPicker3 stores - /// - [DataContract] - internal class MediaWithCropsDto - { - [DataMember(Name = "key")] - public Guid Key { get; set; } - - [DataMember(Name = "mediaKey")] - public Guid MediaKey { get; set; } - - [DataMember(Name = "crops")] - public IEnumerable Crops { get; set; } - - [DataMember(Name = "focalPoint")] - public ImageCropperValue.ImageCropperFocalPoint FocalPoint { get; set; } - } + private bool IsMultipleDataType(PublishedDataType dataType) => dataType.ConfigurationAs().Multiple; } } From 2c59da91b60c32b7762cdf3c755988a980ab5745 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 23 Jun 2021 11:02:36 -0600 Subject: [PATCH 130/147] fix merge --- .../DataSource/BTree.DictionaryOfPropertyDataSerializer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index 1b96538dd0..579e53177d 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -34,7 +34,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource for (var j = 0; j < vcount; j++) { var pdata = new PropertyData(); - pdatas[j] =pdata; + pdatas[j] = pdata; // everything that can be null is read/written as object // even though - culture and segment should never be null here, as 'null' represents From 56b7b0df69e473d49dc89cf3f43817560e66d23a Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 23 Jun 2021 11:13:19 -0600 Subject: [PATCH 131/147] fix build, re-adds braces after merge --- ...Tree.DictionaryOfPropertyDataSerializer.cs | 4 +-- src/Umbraco.Web/PublishedPropertyExtension.cs | 26 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index fa0d853e35..304fc6def0 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -13,12 +13,10 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public IDictionary ReadFrom(Stream stream) { - // read properties count - var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); var dict = new Dictionary(pcount,StringComparer.InvariantCultureIgnoreCase); - var dict = new Dictionary(pcount,StringComparer.InvariantCultureIgnoreCase); // read each property for (var i = 0; i < pcount; i++) { diff --git a/src/Umbraco.Web/PublishedPropertyExtension.cs b/src/Umbraco.Web/PublishedPropertyExtension.cs index b431f24828..4f27a429a4 100644 --- a/src/Umbraco.Web/PublishedPropertyExtension.cs +++ b/src/Umbraco.Web/PublishedPropertyExtension.cs @@ -36,10 +36,17 @@ namespace Umbraco.Web { // we have a value // try to cast or convert it - var value = property.GetValue(culture, segment); - if (value is T valueAsT) return valueAsT; + var value = property.GetValue(culture, segment); + if (value is T valueAsT) + { + return valueAsT; + } + var valueConverted = value.TryConvertTo(); - if (valueConverted) return valueConverted.Result; + if (valueConverted) + { + return valueConverted.Result; + } // cannot cast nor convert the value, nothing we can return but 'default' // note: we don't want to fallback in that case - would make little sense @@ -48,14 +55,23 @@ namespace Umbraco.Web // we don't have a value, try fallback if (PublishedValueFallback.TryGetValue(property, culture, segment, fallback, defaultValue, out var fallbackValue)) + { return fallbackValue; + } // we don't have a value - neither direct nor fallback // give a chance to the converter to return something (eg empty enumerable) var noValue = property.GetValue(culture, segment); - if (noValue is T noValueAsT) return noValueAsT; + if (noValue is T noValueAsT) + { + return noValueAsT; + } + var noValueConverted = noValue.TryConvertTo(); - if (noValueConverted) return noValueConverted.Result; + if (noValueConverted) + { + return noValueConverted.Result; + } // cannot cast noValue nor convert it, nothing we can return but 'default' return default; From 76d206a4badd7b267d861a36043be62cb7ab25cf Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 23 Jun 2021 12:46:50 -0600 Subject: [PATCH 132/147] Fixes back office searching --- src/Umbraco.Web/IPublishedContentQuery.cs | 4 +- .../Models/Mapping/EntityMapDefinition.cs | 2 +- .../Search/IUmbracoTreeSearcherFields.cs | 11 ++- .../Search/IUmbracoTreeSearcherFields2.cs | 17 +++-- src/Umbraco.Web/Search/UmbracoTreeSearcher.cs | 30 ++++++-- .../Search/UmbracoTreeSearcherFields.cs | 75 ++++++++++--------- 6 files changed, 84 insertions(+), 55 deletions(-) diff --git a/src/Umbraco.Web/IPublishedContentQuery.cs b/src/Umbraco.Web/IPublishedContentQuery.cs index f513c1ac02..369fb6ca3d 100644 --- a/src/Umbraco.Web/IPublishedContentQuery.cs +++ b/src/Umbraco.Web/IPublishedContentQuery.cs @@ -8,7 +8,7 @@ using Umbraco.Core.Xml; namespace Umbraco.Web { - using Examine = global::Examine; + // TODO: Merge this into IPublishedContentQuery for v9! public interface IPublishedContentQuery2 : IPublishedContentQuery { /// @@ -32,7 +32,7 @@ namespace Umbraco.Web /// /// While enumerating results, the ambient culture is changed to be the searched culture. /// - IEnumerable Search(string term, int skip, int take, out long totalRecords, string culture = "*", string indexName = Umbraco.Core.Constants.UmbracoIndexes.ExternalIndexName, ISet loadedFields = null); + IEnumerable Search(string term, int skip, int take, out long totalRecords, string culture = "*", string indexName = Constants.UmbracoIndexes.ExternalIndexName, ISet loadedFields = null); } /// diff --git a/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs index 1c4ca6087c..621487ae04 100644 --- a/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/EntityMapDefinition.cs @@ -180,7 +180,7 @@ namespace Umbraco.Web.Models.Mapping target.Name = source.Values.ContainsKey("nodeName") ? source.Values["nodeName"] : "[no name]"; - var culture = context.GetCulture(); + var culture = context.GetCulture()?.ToLowerInvariant(); if(culture.IsNullOrWhiteSpace() == false) { target.Name = source.Values.ContainsKey($"nodeName_{culture}") ? source.Values[$"nodeName_{culture}"] : target.Name; diff --git a/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs b/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs index c5a6c53d19..b3a4ec7055 100644 --- a/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs +++ b/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields.cs @@ -8,19 +8,22 @@ namespace Umbraco.Web.Search public interface IUmbracoTreeSearcherFields { /// - /// Propagate list of searchable fields for all node types + /// The default index fields that are searched on in the back office search for umbraco content entities. /// IEnumerable GetBackOfficeFields(); + /// - /// Propagate list of searchable fields for Members + /// The additional index fields that are searched on in the back office for member entities. /// IEnumerable GetBackOfficeMembersFields(); + /// - /// Propagate list of searchable fields for Media + /// The additional index fields that are searched on in the back office for media entities. /// IEnumerable GetBackOfficeMediaFields(); + /// - /// Propagate list of searchable fields for Documents + /// The additional index fields that are searched on in the back office for document entities. /// IEnumerable GetBackOfficeDocumentFields(); } diff --git a/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs b/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs index 42f592a965..da0cd26644 100644 --- a/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs +++ b/src/Umbraco.Web/Search/IUmbracoTreeSearcherFields2.cs @@ -1,28 +1,29 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Collections.Generic; namespace Umbraco.Web.Search { + // TODO: Merge this interface to IUmbracoTreeSearcherFields for v9. + // We should probably make these method make a little more sense when they are combined so have + // a single method for getting fields to search and fields to load for each category. public interface IUmbracoTreeSearcherFields2 : IUmbracoTreeSearcherFields { /// /// Set of fields for all node types to be loaded /// ISet GetBackOfficeFieldsToLoad(); + /// - /// Set list of fields for Members to be loaded + /// Additional set list of fields for Members to be loaded /// ISet GetBackOfficeMembersFieldsToLoad(); + /// - /// Set of fields for Media to be loaded + /// Additional set of fields for Media to be loaded /// ISet GetBackOfficeMediaFieldsToLoad(); /// - /// Set of fields for Documents to be loaded + /// Additional set of fields for Documents to be loaded /// ISet GetBackOfficeDocumentFieldsToLoad(); } diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs index 839656ebef..ed2d8d3c86 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs @@ -101,7 +101,9 @@ namespace Umbraco.Web.Search string type; var indexName = Constants.UmbracoIndexes.InternalIndexName; var fields = _umbracoTreeSearcherFields.GetBackOfficeFields().ToList(); - ISet fieldsToLoad = null; + ISet fieldsToLoad = _umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 searcherFields2 + ? new HashSet(searcherFields2.GetBackOfficeFieldsToLoad()) + : null; // TODO: WE should try to allow passing in a lucene raw query, however we will still need to do some manual string // manipulation for things like start paths, member types, etc... @@ -124,7 +126,10 @@ namespace Umbraco.Web.Search fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMembersFields()); if (_umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 umbracoTreeSearcherFieldMember) { - fieldsToLoad = umbracoTreeSearcherFieldMember.GetBackOfficeMembersFieldsToLoad(); + foreach(var field in umbracoTreeSearcherFieldMember.GetBackOfficeMembersFieldsToLoad()) + { + fieldsToLoad.Add(field); + } } if (searchFrom != null && searchFrom != Constants.Conventions.MemberTypes.AllMembersListId && searchFrom.Trim() != "-1") { @@ -138,9 +143,13 @@ namespace Umbraco.Web.Search fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeMediaFields()); if (_umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 umbracoTreeSearcherFieldsMedia) { - fieldsToLoad = umbracoTreeSearcherFieldsMedia.GetBackOfficeMediaFieldsToLoad(); + foreach (var field in umbracoTreeSearcherFieldsMedia.GetBackOfficeMediaFieldsToLoad()) + { + fieldsToLoad.Add(field); + } } - var allMediaStartNodes = _umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService _appCaches); + + var allMediaStartNodes = _umbracoContext.Security.CurrentUser.CalculateMediaStartNodeIds(_entityService, _appCaches); AppendPath(sb, UmbracoObjectTypes.Media, allMediaStartNodes, searchFrom, ignoreUserStartNodes, _entityService); break; case UmbracoEntityTypes.Document: @@ -148,7 +157,10 @@ namespace Umbraco.Web.Search fields.AddRange(_umbracoTreeSearcherFields.GetBackOfficeDocumentFields()); if (_umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 umbracoTreeSearcherFieldsDocument) { - fieldsToLoad = umbracoTreeSearcherFieldsDocument.GetBackOfficeDocumentFieldsToLoad(); + foreach (var field in umbracoTreeSearcherFieldsDocument.GetBackOfficeDocumentFieldsToLoad()) + { + fieldsToLoad.Add(field); + } } var allContentStartNodes = _umbracoContext.Security.CurrentUser.CalculateContentStartNodeIds(_entityService, _appCaches); AppendPath(sb, UmbracoObjectTypes.Document, allContentStartNodes, searchFrom, ignoreUserStartNodes, _entityService); @@ -168,7 +180,13 @@ namespace Umbraco.Web.Search return Enumerable.Empty(); } - var result = internalSearcher.CreateQuery().NativeQuery(sb.ToString()).SelectFields(fieldsToLoad) + var examineQuery = internalSearcher.CreateQuery().NativeQuery(sb.ToString()); + if (fieldsToLoad != null) + { + examineQuery.SelectFields(fieldsToLoad); + } + + var result = examineQuery //only return the number of items specified to read up to the amount of records to fill from 0 -> the number of items on the page requested .Execute(Convert.ToInt32(pageSize * (pageIndex + 1))); diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs index 5a2fd91d18..3025f869b4 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcherFields.cs @@ -1,55 +1,62 @@ +using Examine; +using Examine.LuceneEngine.Providers; using System.Collections.Generic; using System.Linq; +using Umbraco.Core.Services; using Umbraco.Examine; namespace Umbraco.Web.Search { public class UmbracoTreeSearcherFields : IUmbracoTreeSearcherFields2 { - private IReadOnlyList _backOfficeFields = new List {"id", "__NodeId", "__Key"}; - public IEnumerable GetBackOfficeFields() + private IReadOnlyList _backOfficeFields = new List {"id", LuceneIndex.ItemIdFieldName, UmbracoExamineIndex.NodeKeyFieldName}; + private readonly ISet _backOfficeFieldsToLoad = new HashSet { "id", LuceneIndex.ItemIdFieldName, UmbracoExamineIndex.NodeKeyFieldName, "nodeName", UmbracoExamineIndex.IconFieldName, LuceneIndex.CategoryFieldName, "parentID", LuceneIndex.ItemTypeFieldName }; + private IReadOnlyList _backOfficeMediaFields = new List { UmbracoExamineIndex.UmbracoFileFieldName }; + private readonly ISet _backOfficeMediaFieldsToLoad = new HashSet { UmbracoExamineIndex.UmbracoFileFieldName }; + private IReadOnlyList _backOfficeMembersFields = new List { "email", "loginName" }; + private readonly ISet _backOfficeMembersFieldsToLoad = new HashSet { "email", "loginName" }; + private readonly ISet _backOfficeDocumentFieldsToLoad = new HashSet { UmbracoContentIndex.VariesByCultureFieldName }; + private readonly ILocalizationService _localizationService; + + public UmbracoTreeSearcherFields(ILocalizationService localizationService) { - return _backOfficeFields; + _localizationService = localizationService; } + /// + public IEnumerable GetBackOfficeFields() => _backOfficeFields; - private IReadOnlyList _backOfficeMembersFields = new List {"email", "loginName"}; - public IEnumerable GetBackOfficeMembersFields() - { - return _backOfficeMembersFields; - } - private IReadOnlyList _backOfficeMediaFields = new List {UmbracoExamineIndex.UmbracoFileFieldName }; - public IEnumerable GetBackOfficeMediaFields() - { - return _backOfficeMediaFields; - } - public IEnumerable GetBackOfficeDocumentFields() - { - return Enumerable.Empty(); - } + /// + public IEnumerable GetBackOfficeMembersFields() => _backOfficeMembersFields; - private readonly ISet _backOfficeFieldsToLoad = new HashSet { "id", "__NodeId", "__Key" }; - public ISet GetBackOfficeFieldsToLoad() - { - return _backOfficeFieldsToLoad; - } + /// + public IEnumerable GetBackOfficeMediaFields() => _backOfficeMediaFields; - private readonly ISet _backOfficeMembersFieldsToLoad = new HashSet { "id", "__NodeId", "__Key", "email", "loginName" }; - public ISet GetBackOfficeMembersFieldsToLoad() - { - return _backOfficeMembersFieldsToLoad; - } + /// + public IEnumerable GetBackOfficeDocumentFields() => Enumerable.Empty(); - private readonly ISet _backOfficeMediaFieldsToLoad = new HashSet { "id", "__NodeId", "__Key", UmbracoExamineIndex.UmbracoFileFieldName }; - public ISet GetBackOfficeMediaFieldsToLoad() - { - return _backOfficeMediaFieldsToLoad; - } - private readonly ISet _backOfficeDocumentFieldsToLoad = new HashSet { "id", "__NodeId", "__Key" }; + /// + public ISet GetBackOfficeFieldsToLoad() => _backOfficeFieldsToLoad; + /// + public ISet GetBackOfficeMembersFieldsToLoad() => _backOfficeMembersFieldsToLoad; + + /// + public ISet GetBackOfficeMediaFieldsToLoad() => _backOfficeMediaFieldsToLoad; + + /// public ISet GetBackOfficeDocumentFieldsToLoad() { - return _backOfficeDocumentFieldsToLoad; + var fields = _backOfficeDocumentFieldsToLoad; + + // We need to load all nodeName_* fields but we won't know those up front so need to get + // all langs (this is cached) + foreach(var field in _localizationService.GetAllLanguages().Select(x => "nodeName_" + x.IsoCode.ToLowerInvariant())) + { + fields.Add(field); + } + + return fields; } } } From 30d57125ce44b25b9d143b829190ecf9c932f79b Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 23 Jun 2021 13:14:22 -0600 Subject: [PATCH 133/147] Adds notes, obsoletes old localize methods --- .../Services/ILocalizedTextService.cs | 7 +++++- .../Implement/LocalizedTextService.cs | 15 +++++------ .../LocalizedTextServiceExtensions.cs | 25 +++++++++++++++++-- .../Editors/BackOfficeController.cs | 2 +- src/Umbraco.Web/Search/UmbracoTreeSearcher.cs | 2 ++ 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/Umbraco.Core/Services/ILocalizedTextService.cs b/src/Umbraco.Core/Services/ILocalizedTextService.cs index 73212546a5..c74aa209c6 100644 --- a/src/Umbraco.Core/Services/ILocalizedTextService.cs +++ b/src/Umbraco.Core/Services/ILocalizedTextService.cs @@ -1,9 +1,13 @@ -using System.Collections; +using System; +using System.Collections; using System.Collections.Generic; using System.Globalization; namespace Umbraco.Core.Services { + // TODO: This needs to be merged into one interface in v9, but better yet + // the Localize method should just the based on area + alias and we should remove + // the one with the 'key' (the concatenated area/alias) to ensure that we never use that again. public interface ILocalizedTextService2 : ILocalizedTextService { @@ -41,6 +45,7 @@ namespace Umbraco.Core.Services /// /// This can be null /// + [Obsolete("Use LocalizedTextServiceExtensions.Localize or ILocalizedTextService2.Localize instead")] string Localize(string key, CultureInfo culture, IDictionary tokens = null); /// diff --git a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs index 8acb75645b..4f1766d6f4 100644 --- a/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs +++ b/src/Umbraco.Core/Services/Implement/LocalizedTextService.cs @@ -1,5 +1,4 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -9,8 +8,6 @@ using Umbraco.Core.Logging; namespace Umbraco.Core.Services.Implement { - // TODO: Convert all of this over to Niels K's localization framework one day - public class LocalizedTextService : ILocalizedTextService2 { private readonly ILogger _logger; @@ -27,9 +24,9 @@ namespace Umbraco.Core.Services.Implement /// public LocalizedTextService(Lazy fileSources, ILogger logger) { - if (logger == null) throw new ArgumentNullException("logger"); + if (logger == null) throw new ArgumentNullException(nameof(logger)); _logger = logger; - if (fileSources == null) throw new ArgumentNullException("fileSources"); + if (fileSources == null) throw new ArgumentNullException(nameof(fileSources)); _dictionarySourceLazy = new Lazy>>>(() => FileSourcesToAreaDictionarySources(fileSources.Value)); _noAreaDictionarySourceLazy = new Lazy>>(() => FileSourcesToNoAreaDictionarySources(fileSources.Value)); _fileSources = fileSources; @@ -78,9 +75,9 @@ namespace Umbraco.Core.Services.Implement /// public LocalizedTextService(IDictionary> source, ILogger logger) { - if (source == null) throw new ArgumentNullException("source"); - if (logger == null) throw new ArgumentNullException("logger"); - _logger = logger; + if (source == null) throw new ArgumentNullException(nameof(source)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _dictionarySourceLazy = new Lazy>>>(() => XmlSourcesToAreaDictionary(source)); _noAreaDictionarySourceLazy = new Lazy>>(() => XmlSourceToNoAreaDictionary(source)); @@ -150,7 +147,7 @@ namespace Umbraco.Core.Services.Implement /// public IDictionary GetAllStoredValues(CultureInfo culture) { - if (culture == null) throw new ArgumentNullException("culture"); + if (culture == null) throw new ArgumentNullException(nameof(culture)); // TODO: Hack, see notes on ConvertToSupportedCultureWithRegionCode culture = ConvertToSupportedCultureWithRegionCode(culture); diff --git a/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs b/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs index 176a1ae240..2911441578 100644 --- a/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs +++ b/src/Umbraco.Core/Services/LocalizedTextServiceExtensions.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Threading; @@ -12,6 +13,8 @@ namespace Umbraco.Core.Services /// public static class LocalizedTextServiceExtensions { + // TODO: Remove these extension methods checking for ILocalizedTextService2 in v9 when these interfaces merge + public static string Localize(this ILocalizedTextService manager, string area, string alias, CultureInfo culture) { if(manager is ILocalizedTextService2 manager2) @@ -23,8 +26,11 @@ namespace Umbraco.Core.Services { fullKey = string.Concat(area, "/", alias); } +#pragma warning disable CS0618 // Type or member is obsolete return manager.Localize(fullKey, culture); +#pragma warning restore CS0618 // Type or member is obsolete } + public static string Localize(this ILocalizedTextService manager, string area, string alias) { if (manager is ILocalizedTextService2 manager2) @@ -36,8 +42,11 @@ namespace Umbraco.Core.Services { fullKey = string.Concat(area, "/", alias); } +#pragma warning disable CS0618 // Type or member is obsolete return manager.Localize(fullKey, Thread.CurrentThread.CurrentUICulture); +#pragma warning restore CS0618 // Type or member is obsolete } + /// /// Localize using the current thread culture /// @@ -57,8 +66,11 @@ namespace Umbraco.Core.Services { fullKey = string.Concat(area, "/", alias); } +#pragma warning disable CS0618 // Type or member is obsolete return manager.Localize(fullKey, Thread.CurrentThread.CurrentUICulture, tokens); +#pragma warning restore CS0618 // Type or member is obsolete } + /// /// Localize using the current thread culture /// @@ -78,10 +90,11 @@ namespace Umbraco.Core.Services { fullKey = string.Concat(area, "/", alias); } +#pragma warning disable CS0618 // Type or member is obsolete return manager.Localize(fullKey, Thread.CurrentThread.CurrentUICulture, tokens); +#pragma warning restore CS0618 // Type or member is obsolete } - /// /// Localize a key without any variables /// @@ -102,7 +115,9 @@ namespace Umbraco.Core.Services { fullKey = string.Concat(area, "/", alias); } +#pragma warning disable CS0618 // Type or member is obsolete return manager.Localize(fullKey, culture, ConvertToDictionaryVars(tokens)); +#pragma warning restore CS0618 // Type or member is obsolete } /// @@ -125,8 +140,11 @@ namespace Umbraco.Core.Services { fullKey = string.Concat(area, "/", alias); } +#pragma warning disable CS0618 // Type or member is obsolete return manager.Localize(fullKey, culture, tokens); +#pragma warning restore CS0618 // Type or member is obsolete } + /// /// Localize using the current thread culture /// @@ -134,6 +152,7 @@ namespace Umbraco.Core.Services /// /// /// + [Obsolete("Use the overload specifying an area and alias instead of key")] public static string Localize(this ILocalizedTextService manager, string key, string[] tokens) { return manager.Localize(key, Thread.CurrentThread.CurrentUICulture, tokens); @@ -146,6 +165,7 @@ namespace Umbraco.Core.Services /// /// /// + [Obsolete("Use the overload specifying an area and alias instead of key")] public static string Localize(this ILocalizedTextService manager, string key, IDictionary tokens = null) { return manager.Localize(key, Thread.CurrentThread.CurrentUICulture, tokens); @@ -159,6 +179,7 @@ namespace Umbraco.Core.Services /// /// /// + [Obsolete("Use the overload specifying an area and alias instead of key")] public static string Localize(this ILocalizedTextService manager, string key, CultureInfo culture, string[] tokens) { return manager.Localize(key, culture, ConvertToDictionaryVars(tokens)); diff --git a/src/Umbraco.Web/Editors/BackOfficeController.cs b/src/Umbraco.Web/Editors/BackOfficeController.cs index baba9da894..a242e72c32 100644 --- a/src/Umbraco.Web/Editors/BackOfficeController.cs +++ b/src/Umbraco.Web/Editors/BackOfficeController.cs @@ -178,7 +178,7 @@ namespace Umbraco.Web.Editors : CultureInfo.GetCultureInfo(GlobalSettings.DefaultUILanguage) : CultureInfo.GetCultureInfo(culture); - + // TODO: Remove this check in v9 when these interfaces merge if(Services.TextService is ILocalizedTextService2 localizedText2) { var nestedDictionary2 = localizedText2.GetAllStoredValuesByAreaAndAlias(cultureInfo); diff --git a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs index ed2d8d3c86..c4560b20c3 100644 --- a/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs +++ b/src/Umbraco.Web/Search/UmbracoTreeSearcher.cs @@ -101,6 +101,8 @@ namespace Umbraco.Web.Search string type; var indexName = Constants.UmbracoIndexes.InternalIndexName; var fields = _umbracoTreeSearcherFields.GetBackOfficeFields().ToList(); + + // TODO: Remove these checks in v9 when these interfaces merge ISet fieldsToLoad = _umbracoTreeSearcherFields is IUmbracoTreeSearcherFields2 searcherFields2 ? new HashSet(searcherFields2.GetBackOfficeFieldsToLoad()) : null; From 44c814a550f1060e4255895a5e9515cbf153b6b8 Mon Sep 17 00:00:00 2001 From: Chad Date: Mon, 28 Jun 2021 12:20:38 +1200 Subject: [PATCH 134/147] Fix Rollback feature not creating diffs when properties are moved between tabs in EditorModel events (#10376) * Find the old property by alias instead of by indexes as if you move around properties in the EditorModel events Rollback breaks. * optimise property lookup - avoids iterating entire property set Co-authored-by: Nathan Woulfe --- .../rollback/rollback.controller.js | 73 +++++++++++-------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/rollback/rollback.controller.js b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/rollback/rollback.controller.js index 0d49c7dd9c..4b0dfcb8b4 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/rollback/rollback.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/common/infiniteeditors/rollback/rollback.controller.js @@ -112,7 +112,6 @@ * This will load in a new version */ function createDiff(currentVersion, previousVersion) { - vm.diff = {}; vm.diff.properties = []; @@ -120,41 +119,55 @@ vm.diff.name = JsDiff.diffWords(currentVersion.name, previousVersion.name); // extract all properties from the tabs and create new object for the diff - currentVersion.tabs.forEach((tab, tabIndex) => { - tab.properties.forEach((property, propertyIndex) => { - var oldProperty = previousVersion.tabs[tabIndex].properties[propertyIndex]; + currentVersion.tabs.forEach(function (tab) { + tab.properties.forEach(function (property) { + let oldTabIndex = -1; + let oldTabPropertyIndex = -1; + const previousVersionTabs = previousVersion.tabs; - // copy existing properties, so it doesn't manipulate existing properties on page - oldProperty = Utilities.copy(oldProperty); - property = Utilities.copy(property); - - // we have to make properties storing values as object into strings (Grid, nested content, etc.) - if(property.value instanceof Object) { - property.value = JSON.stringify(property.value, null, 1); - property.isObject = true; + // find the property by alias, but only search until we find it + for (var oti = 0, length = previousVersionTabs.length; oti < length; oti++) { + const opi = previousVersionTabs[oti].properties.findIndex(p => p.alias === property.alias); + if (opi !== -1) { + oldTabIndex = oti; + oldTabPropertyIndex = opi; + break; + } } - if(oldProperty.value instanceof Object) { - oldProperty.value = JSON.stringify(oldProperty.value, null, 1); - oldProperty.isObject = true; + if (oldTabIndex !== -1 && oldTabPropertyIndex !== -1) { + let oldProperty = previousVersion.tabs[oldTabIndex].properties[oldTabPropertyIndex]; + + // copy existing properties, so it doesn't manipulate existing properties on page + oldProperty = Utilities.copy(oldProperty); + property = Utilities.copy(property); + + // we have to make properties storing values as object into strings (Grid, nested content, etc.) + if (property.value instanceof Object) { + property.value = JSON.stringify(property.value, null, 1); + property.isObject = true; + } + + if (oldProperty.value instanceof Object) { + oldProperty.value = JSON.stringify(oldProperty.value, null, 1); + oldProperty.isObject = true; + } + + // diff requires a string + property.value = property.value ? property.value + '' : ''; + oldProperty.value = oldProperty.value ? oldProperty.value + '' : ''; + + const diffProperty = { + 'alias': property.alias, + 'label': property.label, + 'diff': property.isObject ? JsDiff.diffJson(property.value, oldProperty.value) : JsDiff.diffWords(property.value, oldProperty.value), + 'isObject': property.isObject || oldProperty.isObject + }; + + vm.diff.properties.push(diffProperty); } - - // diff requires a string - property.value = property.value ? property.value + "" : ""; - oldProperty.value = oldProperty.value ? oldProperty.value + "" : ""; - - var diffProperty = { - "alias": property.alias, - "label": property.label, - "diff": (property.isObject) ? JsDiff.diffJson(property.value, oldProperty.value) : JsDiff.diffWords(property.value, oldProperty.value), - "isObject": (property.isObject || oldProperty.isObject) ? true : false - }; - - vm.diff.properties.push(diffProperty); - }); }); - } function rollback() { From 5b9cd1bd87ad2c2440c6227e8aea7994e6c9b192 Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Mon, 28 Jun 2021 09:28:32 +0200 Subject: [PATCH 135/147] Improvements to media pickers/crop handling and URL generation (#10529) --- src/Umbraco.Core/Models/MediaWithCrops.cs | 79 ++++++++++- .../ImageCropperConfiguration.cs | 35 +++++ .../ValueConverters/ImageCropperValue.cs | 53 +++----- .../PropertyEditors/ImageCropperTest.cs | 14 +- .../ImageCropperTemplateCoreExtensions.cs | 126 ++++++++++++++---- .../ImageCropperTemplateExtensions.cs | 45 ++++++- .../MediaPicker3Configuration.cs | 40 ++++++ .../BlockListPropertyValueConverter.cs | 1 + .../MediaPickerWithCropsValueConverter.cs | 23 ++-- src/Umbraco.Web/UrlHelperRenderExtensions.cs | 111 +++++++++------ 10 files changed, 407 insertions(+), 120 deletions(-) diff --git a/src/Umbraco.Core/Models/MediaWithCrops.cs b/src/Umbraco.Core/Models/MediaWithCrops.cs index ef3205bd94..fefb4e6b80 100644 --- a/src/Umbraco.Core/Models/MediaWithCrops.cs +++ b/src/Umbraco.Core/Models/MediaWithCrops.cs @@ -1,15 +1,86 @@ +using System; using Umbraco.Core.Models.PublishedContent; using Umbraco.Core.PropertyEditors.ValueConverters; namespace Umbraco.Core.Models { /// - /// Model used in Razor Views for rendering + /// Represents a media item with local crops. /// - public class MediaWithCrops + /// + public class MediaWithCrops : PublishedContentWrapped { - public IPublishedContent MediaItem { get; set; } + /// + /// Gets the media item. + /// + /// + /// The media item. + /// + [Obsolete("This instance now implements IPublishedContent by wrapping the media item, use the extension methods directly on MediaWithCrops or use the Content property to get the media item instead.")] + public IPublishedContent MediaItem => Content; - public ImageCropperValue LocalCrops { get; set; } + /// + /// Gets the content/media item. + /// + /// + /// The content/media item. + /// + public IPublishedContent Content => Unwrap(); + + /// + /// Gets the local crops. + /// + /// + /// The local crops. + /// + public ImageCropperValue LocalCrops { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The content. + /// The local crops. + public MediaWithCrops(IPublishedContent content, ImageCropperValue localCrops) + : base(content) + { + LocalCrops = localCrops; + } + } + + /// + /// Represents a media item with local crops. + /// + /// The type of the media item. + /// + public class MediaWithCrops : MediaWithCrops + where T : IPublishedContent + { + /// + /// Gets the media item. + /// + /// + /// The media item. + /// + public new T Content { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The content. + /// The local crops. + public MediaWithCrops(T content, ImageCropperValue localCrops) + : base(content, localCrops) + { + Content = content; + } + + /// + /// Performs an implicit conversion from to . + /// + /// The media with crops. + /// + /// The result of the conversion. + /// + public static implicit operator T(MediaWithCrops mediaWithCrops) => mediaWithCrops.Content; } } diff --git a/src/Umbraco.Core/PropertyEditors/ImageCropperConfiguration.cs b/src/Umbraco.Core/PropertyEditors/ImageCropperConfiguration.cs index 2ce6e2ec04..855ec76a5a 100644 --- a/src/Umbraco.Core/PropertyEditors/ImageCropperConfiguration.cs +++ b/src/Umbraco.Core/PropertyEditors/ImageCropperConfiguration.cs @@ -1,4 +1,8 @@ using Newtonsoft.Json; +using System.Collections.Generic; +using System.Linq; +using Umbraco.Core.PropertyEditors.ValueConverters; +using static Umbraco.Core.PropertyEditors.ValueConverters.ImageCropperValue; namespace Umbraco.Core.PropertyEditors { @@ -22,4 +26,35 @@ namespace Umbraco.Core.PropertyEditors public int Height { get; set; } } } + + internal static class ImageCropperConfigurationExtensions + { + /// + /// Applies the configuration to ensure only valid crops are kept and have the correct width/height. + /// + /// The configuration. + public static void ApplyConfiguration(this ImageCropperValue imageCropperValue, ImageCropperConfiguration configuration) + { + var crops = new List(); + + var configuredCrops = configuration?.Crops; + if (configuredCrops != null) + { + foreach (var configuredCrop in configuredCrops) + { + var crop = imageCropperValue.Crops?.FirstOrDefault(x => x.Alias == configuredCrop.Alias); + + crops.Add(new ImageCropperCrop + { + Alias = configuredCrop.Alias, + Width = configuredCrop.Width, + Height = configuredCrop.Height, + Coordinates = crop?.Coordinates + }); + } + } + + imageCropperValue.Crops = crops; + } + } } diff --git a/src/Umbraco.Core/PropertyEditors/ValueConverters/ImageCropperValue.cs b/src/Umbraco.Core/PropertyEditors/ValueConverters/ImageCropperValue.cs index 2c6ec9b8aa..f2151778d9 100644 --- a/src/Umbraco.Core/PropertyEditors/ValueConverters/ImageCropperValue.cs +++ b/src/Umbraco.Core/PropertyEditors/ValueConverters/ImageCropperValue.cs @@ -140,7 +140,7 @@ namespace Umbraco.Core.PropertyEditors.ValueConverters /// Determines whether the value has a specified crop. /// public bool HasCrop(string alias) - => Crops.Any(x => x.Alias == alias); + => Crops != null && Crops.Any(x => x.Alias == alias); /// /// Determines whether the value has a source image. @@ -148,46 +148,35 @@ namespace Umbraco.Core.PropertyEditors.ValueConverters public bool HasImage() => !string.IsNullOrWhiteSpace(Src); - /// - /// Applies a configuration. - /// - /// Ensures that all crops defined in the configuration exists in the value. - internal void ApplyConfiguration(ImageCropperConfiguration configuration) + internal ImageCropperValue Merge(ImageCropperValue imageCropperValue) { - // merge the crop values - the alias + width + height comes from - // configuration, but each crop can store its own coordinates - - var configuredCrops = configuration?.Crops; - if (configuredCrops == null) return; - - //Use Crops if it's not null, otherwise create a new list var crops = Crops?.ToList() ?? new List(); - foreach (var configuredCrop in configuredCrops) + var incomingCrops = imageCropperValue?.Crops; + if (incomingCrops != null) { - var crop = crops.FirstOrDefault(x => x.Alias == configuredCrop.Alias); - if (crop != null) + foreach (var incomingCrop in incomingCrops) { - // found, apply the height & width - crop.Width = configuredCrop.Width; - crop.Height = configuredCrop.Height; - } - else - { - // not found, add - crops.Add(new ImageCropperCrop + var crop = crops.FirstOrDefault(x => x.Alias == incomingCrop.Alias); + if (crop == null) { - Alias = configuredCrop.Alias, - Width = configuredCrop.Width, - Height = configuredCrop.Height - }); + // Add incoming crop + crops.Add(incomingCrop); + } + else if (crop.Coordinates == null) + { + // Use incoming crop coordinates + crop.Coordinates = incomingCrop.Coordinates; + } } } - // assume we don't have to remove the crops in value, that - // are not part of configuration anymore? - - Crops = crops; + return new ImageCropperValue() + { + Src = !string.IsNullOrWhiteSpace(Src) ? Src : imageCropperValue?.Src, + Crops = crops, + FocalPoint = FocalPoint ?? imageCropperValue?.FocalPoint + }; } #region IEquatable diff --git a/src/Umbraco.Tests/PropertyEditors/ImageCropperTest.cs b/src/Umbraco.Tests/PropertyEditors/ImageCropperTest.cs index c5c2b4e61f..c40708770e 100644 --- a/src/Umbraco.Tests/PropertyEditors/ImageCropperTest.cs +++ b/src/Umbraco.Tests/PropertyEditors/ImageCropperTest.cs @@ -82,8 +82,20 @@ namespace Umbraco.Tests.PropertyEditors var mediaFileSystem = new MediaFileSystem(Mock.Of(), config, scheme, logger); + var imageCropperConfiguration = new ImageCropperConfiguration() + { + Crops = new[] + { + new ImageCropperConfiguration.Crop() + { + Alias = "thumb", + Width = 100, + Height = 100 + } + } + }; var dataTypeService = new TestObjects.TestDataTypeService( - new DataType(new ImageCropperPropertyEditor(Mock.Of(), mediaFileSystem, Mock.Of(), Mock.Of())) { Id = 1 }); + new DataType(new ImageCropperPropertyEditor(Mock.Of(), mediaFileSystem, Mock.Of(), Mock.Of())) { Id = 1, Configuration = imageCropperConfiguration }); var factory = new PublishedContentTypeFactory(Mock.Of(), new PropertyValueConverterCollection(Array.Empty()), dataTypeService); diff --git a/src/Umbraco.Web/ImageCropperTemplateCoreExtensions.cs b/src/Umbraco.Web/ImageCropperTemplateCoreExtensions.cs index 766cb1e99f..8773f1bb39 100644 --- a/src/Umbraco.Web/ImageCropperTemplateCoreExtensions.cs +++ b/src/Umbraco.Web/ImageCropperTemplateCoreExtensions.cs @@ -28,9 +28,30 @@ namespace Umbraco.Web return mediaItem.GetCropUrl(imageUrlGenerator, cropAlias: cropAlias, useCropDimensions: true); } + public static string GetCropUrl(this MediaWithCrops mediaWithCrops, string cropAlias, IImageUrlGenerator imageUrlGenerator) + { + return mediaWithCrops.GetCropUrl(imageUrlGenerator, cropAlias: cropAlias, useCropDimensions: true); + } + + [Obsolete("Use the GetCropUrl overload with the updated parameter order and note this implementation has changed to get the URL from the media item.")] public static string GetCropUrl(this IPublishedContent mediaItem, string cropAlias, IImageUrlGenerator imageUrlGenerator, ImageCropperValue imageCropperValue) { - return mediaItem.Url().GetCropUrl(imageUrlGenerator, imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); + return mediaItem.GetCropUrl(imageCropperValue, cropAlias, imageUrlGenerator); + } + + /// + /// Gets the crop URL by using only the specified . + /// + /// The media item. + /// The image cropper value. + /// The crop alias. + /// The image URL generator. + /// + /// The image crop URL. + /// + public static string GetCropUrl(this IPublishedContent mediaItem, ImageCropperValue imageCropperValue, string cropAlias, IImageUrlGenerator imageUrlGenerator) + { + return mediaItem.GetCropUrl(imageUrlGenerator, imageCropperValue, true, cropAlias: cropAlias, useCropDimensions: true); } /// @@ -53,6 +74,11 @@ namespace Umbraco.Web return mediaItem.GetCropUrl(imageUrlGenerator, propertyAlias: propertyAlias, cropAlias: cropAlias, useCropDimensions: true); } + public static string GetCropUrl(this MediaWithCrops mediaWithCrops, string propertyAlias, string cropAlias, IImageUrlGenerator imageUrlGenerator) + { + return mediaWithCrops.GetCropUrl(imageUrlGenerator, propertyAlias: propertyAlias, cropAlias: cropAlias, useCropDimensions: true); + } + /// /// Gets the ImageProcessor URL from the IPublishedContent item. /// @@ -123,7 +149,51 @@ namespace Umbraco.Web ImageCropRatioMode? ratioMode = null, bool upScale = true) { - if (mediaItem == null) throw new ArgumentNullException("mediaItem"); + return mediaItem.GetCropUrl(imageUrlGenerator, null, false, width, height, propertyAlias, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBuster, furtherOptions, ratioMode, upScale); + } + + public static string GetCropUrl( + this MediaWithCrops mediaWithCrops, + IImageUrlGenerator imageUrlGenerator, + int? width = null, + int? height = null, + string propertyAlias = Constants.Conventions.Media.File, + string cropAlias = null, + int? quality = null, + ImageCropMode? imageCropMode = null, + ImageCropAnchor? imageCropAnchor = null, + bool preferFocalPoint = false, + bool useCropDimensions = false, + bool cacheBuster = true, + string furtherOptions = null, + ImageCropRatioMode? ratioMode = null, + bool upScale = true) + { + if (mediaWithCrops == null) throw new ArgumentNullException(nameof(mediaWithCrops)); + + return mediaWithCrops.Content.GetCropUrl(imageUrlGenerator, mediaWithCrops.LocalCrops, false, width, height, propertyAlias, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBuster, furtherOptions, ratioMode, upScale); + } + + private static string GetCropUrl( + this IPublishedContent mediaItem, + IImageUrlGenerator imageUrlGenerator, + ImageCropperValue localCrops, + bool localCropsOnly, + int? width = null, + int? height = null, + string propertyAlias = Constants.Conventions.Media.File, + string cropAlias = null, + int? quality = null, + ImageCropMode? imageCropMode = null, + ImageCropAnchor? imageCropAnchor = null, + bool preferFocalPoint = false, + bool useCropDimensions = false, + bool cacheBuster = true, + string furtherOptions = null, + ImageCropRatioMode? ratioMode = null, + bool upScale = true) + { + if (mediaItem == null) throw new ArgumentNullException(nameof(mediaItem)); var cacheBusterValue = cacheBuster ? mediaItem.UpdateDate.ToFileTimeUtc().ToString(CultureInfo.InvariantCulture) : null; @@ -132,31 +202,38 @@ namespace Umbraco.Web var mediaItemUrl = mediaItem.MediaUrl(propertyAlias: propertyAlias); - //get the default obj from the value converter - var cropperValue = mediaItem.Value(propertyAlias); - - //is it strongly typed? - var stronglyTyped = cropperValue as ImageCropperValue; - if (stronglyTyped != null) + // Only get crops from media when required and used + if (localCropsOnly == false && (imageCropMode == ImageCropMode.Crop || imageCropMode == null)) { - return GetCropUrl( - mediaItemUrl, imageUrlGenerator, stronglyTyped, width, height, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, - cacheBusterValue, furtherOptions, ratioMode, upScale); + // Get the default cropper value from the value converter + var cropperValue = mediaItem.Value(propertyAlias); + + var mediaCrops = cropperValue as ImageCropperValue; + + if (mediaCrops == null && cropperValue is JObject jobj) + { + mediaCrops = jobj.ToObject(); + } + + if (mediaCrops == null && cropperValue is string imageCropperValue && + string.IsNullOrEmpty(imageCropperValue) == false && imageCropperValue.DetectIsJson()) + { + mediaCrops = imageCropperValue.DeserializeImageCropperValue(); + } + + // Merge crops + if (localCrops == null) + { + localCrops = mediaCrops; + } + else if (mediaCrops != null) + { + localCrops = localCrops.Merge(mediaCrops); + } } - //this shouldn't be the case but we'll check - var jobj = cropperValue as JObject; - if (jobj != null) - { - stronglyTyped = jobj.ToObject(); - return GetCropUrl( - mediaItemUrl, imageUrlGenerator, stronglyTyped, width, height, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, - cacheBusterValue, furtherOptions, ratioMode, upScale); - } - - //it's a single string return GetCropUrl( - mediaItemUrl, imageUrlGenerator, width, height, mediaItemUrl, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, + mediaItemUrl, imageUrlGenerator, localCrops, width, height, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, upScale); } @@ -237,6 +314,7 @@ namespace Umbraco.Web { cropDataSet = imageCropperValue.DeserializeImageCropperValue(); } + return GetCropUrl( imageUrl, imageUrlGenerator, cropDataSet, width, height, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, upScale); @@ -381,10 +459,10 @@ namespace Umbraco.Web return imageUrlGenerator.GetImageUrl(options); } + [Obsolete("Use GetCrop to merge local and media crops, get automatic cache buster value and have more parameters.")] public static string GetLocalCropUrl(this MediaWithCrops mediaWithCrops, string alias, IImageUrlGenerator imageUrlGenerator, string cacheBusterValue) { return mediaWithCrops.LocalCrops.Src + mediaWithCrops.LocalCrops.GetCropUrl(alias, imageUrlGenerator, cacheBusterValue: cacheBusterValue); - } } } diff --git a/src/Umbraco.Web/ImageCropperTemplateExtensions.cs b/src/Umbraco.Web/ImageCropperTemplateExtensions.cs index 51845946f1..d9218a8974 100644 --- a/src/Umbraco.Web/ImageCropperTemplateExtensions.cs +++ b/src/Umbraco.Web/ImageCropperTemplateExtensions.cs @@ -30,7 +30,21 @@ namespace Umbraco.Web /// public static string GetCropUrl(this IPublishedContent mediaItem, string cropAlias) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaItem, cropAlias, Current.ImageUrlGenerator); - public static string GetCropUrl(this IPublishedContent mediaItem, string cropAlias, ImageCropperValue imageCropperValue) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaItem, cropAlias, Current.ImageUrlGenerator, imageCropperValue); + public static string GetCropUrl(this MediaWithCrops mediaWithCrops, string cropAlias) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaWithCrops, cropAlias, Current.ImageUrlGenerator); + + [Obsolete("Use the GetCropUrl overload with the updated parameter order and note this implementation has changed to get the URL from the media item.")] + public static string GetCropUrl(this IPublishedContent mediaItem, string cropAlias, ImageCropperValue imageCropperValue) => mediaItem.GetCropUrl(imageCropperValue, cropAlias); + + /// + /// Gets the crop URL by using only the specified . + /// + /// The media item. + /// The image cropper value. + /// The crop alias. + /// + /// The image crop URL. + /// + public static string GetCropUrl(this IPublishedContent mediaItem, ImageCropperValue imageCropperValue, string cropAlias) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaItem, imageCropperValue, cropAlias, Current.ImageUrlGenerator); /// /// Gets the ImageProcessor URL by the crop alias using the specified property containing the image cropper Json data on the IPublishedContent item. @@ -49,6 +63,8 @@ namespace Umbraco.Web /// public static string GetCropUrl(this IPublishedContent mediaItem, string propertyAlias, string cropAlias) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaItem, propertyAlias, cropAlias, Current.ImageUrlGenerator); + public static string GetCropUrl(this MediaWithCrops mediaWithCrops, string propertyAlias, string cropAlias) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaWithCrops, propertyAlias, cropAlias, Current.ImageUrlGenerator); + /// /// Gets the ImageProcessor URL from the IPublishedContent item. /// @@ -118,12 +134,21 @@ namespace Umbraco.Web ImageCropRatioMode? ratioMode = null, bool upScale = true) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaItem, Current.ImageUrlGenerator, width, height, propertyAlias, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBuster, furtherOptions, ratioMode, upScale); - public static string GetLocalCropUrl(this MediaWithCrops mediaWithCrops, - string alias, - string cacheBusterValue = null) - => ImageCropperTemplateCoreExtensions.GetLocalCropUrl(mediaWithCrops, alias, Current.ImageUrlGenerator, cacheBusterValue); - - + public static string GetCropUrl( + this MediaWithCrops mediaWithCrops, + int? width = null, + int? height = null, + string propertyAlias = Constants.Conventions.Media.File, + string cropAlias = null, + int? quality = null, + ImageCropMode? imageCropMode = null, + ImageCropAnchor? imageCropAnchor = null, + bool preferFocalPoint = false, + bool useCropDimensions = false, + bool cacheBuster = true, + string furtherOptions = null, + ImageCropRatioMode? ratioMode = null, + bool upScale = true) => ImageCropperTemplateCoreExtensions.GetCropUrl(mediaWithCrops, Current.ImageUrlGenerator, width, height, propertyAlias, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBuster, furtherOptions, ratioMode, upScale); /// /// Gets the ImageProcessor URL from the image path. @@ -261,6 +286,12 @@ namespace Umbraco.Web ImageCropRatioMode? ratioMode = null, bool upScale = true) => ImageCropperTemplateCoreExtensions.GetCropUrl(imageUrl, Current.ImageUrlGenerator, cropDataSet, width, height, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, upScale); + [Obsolete("Use GetCrop to merge local and media crops, get automatic cache buster value and have more parameters.")] + public static string GetLocalCropUrl(this MediaWithCrops mediaWithCrops, + string alias, + string cacheBusterValue = null) + => ImageCropperTemplateCoreExtensions.GetLocalCropUrl(mediaWithCrops, alias, Current.ImageUrlGenerator, cacheBusterValue); + private static readonly JsonSerializerSettings ImageCropperValueJsonSerializerSettings = new JsonSerializerSettings { Culture = CultureInfo.InvariantCulture, diff --git a/src/Umbraco.Web/PropertyEditors/MediaPicker3Configuration.cs b/src/Umbraco.Web/PropertyEditors/MediaPicker3Configuration.cs index 4c3c6564a5..1a6a1cde0b 100644 --- a/src/Umbraco.Web/PropertyEditors/MediaPicker3Configuration.cs +++ b/src/Umbraco.Web/PropertyEditors/MediaPicker3Configuration.cs @@ -1,6 +1,10 @@ using Newtonsoft.Json; +using System.Collections.Generic; +using System.Linq; using Umbraco.Core; using Umbraco.Core.PropertyEditors; +using Umbraco.Core.PropertyEditors.ValueConverters; +using static Umbraco.Core.PropertyEditors.ValueConverters.ImageCropperValue; namespace Umbraco.Web.PropertyEditors { @@ -57,4 +61,40 @@ namespace Umbraco.Web.PropertyEditors public int Height { get; set; } } } + + internal static class MediaPicker3ConfigurationExtensions + { + /// + /// Applies the configuration to ensure only valid crops are kept and have the correct width/height. + /// + /// The configuration. + public static void ApplyConfiguration(this ImageCropperValue imageCropperValue, MediaPicker3Configuration configuration) + { + var crops = new List(); + + var configuredCrops = configuration?.Crops; + if (configuredCrops != null) + { + foreach (var configuredCrop in configuredCrops) + { + var crop = imageCropperValue.Crops?.FirstOrDefault(x => x.Alias == configuredCrop.Alias); + + crops.Add(new ImageCropperCrop + { + Alias = configuredCrop.Alias, + Width = configuredCrop.Width, + Height = configuredCrop.Height, + Coordinates = crop?.Coordinates + }); + } + } + + imageCropperValue.Crops = crops; + + if (configuration?.EnableLocalFocalPoint == false) + { + imageCropperValue.FocalPoint = null; + } + } + } } diff --git a/src/Umbraco.Web/PropertyEditors/ValueConverters/BlockListPropertyValueConverter.cs b/src/Umbraco.Web/PropertyEditors/ValueConverters/BlockListPropertyValueConverter.cs index f46c118174..5d216f2b4c 100644 --- a/src/Umbraco.Web/PropertyEditors/ValueConverters/BlockListPropertyValueConverter.cs +++ b/src/Umbraco.Web/PropertyEditors/ValueConverters/BlockListPropertyValueConverter.cs @@ -120,6 +120,7 @@ namespace Umbraco.Web.PropertyEditors.ValueConverters settingsData = null; } + // TODO: This should be optimized/cached, as calling Activator.CreateInstance is slow var layoutType = typeof(BlockListItem<,>).MakeGenericType(contentData.GetType(), settingsData?.GetType() ?? typeof(IPublishedElement)); var layoutRef = (BlockListItem)Activator.CreateInstance(layoutType, contentGuidUdi, contentData, settingGuidUdi, settingsData); diff --git a/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs b/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs index f2f055d698..17907e8546 100644 --- a/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs +++ b/src/Umbraco.Web/PropertyEditors/ValueConverters/MediaPickerWithCropsValueConverter.cs @@ -51,22 +51,27 @@ namespace Umbraco.Web.PropertyEditors.ValueConverters var mediaItems = new List(); var dtos = MediaPicker3PropertyEditor.MediaPicker3PropertyValueEditor.Deserialize(inter); + var configuration = propertyType.DataType.ConfigurationAs(); foreach (var dto in dtos) { var mediaItem = _publishedSnapshotAccessor.PublishedSnapshot.Media.GetById(preview, dto.MediaKey); if (mediaItem != null) { - mediaItems.Add(new MediaWithCrops + var localCrops = new ImageCropperValue { - MediaItem = mediaItem, - LocalCrops = new ImageCropperValue - { - Crops = dto.Crops, - FocalPoint = dto.FocalPoint, - Src = mediaItem.Url() - } - }); + Crops = dto.Crops, + FocalPoint = dto.FocalPoint, + Src = mediaItem.Url() + }; + + localCrops.ApplyConfiguration(configuration); + + // TODO: This should be optimized/cached, as calling Activator.CreateInstance is slow + var mediaWithCropsType = typeof(MediaWithCrops<>).MakeGenericType(mediaItem.GetType()); + var mediaWithCrops = (MediaWithCrops)Activator.CreateInstance(mediaWithCropsType, mediaItem, localCrops); + + mediaItems.Add(mediaWithCrops); if (!isMultiple) { diff --git a/src/Umbraco.Web/UrlHelperRenderExtensions.cs b/src/Umbraco.Web/UrlHelperRenderExtensions.cs index 592c88945b..2c547c841e 100644 --- a/src/Umbraco.Web/UrlHelperRenderExtensions.cs +++ b/src/Umbraco.Web/UrlHelperRenderExtensions.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Web; using System.Web.Mvc; using Umbraco.Core; +using Umbraco.Core.Models; using Umbraco.Core.Models.PublishedContent; using Umbraco.Core.PropertyEditors.ValueConverters; using Umbraco.Web.Composing; @@ -17,9 +18,10 @@ namespace Umbraco.Web /// public static class UrlHelperRenderExtensions { - private static readonly IHtmlString EmptyHtmlString = new HtmlString(string.Empty); + private static IHtmlString CreateHtmlString(string value, bool htmlEncode) => htmlEncode ? new HtmlString(HttpUtility.HtmlEncode(value)) : new HtmlString(value); + #region GetCropUrl /// @@ -42,7 +44,17 @@ namespace Umbraco.Web if (mediaItem == null) return EmptyHtmlString; var url = mediaItem.GetCropUrl(cropAlias: cropAlias, useCropDimensions: true); - return htmlEncode ? new HtmlString(HttpUtility.HtmlEncode(url)) : new HtmlString(url); + + return CreateHtmlString(url, htmlEncode); + } + + public static IHtmlString GetCropUrl(this UrlHelper urlHelper, MediaWithCrops mediaWithCrops, string cropAlias, bool htmlEncode = true) + { + if (mediaWithCrops == null) return EmptyHtmlString; + + var url = mediaWithCrops.GetCropUrl(cropAlias: cropAlias, useCropDimensions: true); + + return CreateHtmlString(url, htmlEncode); } /// @@ -70,7 +82,17 @@ namespace Umbraco.Web if (mediaItem == null) return EmptyHtmlString; var url = mediaItem.GetCropUrl(propertyAlias: propertyAlias, cropAlias: cropAlias, useCropDimensions: true); - return htmlEncode ? new HtmlString(HttpUtility.HtmlEncode(url)) : new HtmlString(url); + + return CreateHtmlString(url, htmlEncode); + } + + public static IHtmlString GetCropUrl(this UrlHelper urlHelper, MediaWithCrops mediaWithCrops, string propertyAlias, string cropAlias, bool htmlEncode = true) + { + if (mediaWithCrops == null) return EmptyHtmlString; + + var url = mediaWithCrops.GetCropUrl(propertyAlias: propertyAlias, cropAlias: cropAlias, useCropDimensions: true); + + return CreateHtmlString(url, htmlEncode); } /// @@ -150,10 +172,33 @@ namespace Umbraco.Web { if (mediaItem == null) return EmptyHtmlString; - var url = mediaItem.GetCropUrl(width, height, propertyAlias, cropAlias, quality, imageCropMode, - imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBuster, furtherOptions, ratioMode, - upScale); - return htmlEncode ? new HtmlString(HttpUtility.HtmlEncode(url)) : new HtmlString(url); + var url = mediaItem.GetCropUrl(width, height, propertyAlias, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBuster, furtherOptions, ratioMode, upScale); + + return CreateHtmlString(url, htmlEncode); + } + + public static IHtmlString GetCropUrl(this UrlHelper urlHelper, + MediaWithCrops mediaWithCrops, + int? width = null, + int? height = null, + string propertyAlias = Umbraco.Core.Constants.Conventions.Media.File, + string cropAlias = null, + int? quality = null, + ImageCropMode? imageCropMode = null, + ImageCropAnchor? imageCropAnchor = null, + bool preferFocalPoint = false, + bool useCropDimensions = false, + bool cacheBuster = true, + string furtherOptions = null, + ImageCropRatioMode? ratioMode = null, + bool upScale = true, + bool htmlEncode = true) + { + if (mediaWithCrops == null) return EmptyHtmlString; + + var url = mediaWithCrops.GetCropUrl(width, height, propertyAlias, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBuster, furtherOptions, ratioMode, upScale); + + return CreateHtmlString(url, htmlEncode); } /// @@ -231,10 +276,18 @@ namespace Umbraco.Web bool upScale = true, bool htmlEncode = true) { - var url = imageUrl.GetCropUrl(width, height, imageCropperValue, cropAlias, quality, imageCropMode, - imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, - upScale); - return htmlEncode ? new HtmlString(HttpUtility.HtmlEncode(url)) : new HtmlString(url); + var url = imageUrl.GetCropUrl(width, height, imageCropperValue, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, upScale); + + return CreateHtmlString(url, htmlEncode); + } + + public static IHtmlString GetCropUrl(this UrlHelper urlHelper, ImageCropperValue imageCropperValue, string cropAlias, bool htmlEncode = true) + { + if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; + + var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, cropAlias: cropAlias, useCropDimensions: true); + + return CreateHtmlString(url, htmlEncode); } public static IHtmlString GetCropUrl(this UrlHelper urlHelper, @@ -253,41 +306,13 @@ namespace Umbraco.Web bool upScale = true, bool htmlEncode = true) { - if (imageCropperValue == null) return EmptyHtmlString; + if (imageCropperValue == null || string.IsNullOrEmpty(imageCropperValue.Src)) return EmptyHtmlString; - var imageUrl = imageCropperValue.Src; - var url = imageUrl.GetCropUrl(imageCropperValue, width, height, cropAlias, quality, imageCropMode, - imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, - upScale); - return htmlEncode ? new HtmlString(HttpUtility.HtmlEncode(url)) : new HtmlString(url); + var url = imageCropperValue.Src.GetCropUrl(imageCropperValue, width, height, cropAlias, quality, imageCropMode, imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, upScale); + + return CreateHtmlString(url, htmlEncode); } - public static IHtmlString GetCropUrl(this UrlHelper urlHelper, - ImageCropperValue imageCropperValue, - string cropAlias, - int? width = null, - int? height = null, - int? quality = null, - ImageCropMode? imageCropMode = null, - ImageCropAnchor? imageCropAnchor = null, - bool preferFocalPoint = false, - bool useCropDimensions = true, - string cacheBusterValue = null, - string furtherOptions = null, - ImageCropRatioMode? ratioMode = null, - bool upScale = true, - bool htmlEncode = true) - { - if (imageCropperValue == null) return EmptyHtmlString; - - var imageUrl = imageCropperValue.Src; - var url = imageUrl.GetCropUrl(imageCropperValue, width, height, cropAlias, quality, imageCropMode, - imageCropAnchor, preferFocalPoint, useCropDimensions, cacheBusterValue, furtherOptions, ratioMode, - upScale); - return htmlEncode ? new HtmlString(HttpUtility.HtmlEncode(url)) : new HtmlString(url); - } - - #endregion /// From 83ee9b4699d3428dbb9342f7afbacb592df25d9c Mon Sep 17 00:00:00 2001 From: Mole Date: Mon, 28 Jun 2021 09:32:42 +0200 Subject: [PATCH 136/147] Reduce sql queries when rendering blocklists in the content panel (#10521) Co-authored-by: Nikolaj --- src/Umbraco.Core/Services/IContentService.cs | 7 +- .../Services/Implement/ContentService.cs | 31 +++++- src/Umbraco.Web/Editors/ContentController.cs | 94 ++++++++++++++----- .../Models/Mapping/ContentMapDefinition.cs | 48 ++++++++-- 4 files changed, 141 insertions(+), 39 deletions(-) diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index c291500533..e94d40184e 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -323,7 +323,7 @@ namespace Umbraco.Core.Services /// /// Empties the Recycle Bin by deleting all that resides in the bin /// - /// Optional Id of the User emptying the Recycle Bin + /// Optional Id of the User emptying the Recycle Bin OperationResult EmptyRecycleBin(int userId = Constants.Security.SuperUserId); /// @@ -499,6 +499,11 @@ namespace Umbraco.Core.Services /// IContent Create(string name, int parentId, string documentTypeAlias, int userId = Constants.Security.SuperUserId); + /// + /// Creates a document + /// + IContent Create(string name, int parentId, IContentType contentType, int userId = Constants.Security.SuperUserId); + /// /// Creates a document. /// diff --git a/src/Umbraco.Core/Services/Implement/ContentService.cs b/src/Umbraco.Core/Services/Implement/ContentService.cs index dc914ca3ac..af14d7fa69 100644 --- a/src/Umbraco.Core/Services/Implement/ContentService.cs +++ b/src/Umbraco.Core/Services/Implement/ContentService.cs @@ -188,11 +188,34 @@ namespace Umbraco.Core.Services.Implement // TODO: what about culture? var contentType = GetContentType(contentTypeAlias); - if (contentType == null) - throw new ArgumentException("No content type with that alias.", nameof(contentTypeAlias)); + return Create(name, parentId, contentType, userId); + } + + /// + /// Creates an object of a specified content type. + /// + /// This method simply returns a new, non-persisted, IContent without any identity. It + /// is intended as a shortcut to creating new content objects that does not invoke a save + /// operation against the database. + /// + /// The name of the content object. + /// The identifier of the parent, or -1. + /// The content type of the content + /// The optional id of the user creating the content. + /// The content object. + public IContent Create(string name, int parentId, IContentType contentType, + int userId = Constants.Security.SuperUserId) + { + if (contentType is null) + { + throw new ArgumentException("Content type must be specified", nameof(contentType)); + } + var parent = parentId > 0 ? GetById(parentId) : null; - if (parentId > 0 && parent == null) + if (parentId > 0 && parent is null) + { throw new ArgumentException("No content with that id.", nameof(parentId)); + } var content = new Content(name, parentId, contentType); using (var scope = ScopeProvider.CreateScope()) @@ -1088,7 +1111,7 @@ namespace Umbraco.Core.Services.Implement /// /// /// Business logic cases such: as unpublishing a mandatory culture, or unpublishing the last culture, checking for pending scheduled publishing, etc... is dealt with in this method. - /// There is quite a lot of cases to take into account along with logic that needs to deal with scheduled saving/publishing, branch saving/publishing, etc... + /// There is quite a lot of cases to take into account along with logic that needs to deal with scheduled saving/publishing, branch saving/publishing, etc... /// /// private PublishResult CommitDocumentChangesInternal(IScope scope, IContent content, diff --git a/src/Umbraco.Web/Editors/ContentController.cs b/src/Umbraco.Web/Editors/ContentController.cs index fc473d729d..9c384edd75 100644 --- a/src/Umbraco.Web/Editors/ContentController.cs +++ b/src/Umbraco.Web/Editors/ContentController.cs @@ -37,7 +37,7 @@ using Umbraco.Core.Models.Entities; using Umbraco.Core.Persistence; using Umbraco.Core.Security; using Umbraco.Web.Routing; -using Umbraco.Core.Collections; +using Umbraco.Core.Mapping; using Umbraco.Core.Scoping; namespace Umbraco.Web.Editors @@ -388,20 +388,65 @@ namespace Umbraco.Web.Editors } } - private ContentItemDisplay GetEmpty(IContentType contentType, int parentId) + private ContentItemDisplay CleanContentItemDisplay(ContentItemDisplay display) { - var emptyContent = Services.ContentService.Create("", parentId, contentType.Alias, Security.GetUserId().ResultOr(0)); - var mapped = MapToDisplay(emptyContent); // translate the content type name if applicable - mapped.ContentTypeName = Services.TextService.UmbracoDictionaryTranslate(mapped.ContentTypeName); + display.ContentTypeName = Services.TextService.UmbracoDictionaryTranslate(display.ContentTypeName); // if your user type doesn't have access to the Settings section it would not get this property mapped - if (mapped.DocumentType != null) - mapped.DocumentType.Name = Services.TextService.UmbracoDictionaryTranslate(mapped.DocumentType.Name); + if (display.DocumentType != null) + display.DocumentType.Name = Services.TextService.UmbracoDictionaryTranslate(display.DocumentType.Name); //remove the listview app if it exists - mapped.ContentApps = mapped.ContentApps.Where(x => x.Alias != "umbListView").ToList(); + display.ContentApps = display.ContentApps.Where(x => x.Alias != "umbListView").ToList(); + return display; + } - return mapped; + private ContentItemDisplay GetEmpty(IContentType contentType, int parentId) + { + var emptyContent = Services.ContentService.Create("", parentId, contentType, Security.GetUserId().ResultOr(0)); + var mapped = MapToDisplay(emptyContent); + return CleanContentItemDisplay(mapped); + } + + /// + /// Gets an empty for each content type in the IEnumerable, all with the same parent ID + /// + /// Will attempt to re-use the same permissions for every content as long as the path and user are the same + /// + /// + /// + private IEnumerable GetEmpties(IEnumerable contentTypes, int parentId) + { + var result = new List(); + var userId = Security.GetUserId().ResultOr(0); + var currentUser = Security.CurrentUser; + // We know that if the ID is less than 0 the parent is null. + // Since this is called with parent ID it's safe to assume that the parent is the same for all the content types. + var parent = parentId > 0 ? Services.ContentService.GetById(parentId) : null; + // Since the parent is the same and the path used to get permissions is based on the parent we only have to do it once + var path = parent == null ? "-1" : parent.Path; + var permissions = new Dictionary + { + [path] = Services.UserService.GetPermissionsForPath(currentUser, path) + }; + + foreach (var contentType in contentTypes) + { + var emptyContent = Services.ContentService.Create("", parentId, contentType, userId); + + var mapped = MapToDisplay(emptyContent, context => + { + // Since the permissions depend on current user and path, we add both of these to context as well, + // that way we can compare the path and current user when mapping, if they're the same just take permissions + // and skip getting them again, in theory they should always be the same, but better safe than sorry., + context.Items["Parent"] = parent; + context.Items["CurrentUser"] = currentUser; + context.Items["Permissions"] = permissions; + }); + result.Add(CleanContentItemDisplay(mapped)); + } + + return result; } /// @@ -412,22 +457,9 @@ namespace Umbraco.Web.Editors [OutgoingEditorModelEvent] public IDictionary GetEmptyByKeys([FromUri] Guid[] contentTypeKeys, [FromUri] int parentId) { - var result = new Dictionary(); - using var scope = _scopeProvider.CreateScope(autoComplete: true); var contentTypes = Services.ContentTypeService.GetAll(contentTypeKeys).ToList(); - - foreach (var contentType in contentTypes) - { - if (contentType is null) - { - throw new HttpResponseException(HttpStatusCode.NotFound); - } - - result.Add(contentType.Key, GetEmpty(contentType, parentId)); - } - - return result; + return GetEmpties(contentTypes, parentId).ToDictionary(x => x.ContentTypeKey); } [OutgoingEditorModelEvent] @@ -2274,12 +2306,22 @@ namespace Umbraco.Web.Editors /// /// /// - private ContentItemDisplay MapToDisplay(IContent content) - { - var display = Mapper.Map(content, context => + private ContentItemDisplay MapToDisplay(IContent content) => + MapToDisplay(content, context => { context.Items["CurrentUser"] = Security.CurrentUser; }); + + /// + /// Used to map an instance to a and ensuring AllowPreview is set correctly. + /// Also allows you to pass in an action for the mapper context where you can pass additional information on to the mapper. + /// + /// + /// + /// + private ContentItemDisplay MapToDisplay(IContent content, Action contextOptions) + { + var display = Mapper.Map(content, contextOptions); display.AllowPreview = display.AllowPreview && content.Trashed == false && content.ContentType.IsElement == false; return display; } diff --git a/src/Umbraco.Web/Models/Mapping/ContentMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/ContentMapDefinition.cs index 0360ee568b..cb2d0f1aba 100644 --- a/src/Umbraco.Web/Models/Mapping/ContentMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/ContentMapDefinition.cs @@ -86,7 +86,20 @@ namespace Umbraco.Web.Models.Mapping // Umbraco.Code.MapAll -AllowPreview -Errors -PersistedContent private void Map(IContent source, ContentItemDisplay target, MapperContext context) { - target.AllowedActions = GetActions(source); + // Both GetActions and DetermineIsChildOfListView use parent, so get it once here + // Parent might already be in context, so check there before using content service + IContent parent; + if (context.Items.TryGetValue("Parent", out var parentObj) && + parentObj is IContent typedParent) + { + parent = typedParent; + } + else + { + parent = _contentService.GetParent(source); + } + + target.AllowedActions = GetActions(source, parent, context); target.AllowedTemplates = GetAllowedTemplates(source); target.ContentApps = _commonMapper.GetContentApps(source); target.ContentTypeId = source.ContentType.Id; @@ -97,7 +110,7 @@ namespace Umbraco.Web.Models.Mapping target.Icon = source.ContentType.Icon; target.Id = source.Id; target.IsBlueprint = source.Blueprint; - target.IsChildOfListView = DetermineIsChildOfListView(source, context); + target.IsChildOfListView = DetermineIsChildOfListView(source, parent, context); target.IsContainer = source.ContentType.IsContainer; target.IsElement = source.ContentType.IsElement; target.Key = source.Key; @@ -156,7 +169,7 @@ namespace Umbraco.Web.Models.Mapping target.VariesByCulture = source.ContentType.VariesByCulture(); } - private IEnumerable GetActions(IContent source) + private IEnumerable GetActions(IContent source, IContent parent, MapperContext context) { var umbracoContext = _umbracoContextAccessor.UmbracoContext; @@ -169,10 +182,24 @@ namespace Umbraco.Web.Models.Mapping path = source.Path; else { - var parent = _contentService.GetById(source.ParentId); path = parent == null ? "-1" : parent.Path; } + // A bit of a mess, but we need to ensure that all the required values are here AND that they're the right type. + if (context.Items.TryGetValue("CurrentUser", out var userObject) && + context.Items.TryGetValue("Permissions", out var permissionsObject) && + userObject is IUser currentUser && + permissionsObject is Dictionary permissionsDict) + { + // If we already have permissions for a given path, + // and the current user is the same as was used to generate the permissions, return the stored permissions. + if (umbracoContext.Security.CurrentUser.Id == currentUser.Id && + permissionsDict.TryGetValue(path, out var permissions)) + { + return permissions.GetAllPermissions(); + } + } + // TODO: This is certainly not ideal usage here - perhaps the best way to deal with this in the future is // with the IUmbracoContextAccessor. In the meantime, if used outside of a web app this will throw a null // reference exception :( @@ -232,6 +259,7 @@ namespace Umbraco.Web.Models.Mapping /// Checks if the content item is a descendant of a list view /// /// + /// /// /// /// Returns true if the content item is a descendant of a list view and where the content is @@ -243,7 +271,7 @@ namespace Umbraco.Web.Models.Mapping /// false because the item is technically not being rendered as part of a list view but instead as a /// real tree node. If we didn't perform this check then tree syncing wouldn't work correctly. /// - private bool DetermineIsChildOfListView(IContent source, MapperContext context) + private bool DetermineIsChildOfListView(IContent source, IContent parent, MapperContext context) { var userStartNodes = Array.Empty(); @@ -258,12 +286,10 @@ namespace Umbraco.Web.Models.Mapping // return false if this is the user's actual start node, the node will be rendered in the tree // regardless of if it's a list view or not if (userStartNodes.Contains(source.Id)) - return false; + return false; } } - var parent = _contentService.GetParent(source); - if (parent == null) return false; @@ -297,6 +323,12 @@ namespace Umbraco.Web.Models.Mapping private IDictionary GetAllowedTemplates(IContent source) { + // Element types can't have templates, so no need to query to get the content type + if (source.ContentType.IsElement) + { + return new Dictionary(); + } + var contentType = _contentTypeService.Get(source.ContentTypeId); return contentType.AllowedTemplates From 31e9c875f5f2ec6d45f51a6ebc8770a5427e675b Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Mon, 28 Jun 2021 09:36:09 +0200 Subject: [PATCH 137/147] Bump version to 8.15.0-rc --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 3619dc1371..6ef4d6ce85 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.14.0")] -[assembly: AssemblyInformationalVersion("8.14.0")] +[assembly: AssemblyFileVersion("8.15.0")] +[assembly: AssemblyInformationalVersion("8.15.0-rc")] diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 180837bc80..d6b2374c8e 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -330,7 +330,7 @@ $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v14.0\Web\Microsoft.Web.Publishing.Tasks.dll $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v15.0\Web\Microsoft.Web.Publishing.Tasks.dll $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v16.0\Web\Microsoft.Web.Publishing.Tasks.dll - $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v17.0\Web\Microsoft.Web.Publishing.Tasks.dll + $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v17.0\Web\Microsoft.Web.Publishing.Tasks.dll @@ -348,9 +348,9 @@ False True - 8140 + 8150 / - http://localhost:8140 + http://localhost:8150 8131 / http://localhost:8131 @@ -436,4 +436,4 @@ - + \ No newline at end of file From 897cb11f005c706f4b7f80907e4395c87b3a4fc8 Mon Sep 17 00:00:00 2001 From: Elitsa Marinovska <21998037+elit0451@users.noreply.github.com> Date: Tue, 29 Jun 2021 09:55:02 +0200 Subject: [PATCH 138/147] Nested content optimizations (#10236) * Reducing the large amount of objects allocated and make use of the attemped dictionary cache * Reverting optimization by passing data from a higher level - not really possible to achieve what we wanted and adding faster way to lookup value editors --- .../PropertyEditors/NestedContentPropertyEditor.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs index 1047c4317d..b0eeacacd9 100644 --- a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs @@ -156,6 +156,7 @@ namespace Umbraco.Web.PropertyEditors public override object ToEditor(Property property, IDataTypeService dataTypeService, string culture = null, string segment = null) { var val = property.GetValue(culture, segment); + var valEditors = new Dictionary(); var rows = _nestedContentValues.GetPropertyValues(val); @@ -184,8 +185,15 @@ namespace Umbraco.Web.PropertyEditors continue; } - var tempConfig = dataTypeService.GetDataType(prop.Value.PropertyType.DataTypeId).Configuration; - var valEditor = propEditor.GetValueEditor(tempConfig); + var dataTypeId = prop.Value.PropertyType.DataTypeId; + if (!valEditors.TryGetValue(dataTypeId, out var valEditor)) + { + var tempConfig = dataTypeService.GetDataType(dataTypeId).Configuration; + valEditor = propEditor.GetValueEditor(tempConfig); + + valEditors.Add(dataTypeId, valEditor); + } + var convValue = valEditor.ToEditor(tempProp, dataTypeService); // update the raw value since this is what will get serialized out From 3e89e468406f4eb3ad33522bcad8a357fc7bff14 Mon Sep 17 00:00:00 2001 From: Elitsa Marinovska <21998037+elit0451@users.noreply.github.com> Date: Tue, 29 Jun 2021 09:55:02 +0200 Subject: [PATCH 139/147] Nested content optimizations (#10236) * Reducing the large amount of objects allocated and make use of the attemped dictionary cache * Reverting optimization by passing data from a higher level - not really possible to achieve what we wanted and adding faster way to lookup value editors (cherry picked from commit 897cb11f005c706f4b7f80907e4395c87b3a4fc8) --- .../PropertyEditors/NestedContentPropertyEditor.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs index 1047c4317d..b0eeacacd9 100644 --- a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs @@ -156,6 +156,7 @@ namespace Umbraco.Web.PropertyEditors public override object ToEditor(Property property, IDataTypeService dataTypeService, string culture = null, string segment = null) { var val = property.GetValue(culture, segment); + var valEditors = new Dictionary(); var rows = _nestedContentValues.GetPropertyValues(val); @@ -184,8 +185,15 @@ namespace Umbraco.Web.PropertyEditors continue; } - var tempConfig = dataTypeService.GetDataType(prop.Value.PropertyType.DataTypeId).Configuration; - var valEditor = propEditor.GetValueEditor(tempConfig); + var dataTypeId = prop.Value.PropertyType.DataTypeId; + if (!valEditors.TryGetValue(dataTypeId, out var valEditor)) + { + var tempConfig = dataTypeService.GetDataType(dataTypeId).Configuration; + valEditor = propEditor.GetValueEditor(tempConfig); + + valEditors.Add(dataTypeId, valEditor); + } + var convValue = valEditor.ToEditor(tempProp, dataTypeService); // update the raw value since this is what will get serialized out From 917c89cde7825ea92bbcc0f1535b7fd00e7b0754 Mon Sep 17 00:00:00 2001 From: Elitsa Marinovska <21998037+elit0451@users.noreply.github.com> Date: Tue, 29 Jun 2021 11:46:07 +0200 Subject: [PATCH 140/147] Ability to configure a remote URL for displaying different content on the content dashboard (#10257) * Enabling configuration of remote URL for fetching content on the content dashboard and adding additional params indicating whether solution is running on Umbraco Cloud * Determining if Umbraco is running on Umbraco Cloud by checking whether Umbraco.Deploy.Cloud.dll is present and making its value more accessible for other contexts. Passing a hosting parameter as string instead of boolean * Changing from Unknown to OnPremises * Updated content dashboard to use hard-coded URL but path derived from configuration. Co-authored-by: Andy Butland --- src/Umbraco.Core/Constants-AppSettings.cs | 7 ++++++- .../Dashboards/ContentDashboardSettings.cs | 10 ++++++++++ .../Dashboards/IContentDashboardSettings.cs | 6 ++++++ src/Umbraco.Web.UI/web.Template.config | 1 + .../Editors/DashboardController.cs | 19 ++++++++++++++++--- 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Core/Constants-AppSettings.cs b/src/Umbraco.Core/Constants-AppSettings.cs index 1f096ab9f9..99ea26b4d6 100644 --- a/src/Umbraco.Core/Constants-AppSettings.cs +++ b/src/Umbraco.Core/Constants-AppSettings.cs @@ -109,12 +109,17 @@ namespace Umbraco.Core /// A true or false indicating whether umbraco should force a secure (https) connection to the backoffice. /// public const string UseHttps = "Umbraco.Core.UseHttps"; - + /// /// A true/false value indicating whether the content dashboard should be visible for all user groups. /// public const string AllowContentDashboardAccessToAllUsers = "Umbraco.Core.AllowContentDashboardAccessToAllUsers"; + /// + /// The path to use when constructing the URL for retrieving data for the content dashboard. + /// + public const string ContentDashboardPath = "Umbraco.Core.ContentDashboardPath"; + /// /// TODO: FILL ME IN /// diff --git a/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs b/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs index f8fb5c7b06..b370f93eca 100644 --- a/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs +++ b/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs @@ -4,6 +4,7 @@ namespace Umbraco.Core.Dashboards { public class ContentDashboardSettings: IContentDashboardSettings { + private const string DefaultContentDashboardPath = "cms"; /// /// Gets a value indicating whether the content dashboard should be available to all users. @@ -20,5 +21,14 @@ namespace Umbraco.Core.Dashboards return value; } } + + /// + /// Gets the path to use when constructing the URL for retrieving data for the content dashboard. + /// + /// The URL path. + public string ContentDashboardPath => + ConfigurationManager.AppSettings.ContainsKey(Constants.AppSettings.ContentDashboardPath) + ? ConfigurationManager.AppSettings[Constants.AppSettings.ContentDashboardPath] + : DefaultContentDashboardPath; } } diff --git a/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs b/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs index 862a28b90e..f5c4e3da78 100644 --- a/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs +++ b/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs @@ -10,5 +10,11 @@ /// and the default access rules for that dashboard will be in use. /// bool AllowContentDashboardAccessToAllUsers { get; } + + /// + /// Gets the path to use when constructing the URL for retrieving data for the content dashboard. + /// + /// The URL path. + string ContentDashboardPath { get; } } } diff --git a/src/Umbraco.Web.UI/web.Template.config b/src/Umbraco.Web.UI/web.Template.config index 8c4b421839..c6b1eb686c 100644 --- a/src/Umbraco.Web.UI/web.Template.config +++ b/src/Umbraco.Web.UI/web.Template.config @@ -38,6 +38,7 @@ + diff --git a/src/Umbraco.Web/Editors/DashboardController.cs b/src/Umbraco.Web/Editors/DashboardController.cs index 97db8818f2..aa9691e3dd 100644 --- a/src/Umbraco.Web/Editors/DashboardController.cs +++ b/src/Umbraco.Web/Editors/DashboardController.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Services; using Umbraco.Core.Dashboards; +using Umbraco.Core.IO; using Umbraco.Core.Models; using Umbraco.Web.Services; @@ -32,14 +33,19 @@ namespace Umbraco.Web.Editors public class DashboardController : UmbracoApiController { private readonly IDashboardService _dashboardService; + private readonly IContentDashboardSettings _dashboardSettings; /// /// Initializes a new instance of the with all its dependencies. /// - public DashboardController(IGlobalSettings globalSettings, IUmbracoContextAccessor umbracoContextAccessor, ISqlContext sqlContext, ServiceContext services, AppCaches appCaches, IProfilingLogger logger, IRuntimeState runtimeState, IDashboardService dashboardService, UmbracoHelper umbracoHelper) + public DashboardController(IGlobalSettings globalSettings, IUmbracoContextAccessor umbracoContextAccessor, + ISqlContext sqlContext, ServiceContext services, AppCaches appCaches, IProfilingLogger logger, + IRuntimeState runtimeState, IDashboardService dashboardService, UmbracoHelper umbracoHelper, + IContentDashboardSettings dashboardSettings) : base(globalSettings, umbracoContextAccessor, sqlContext, services, appCaches, logger, runtimeState, umbracoHelper) { _dashboardService = dashboardService; + _dashboardSettings = dashboardSettings; } //we have just one instance of HttpClient shared for the entire application @@ -47,7 +53,7 @@ namespace Umbraco.Web.Editors //we have baseurl as a param to make previewing easier, so we can test with a dev domain from client side [ValidateAngularAntiForgeryToken] - public async Task GetRemoteDashboardContent(string section, string baseUrl = "https://dashboard.umbraco.org/") + public async Task GetRemoteDashboardContent(string section, string baseUrl = "https://dashboard.umbraco.com/") { var user = Security.CurrentUser; var allowedSections = string.Join(",", user.AllowedSections); @@ -55,7 +61,14 @@ namespace Umbraco.Web.Editors var version = UmbracoVersion.SemanticVersion.ToSemanticString(); var isAdmin = user.IsAdmin(); - var url = string.Format(baseUrl + "{0}?section={0}&allowed={1}&lang={2}&version={3}&admin={4}", section, allowedSections, language, version, isAdmin); + var url = string.Format("{0}{1}?section={2}&allowed={3}&lang={4}&version={5}&admin={6}", + baseUrl, + _dashboardSettings.ContentDashboardPath, + section, + allowedSections, + language, + version, + isAdmin); var key = "umbraco-dynamic-dashboard-" + language + allowedSections.Replace(",", "-") + section; var content = AppCaches.RuntimeCache.GetCacheItem(key); From 4f46edbaa9dd69cd3870b227e376cabe45de67cd Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Mon, 28 Jun 2021 09:36:09 +0200 Subject: [PATCH 141/147] Bump version to 8.15.0-rc --- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index 3619dc1371..6ef4d6ce85 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -18,5 +18,5 @@ using System.Resources; [assembly: AssemblyVersion("8.0.0")] // these are FYI and changed automatically -[assembly: AssemblyFileVersion("8.14.0")] -[assembly: AssemblyInformationalVersion("8.14.0")] +[assembly: AssemblyFileVersion("8.15.0")] +[assembly: AssemblyInformationalVersion("8.15.0-rc")] diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 180837bc80..d6b2374c8e 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -330,7 +330,7 @@ $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v14.0\Web\Microsoft.Web.Publishing.Tasks.dll $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v15.0\Web\Microsoft.Web.Publishing.Tasks.dll $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v16.0\Web\Microsoft.Web.Publishing.Tasks.dll - $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v17.0\Web\Microsoft.Web.Publishing.Tasks.dll + $(MSBuildExtensionsPath)\Microsoft\VisualStudio\v17.0\Web\Microsoft.Web.Publishing.Tasks.dll @@ -348,9 +348,9 @@ False True - 8140 + 8150 / - http://localhost:8140 + http://localhost:8150 8131 / http://localhost:8131 @@ -436,4 +436,4 @@ - + \ No newline at end of file From 861fc40572c82564a5f925bdda36323d94033f55 Mon Sep 17 00:00:00 2001 From: Elitsa Marinovska <21998037+elit0451@users.noreply.github.com> Date: Tue, 29 Jun 2021 11:46:07 +0200 Subject: [PATCH 142/147] Ability to configure a remote URL for displaying different content on the content dashboard (#10257) * Enabling configuration of remote URL for fetching content on the content dashboard and adding additional params indicating whether solution is running on Umbraco Cloud * Determining if Umbraco is running on Umbraco Cloud by checking whether Umbraco.Deploy.Cloud.dll is present and making its value more accessible for other contexts. Passing a hosting parameter as string instead of boolean * Changing from Unknown to OnPremises * Updated content dashboard to use hard-coded URL but path derived from configuration. Co-authored-by: Andy Butland (cherry picked from commit 917c89cde7825ea92bbcc0f1535b7fd00e7b0754) --- src/Umbraco.Core/Constants-AppSettings.cs | 7 ++++++- .../Dashboards/ContentDashboardSettings.cs | 10 ++++++++++ .../Dashboards/IContentDashboardSettings.cs | 6 ++++++ src/Umbraco.Web.UI/web.Template.config | 1 + .../Editors/DashboardController.cs | 19 ++++++++++++++++--- 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/Umbraco.Core/Constants-AppSettings.cs b/src/Umbraco.Core/Constants-AppSettings.cs index 1f096ab9f9..99ea26b4d6 100644 --- a/src/Umbraco.Core/Constants-AppSettings.cs +++ b/src/Umbraco.Core/Constants-AppSettings.cs @@ -109,12 +109,17 @@ namespace Umbraco.Core /// A true or false indicating whether umbraco should force a secure (https) connection to the backoffice. /// public const string UseHttps = "Umbraco.Core.UseHttps"; - + /// /// A true/false value indicating whether the content dashboard should be visible for all user groups. /// public const string AllowContentDashboardAccessToAllUsers = "Umbraco.Core.AllowContentDashboardAccessToAllUsers"; + /// + /// The path to use when constructing the URL for retrieving data for the content dashboard. + /// + public const string ContentDashboardPath = "Umbraco.Core.ContentDashboardPath"; + /// /// TODO: FILL ME IN /// diff --git a/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs b/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs index f8fb5c7b06..b370f93eca 100644 --- a/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs +++ b/src/Umbraco.Core/Dashboards/ContentDashboardSettings.cs @@ -4,6 +4,7 @@ namespace Umbraco.Core.Dashboards { public class ContentDashboardSettings: IContentDashboardSettings { + private const string DefaultContentDashboardPath = "cms"; /// /// Gets a value indicating whether the content dashboard should be available to all users. @@ -20,5 +21,14 @@ namespace Umbraco.Core.Dashboards return value; } } + + /// + /// Gets the path to use when constructing the URL for retrieving data for the content dashboard. + /// + /// The URL path. + public string ContentDashboardPath => + ConfigurationManager.AppSettings.ContainsKey(Constants.AppSettings.ContentDashboardPath) + ? ConfigurationManager.AppSettings[Constants.AppSettings.ContentDashboardPath] + : DefaultContentDashboardPath; } } diff --git a/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs b/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs index 862a28b90e..f5c4e3da78 100644 --- a/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs +++ b/src/Umbraco.Core/Dashboards/IContentDashboardSettings.cs @@ -10,5 +10,11 @@ /// and the default access rules for that dashboard will be in use. /// bool AllowContentDashboardAccessToAllUsers { get; } + + /// + /// Gets the path to use when constructing the URL for retrieving data for the content dashboard. + /// + /// The URL path. + string ContentDashboardPath { get; } } } diff --git a/src/Umbraco.Web.UI/web.Template.config b/src/Umbraco.Web.UI/web.Template.config index 8c4b421839..c6b1eb686c 100644 --- a/src/Umbraco.Web.UI/web.Template.config +++ b/src/Umbraco.Web.UI/web.Template.config @@ -38,6 +38,7 @@ + diff --git a/src/Umbraco.Web/Editors/DashboardController.cs b/src/Umbraco.Web/Editors/DashboardController.cs index 97db8818f2..aa9691e3dd 100644 --- a/src/Umbraco.Web/Editors/DashboardController.cs +++ b/src/Umbraco.Web/Editors/DashboardController.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Services; using Umbraco.Core.Dashboards; +using Umbraco.Core.IO; using Umbraco.Core.Models; using Umbraco.Web.Services; @@ -32,14 +33,19 @@ namespace Umbraco.Web.Editors public class DashboardController : UmbracoApiController { private readonly IDashboardService _dashboardService; + private readonly IContentDashboardSettings _dashboardSettings; /// /// Initializes a new instance of the with all its dependencies. /// - public DashboardController(IGlobalSettings globalSettings, IUmbracoContextAccessor umbracoContextAccessor, ISqlContext sqlContext, ServiceContext services, AppCaches appCaches, IProfilingLogger logger, IRuntimeState runtimeState, IDashboardService dashboardService, UmbracoHelper umbracoHelper) + public DashboardController(IGlobalSettings globalSettings, IUmbracoContextAccessor umbracoContextAccessor, + ISqlContext sqlContext, ServiceContext services, AppCaches appCaches, IProfilingLogger logger, + IRuntimeState runtimeState, IDashboardService dashboardService, UmbracoHelper umbracoHelper, + IContentDashboardSettings dashboardSettings) : base(globalSettings, umbracoContextAccessor, sqlContext, services, appCaches, logger, runtimeState, umbracoHelper) { _dashboardService = dashboardService; + _dashboardSettings = dashboardSettings; } //we have just one instance of HttpClient shared for the entire application @@ -47,7 +53,7 @@ namespace Umbraco.Web.Editors //we have baseurl as a param to make previewing easier, so we can test with a dev domain from client side [ValidateAngularAntiForgeryToken] - public async Task GetRemoteDashboardContent(string section, string baseUrl = "https://dashboard.umbraco.org/") + public async Task GetRemoteDashboardContent(string section, string baseUrl = "https://dashboard.umbraco.com/") { var user = Security.CurrentUser; var allowedSections = string.Join(",", user.AllowedSections); @@ -55,7 +61,14 @@ namespace Umbraco.Web.Editors var version = UmbracoVersion.SemanticVersion.ToSemanticString(); var isAdmin = user.IsAdmin(); - var url = string.Format(baseUrl + "{0}?section={0}&allowed={1}&lang={2}&version={3}&admin={4}", section, allowedSections, language, version, isAdmin); + var url = string.Format("{0}{1}?section={2}&allowed={3}&lang={4}&version={5}&admin={6}", + baseUrl, + _dashboardSettings.ContentDashboardPath, + section, + allowedSections, + language, + version, + isAdmin); var key = "umbraco-dynamic-dashboard-" + language + allowedSections.Replace(",", "-") + section; var content = AppCaches.RuntimeCache.GetCacheItem(key); From dc27a31a6fe35a885eab9079d2e002fc53da6391 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 29 Jun 2021 13:35:22 +0200 Subject: [PATCH 143/147] Fix creating / editing datatype in infinite editing mode, related to #10163 --- .../src/common/services/datatypehelper.service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/services/datatypehelper.service.js b/src/Umbraco.Web.UI.Client/src/common/services/datatypehelper.service.js index f4317b51b7..7aff3faaaf 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/datatypehelper.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/datatypehelper.service.js @@ -14,7 +14,7 @@ function dataTypeHelper() { for (var i = 0; i < preVals.length; i++) { preValues.push({ hideLabel: preVals[i].hideLabel, - alias: preVals[i].key, + alias: preVals[i].key != undefined ? preVals[i].key : preVals[i].alias, description: preVals[i].description, label: preVals[i].label, view: preVals[i].view, From c1a4e07e87df428d71314174854c954749842144 Mon Sep 17 00:00:00 2001 From: Mads Rasmussen Date: Wed, 30 Jun 2021 15:41:07 +0200 Subject: [PATCH 144/147] #10520 Correct mandatory markers in nested content (#10563) --- .../src/less/components/umb-nested-content.less | 11 ----------- .../src/views/components/property/umb-property.html | 2 +- .../nestedcontent/nestedcontent.editor.html | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less b/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less index 9dd40a4386..f6c252cc4d 100644 --- a/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less +++ b/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less @@ -15,17 +15,6 @@ pointer-events: none; } -.umb-nested-content--mandatory { - /* - yeah so this is a pain, but we must be super specific in targeting the mandatory property labels, - otherwise all properties within a reqired, nested, nested content property will all appear mandatory - */ - .umb-property > ng-form > .control-group > .umb-el-wrap > .control-header label:after { - content: '*'; - color: @red; - } -} - .umb-nested-content-overlay { position: absolute; top: 0; diff --git a/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html b/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html index 5b8e6d8f04..51a6f65c9c 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html @@ -16,7 +16,7 @@ {{vm.property.label}} - + * diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html index 125e920fe6..e14bd03291 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html @@ -4,7 +4,7 @@ From 0b3c4a726c9d97480a804aaf6fdeb317ee8b59cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Wed, 30 Jun 2021 16:27:53 +0200 Subject: [PATCH 145/147] #10520 ensure property culture in Nested Content (#10562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Niels Lyngsø --- .../nestedcontent/nestedcontent.controller.js | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js index 68d375360e..446fb8c076 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js @@ -190,7 +190,7 @@ }; vm.openNodeTypePicker = function ($event) { - + if (vm.nodes.length >= vm.maxItems) { return; } @@ -537,15 +537,20 @@ if (tab) { scaffold.variants[0].tabs.push(tab); - tab.properties.forEach(function (property) { + tab.properties.forEach( + function (property) { if (_.find(notSupported, function (x) { return x === property.editor; })) { property.notSupported = true; // TODO: Not supported message to be replaced with 'content_nestedContentEditorNotSupported' dictionary key. Currently not possible due to async/timing quirk. property.notSupportedMessage = "Property " + property.label + " uses editor " + property.editor + " which is not supported by Nested Content."; } - }); + } + ); } + // Ensure Culture Data for Complex Validation. + ensureCultureData(scaffold); + // Store the scaffold object vm.scaffolds.push(scaffold); } @@ -558,6 +563,29 @@ }); }); + /** + * Ensure that the containing content variant language and current property culture is transferred along + * to the scaffolded content object representing this block. + * This is required for validation along with ensuring that the umb-property inheritance is constantly maintained. + * @param {any} content + */ + function ensureCultureData(content) { + + if (!content || !vm.umbVariantContent || !vm.umbProperty) return; + + if (vm.umbVariantContent.editor.content.language) { + // set the scaffolded content's language to the language of the current editor + content.language = vm.umbVariantContent.editor.content.language; + } + // currently we only ever deal with invariant content for blocks so there's only one + content.variants[0].tabs.forEach(tab => { + tab.properties.forEach(prop => { + // set the scaffolded property to the culture of the containing property + prop.culture = vm.umbProperty.property.culture; + }); + }); + } + var initIfAllScaffoldsHaveLoaded = function () { // Initialize when all scaffolds have loaded if (model.config.contentTypes.length === scaffoldsLoaded) { From 3e226384c3eb7940cad6357f3dbab4411c1a5bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niels=20Lyngs=C3=B8?= Date: Wed, 30 Jun 2021 16:27:53 +0200 Subject: [PATCH 146/147] #10520 ensure property culture in Nested Content (#10562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Niels Lyngsø (cherry picked from commit 0b3c4a726c9d97480a804aaf6fdeb317ee8b59cf) --- .../nestedcontent/nestedcontent.controller.js | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js index 68d375360e..446fb8c076 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.controller.js @@ -190,7 +190,7 @@ }; vm.openNodeTypePicker = function ($event) { - + if (vm.nodes.length >= vm.maxItems) { return; } @@ -537,15 +537,20 @@ if (tab) { scaffold.variants[0].tabs.push(tab); - tab.properties.forEach(function (property) { + tab.properties.forEach( + function (property) { if (_.find(notSupported, function (x) { return x === property.editor; })) { property.notSupported = true; // TODO: Not supported message to be replaced with 'content_nestedContentEditorNotSupported' dictionary key. Currently not possible due to async/timing quirk. property.notSupportedMessage = "Property " + property.label + " uses editor " + property.editor + " which is not supported by Nested Content."; } - }); + } + ); } + // Ensure Culture Data for Complex Validation. + ensureCultureData(scaffold); + // Store the scaffold object vm.scaffolds.push(scaffold); } @@ -558,6 +563,29 @@ }); }); + /** + * Ensure that the containing content variant language and current property culture is transferred along + * to the scaffolded content object representing this block. + * This is required for validation along with ensuring that the umb-property inheritance is constantly maintained. + * @param {any} content + */ + function ensureCultureData(content) { + + if (!content || !vm.umbVariantContent || !vm.umbProperty) return; + + if (vm.umbVariantContent.editor.content.language) { + // set the scaffolded content's language to the language of the current editor + content.language = vm.umbVariantContent.editor.content.language; + } + // currently we only ever deal with invariant content for blocks so there's only one + content.variants[0].tabs.forEach(tab => { + tab.properties.forEach(prop => { + // set the scaffolded property to the culture of the containing property + prop.culture = vm.umbProperty.property.culture; + }); + }); + } + var initIfAllScaffoldsHaveLoaded = function () { // Initialize when all scaffolds have loaded if (model.config.contentTypes.length === scaffoldsLoaded) { From fc108511d28153495353abd8527032a78170df8f Mon Sep 17 00:00:00 2001 From: Mads Rasmussen Date: Wed, 30 Jun 2021 15:41:07 +0200 Subject: [PATCH 147/147] #10520 Correct mandatory markers in nested content (#10563) (cherry picked from commit c1a4e07e87df428d71314174854c954749842144) --- .../src/less/components/umb-nested-content.less | 11 ----------- .../src/views/components/property/umb-property.html | 2 +- .../nestedcontent/nestedcontent.editor.html | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less b/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less index 9dd40a4386..f6c252cc4d 100644 --- a/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less +++ b/src/Umbraco.Web.UI.Client/src/less/components/umb-nested-content.less @@ -15,17 +15,6 @@ pointer-events: none; } -.umb-nested-content--mandatory { - /* - yeah so this is a pain, but we must be super specific in targeting the mandatory property labels, - otherwise all properties within a reqired, nested, nested content property will all appear mandatory - */ - .umb-property > ng-form > .control-group > .umb-el-wrap > .control-header label:after { - content: '*'; - color: @red; - } -} - .umb-nested-content-overlay { position: absolute; top: 0; diff --git a/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html b/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html index 5b8e6d8f04..51a6f65c9c 100644 --- a/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html +++ b/src/Umbraco.Web.UI.Client/src/views/components/property/umb-property.html @@ -16,7 +16,7 @@ {{vm.property.label}} - + * diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html index 125e920fe6..e14bd03291 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/nestedcontent/nestedcontent.editor.html @@ -4,7 +4,7 @@