From 67a9b5bb9770908f431760f88783b00d64839fae Mon Sep 17 00:00:00 2001 From: Shannon Date: Fri, 25 Sep 2020 00:32:11 +1000 Subject: [PATCH] Refactors the serialization of the content data that is stored in the nucache table. This had to change because we need to resolve content type data in order to check if the property should be compressed and we cannot do that data lookup while the data is being processed since we get an open data reader exception. This is fixed now by using a serializer factory instead so the Create method can do any initialization needed prior to running any serialization operation. Renames a few things so we dont have ContentNested (whatever that meant ) --- ...StoragePropertyEditorCompressionOptions.cs | 45 +++++----- .../IPropertyCompressionOptions.cs | 8 +- .../ContentSerializationTests.cs | 20 ++--- .../PublishedContent/NuCacheChildrenTests.cs | 6 +- .../PublishedContent/NuCacheTests.cs | 6 +- .../Scoping/ScopedNuCacheTests.cs | 6 +- .../ContentTypeServiceVariantsTests.cs | 6 +- .../Editors/NuCacheStatusController.cs | 5 ++ .../BlockListPropertyEditor.cs | 1 + .../PropertyEditors/GridPropertyEditor.cs | 1 + .../PropertyEditors/MarkdownPropertyEditor.cs | 1 + .../NestedContentPropertyEditor.cs | 1 + .../NoopPropertyCompressionOptions.cs | 12 --- .../PropertyEditors/RichTextPropertyEditor.cs | 1 + .../PropertyEditors/TextAreaPropertyEditor.cs | 1 + .../PublishedCache/NuCache/ContentNodeKit.cs | 1 + ...NestedData.cs => ContentCacheDataModel.cs} | 4 +- .../ContentCacheDataSerializationResult.cs | 47 +++++++++++ .../ContentCacheDataSerializerEntityType.cs | 13 +++ .../NuCache/DataSource/DatabaseDataSource.cs | 82 +++++++++++-------- .../DataSource/IContentCacheDataSerializer.cs | 18 ++++ .../IContentCacheDataSerializerFactory.cs | 16 ++++ .../IContentNestedDataSerializer.cs | 22 ----- .../JsonContentNestedDataSerializer.cs | 21 ++--- .../JsonContentNestedDataSerializerFactory.cs | 10 +++ .../MsgPackContentNestedDataSerializer.cs | 77 +++++++++-------- ...gPackContentNestedDataSerializerFactory.cs | 62 ++++++++++++++ .../PublishedCache/NuCache/NuCacheComposer.cs | 11 +-- .../NuCache/PublishedSnapshotService.cs | 68 ++++++++------- src/Umbraco.Web/Umbraco.Web.csproj | 10 ++- 30 files changed, 377 insertions(+), 205 deletions(-) delete mode 100644 src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs rename src/Umbraco.Web/PublishedCache/NuCache/DataSource/{ContentNestedData.cs => ContentCacheDataModel.cs} (93%) create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs delete mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs create mode 100644 src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs index 15795bb61c..a99452a5b1 100644 --- a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs @@ -1,44 +1,47 @@ using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; -using Umbraco.Core.Services; +using Umbraco.Core.Models; namespace Umbraco.Core.PropertyEditors { /// - /// Ensures all property types that have an editor storing a complex value are compressed + /// Ensures all property types that have a property editor attributed with use data compression /// - public class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions + internal class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions { - private readonly IContentTypeService _contentTypeService; + private readonly IReadOnlyDictionary _contentTypes; private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); + private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache; - public CompressedStoragePropertyEditorCompressionOptions(PropertyEditorCollection propertyEditors) + public CompressedStoragePropertyEditorCompressionOptions( + IReadOnlyDictionary contentTypes, + PropertyEditorCollection propertyEditors, + ConcurrentDictionary<(int, string), CompressedStorageAttribute> compressedStoragePropertyEditorCache) { - _propertyEditors = propertyEditors; + _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); + _propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors)); + _compressedStoragePropertyEditorCache = compressedStoragePropertyEditorCache; } public bool IsCompressed(int contentTypeId, string alias) { - return false; - //var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => - //{ - // var ct = _contentTypeService.Get(contentTypeId); - // if (ct == null) return null; + var compressedStorage = _compressedStoragePropertyEditorCache.GetOrAdd((contentTypeId, alias), x => + { + if (!_contentTypes.TryGetValue(contentTypeId, out var ct)) + return null; - // var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); - // if (propertyType == null) return null; + var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + if (propertyType == null) return null; - // if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; + if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; - // var editor = propertyEditor.GetValueEditor(); - // if (editor == null) return null; + var attribute = propertyEditor.GetType().GetCustomAttribute(true); + return attribute; + }); - // return editor.ValueType; - //}); - - //return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; + return compressedStorage?.IsCompressed ?? false; } } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs index f5aaf3dc57..d1add38f19 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs @@ -1,10 +1,12 @@ -namespace Umbraco.Core.PropertyEditors +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors { /// /// Determines if a property type's value should be compressed /// public interface IPropertyCompressionOptions - { - bool IsCompressed(int contentTypeId, string alias); + { + bool IsCompressed(int contentTypeId, string propertyTypeAlias); } } diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index c85973f4b0..4be80083b8 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -1,10 +1,8 @@ -using NUnit.Framework; +using Moq; +using NUnit.Framework; using System; -using System.Collections; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Umbraco.Core.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Tests.PublishedContent @@ -16,10 +14,10 @@ namespace Umbraco.Tests.PublishedContent public void Ensure_Same_Results() { var jsonSerializer = new JsonContentNestedDataSerializer(); - var msgPackSerializer = new MsgPackContentNestedDataSerializer(); + var msgPackSerializer = new MsgPackContentNestedDataSerializer(Mock.Of()); var now = DateTime.Now; - var content = new ContentNestedData + var content = new ContentCacheDataModel { PropertyData = new Dictionary { @@ -55,14 +53,14 @@ namespace Umbraco.Tests.PublishedContent UrlSegment = "home" }; - var json = jsonSerializer.Serialize(1, content); - var msgPack = msgPackSerializer.Serialize(1, content); + var json = jsonSerializer.Serialize(1, content).StringData; + var msgPack = msgPackSerializer.Serialize(1, content).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(1, json); - var msgPackContent = msgPackSerializer.Deserialize(1, msgPack); + var jsonContent = jsonSerializer.Deserialize(1, json, null); + var msgPackContent = msgPackSerializer.Deserialize(1, null, msgPack); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index fef096498c..afba2dcc4f 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -37,7 +37,7 @@ namespace Umbraco.Tests.PublishedContent private ContentType _contentTypeInvariant; private ContentType _contentTypeVariant; private TestDataSource _source; - private IContentNestedDataSerializer _contentNestedDataSerializer; + private IContentCacheDataSerializerFactory _contentNestedDataSerializerFactory; [TearDown] public void Teardown() @@ -135,7 +135,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache _source = new TestDataSource(kits()); - _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); + _contentNestedDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); // at last, create the complete NuCache snapshot service! var options = new PublishedSnapshotServiceOptions { IgnoreLocalDb = true }; @@ -158,7 +158,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - _contentNestedDataSerializer); + _contentNestedDataSerializerFactory); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 792ccc8529..eee3500495 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -33,7 +33,7 @@ namespace Umbraco.Tests.PublishedContent { private IPublishedSnapshotService _snapshotService; private IVariationContextAccessor _variationAccesor; - private IContentNestedDataSerializer _contentNestedDataSerializer; + private IContentCacheDataSerializerFactory _contentNestedDataSerializerFactory; private ContentType _contentType; private PropertyType _propertyType; @@ -115,7 +115,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache var dataSource = new TestDataSource(kit); - _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); + _contentNestedDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); var runtime = Mock.Of(); Mock.Get(runtime).Setup(x => x.Level).Returns(RuntimeLevel.Run); @@ -204,7 +204,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - _contentNestedDataSerializer); + _contentNestedDataSerializerFactory); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index 5f72947382..be10db3a9d 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -82,7 +82,7 @@ namespace Umbraco.Tests.Scoping var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); + var nestedContentDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); return new PublishedSnapshotService( options, null, @@ -96,12 +96,12 @@ namespace Umbraco.Tests.Scoping ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(nestedContentDataSerializer), + new DatabaseDataSource(nestedContentDataSerializerFactory), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - nestedContentDataSerializer); + nestedContentDataSerializerFactory); } protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable urlProviders = null) diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index 938b14c3a9..aaad60f7e9 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -53,7 +53,7 @@ namespace Umbraco.Tests.Services var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); + var nestedContentDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); return new PublishedSnapshotService( options, @@ -68,12 +68,12 @@ namespace Umbraco.Tests.Services ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(nestedContentDataSerializer), + new DatabaseDataSource(nestedContentDataSerializerFactory), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - nestedContentDataSerializer); + nestedContentDataSerializerFactory); } public class LocalServerMessenger : ServerMessengerBase diff --git a/src/Umbraco.Web/Editors/NuCacheStatusController.cs b/src/Umbraco.Web/Editors/NuCacheStatusController.cs index 86dbdd4e01..589c763363 100644 --- a/src/Umbraco.Web/Editors/NuCacheStatusController.cs +++ b/src/Umbraco.Web/Editors/NuCacheStatusController.cs @@ -35,6 +35,11 @@ namespace Umbraco.Web.Editors service.RebuildContentDbCache(); service.RebuildMediaDbCache(); service.RebuildMemberDbCache(); + + // TODO: Shouldn't this just be ?? + // service.Rebuild(); + + return service.GetStatus(); } diff --git a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs index 42023382f1..2c1221e99e 100644 --- a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs @@ -13,6 +13,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a block list property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.BlockList, "Block List", diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 862837381a..7ce312c516 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -17,6 +17,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a grid property and parameter editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.Grid, "Grid layout", diff --git a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs index 2d66da5461..6ce66aaa00 100644 --- a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs @@ -7,6 +7,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a markdown editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.MarkdownEditor, "Markdown editor", diff --git a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs index 8f25449f99..ffe0051607 100644 --- a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs @@ -22,6 +22,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a nested content property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.NestedContent, "Nested Content", diff --git a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs deleted file mode 100644 index 6f626938bc..0000000000 --- a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Umbraco.Core.PropertyEditors; - -namespace Umbraco.Web.PropertyEditors -{ - /// - /// Disables all compression for all properties - /// - internal class NoopPropertyCompressionOptions : IPropertyCompressionOptions - { - public bool IsCompressed(int contentTypeId, string alias) => false; - } -} diff --git a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs index 42777f11ad..7c7a358bf3 100644 --- a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs @@ -17,6 +17,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a rich text property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TinyMce, "Rich Text Editor", diff --git a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs index c7bc2efbda..878330820a 100644 --- a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs @@ -7,6 +7,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a textarea property and parameter editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TextArea, EditorType.PropertyValue | EditorType.MacroParameter, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs index 61fb5c12a3..43aa9a14d7 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs @@ -15,6 +15,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public bool IsNull => ContentTypeId < 0; + public static ContentNodeKit Empty { get; } = new ContentNodeKit(); public static ContentNodeKit Null { get; } = new ContentNodeKit { ContentTypeId = -1 }; public void Build( diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs similarity index 93% rename from src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs rename to src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs index 1a49aaaf62..40acdfdb55 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs @@ -7,10 +7,10 @@ using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// - /// The content item 1:M data that is serialized to JSON + /// The content model stored in the content cache database table serialized as JSON /// [DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys - public class ContentNestedData + public class ContentCacheDataModel { // TODO: We don't want to allocate empty arrays //dont serialize empty properties diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs new file mode 100644 index 0000000000..7cd388d712 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public struct ContentCacheDataSerializationResult : IEquatable + { + public ContentCacheDataSerializationResult(string stringData, byte[] byteData) + { + StringData = stringData; + ByteData = byteData; + } + + public string StringData { get; } + public byte[] ByteData { get; } + + public override bool Equals(object obj) + { + return obj is ContentCacheDataSerializationResult result && Equals(result); + } + + public bool Equals(ContentCacheDataSerializationResult other) + { + return StringData == other.StringData && + EqualityComparer.Default.Equals(ByteData, other.ByteData); + } + + public override int GetHashCode() + { + var hashCode = 1910544615; + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(StringData); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(ByteData); + return hashCode; + } + + public static bool operator ==(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right) + { + return left.Equals(right); + } + + public static bool operator !=(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right) + { + return !(left == right); + } + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs new file mode 100644 index 0000000000..e5b15f8dce --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs @@ -0,0 +1,13 @@ +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + [Flags] + public enum ContentCacheDataSerializerEntityType + { + Document = 1, + Media = 2, + Member = 4 + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 0b3003d7ec..e60fd6623e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -2,14 +2,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using Newtonsoft.Json; using NPoco; using Umbraco.Core; using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Dtos; using Umbraco.Core.Scoping; -using Umbraco.Core.Serialization; using Umbraco.Web.Composing; using static Umbraco.Core.Persistence.NPocoSqlExtensions.Statics; @@ -21,11 +19,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class DatabaseDataSource : IDataSource { private const int PageSize = 500; - private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory; - public DatabaseDataSource(IContentNestedDataSerializer contentNestedDataSerializer) + public DatabaseDataSource(IContentCacheDataSerializerFactory contentCacheDataSerializerFactory) { - _contentNestedDataSerializer = contentNestedDataSerializer; + _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; } // we want arrays, we want them all loaded, not an enumerable @@ -110,7 +108,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); - return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto); + + if (dto == null) return ContentNodeKit.Empty; + + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + return CreateContentNodeKit(dto, serializer); } public IEnumerable GetAllContentSources(IScope scope) @@ -125,12 +127,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -143,12 +147,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -161,12 +167,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -201,7 +209,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); - return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto); + + if (dto == null) return ContentNodeKit.Empty; + + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + return CreateMediaNodeKit(dto, serializer); } public IEnumerable GetAllMediaSources(IScope scope) @@ -210,11 +222,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } public IEnumerable GetBranchMediaSources(IScope scope, int id) @@ -226,11 +242,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } public IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids) @@ -242,14 +262,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } - private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer) { ContentData d = null; ContentData p = null; @@ -264,9 +288,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); + var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); d = new ContentData { @@ -276,9 +298,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.EditWriterId, - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData, - UrlSegment = nested.UrlSegment + Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedContent.CultureData, + UrlSegment = deserializedContent.UrlSegment }; } } @@ -293,21 +315,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.PubDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.PubData); + var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.PubData, dto.PubDataRaw); p = new ContentData { Name = dto.PubName, - UrlSegment = nested.UrlSegment, + UrlSegment = deserializedContent.UrlSegment, Published = true, TemplateId = dto.PubTemplateId, VersionId = dto.VersionId, VersionDate = dto.PubVersionDate, WriterId = dto.PubWriterId, - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData + Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedContent.CultureData }; } } @@ -326,14 +346,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return s; } - private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer) { if (dto.EditData == null && dto.EditDataRaw == null) throw new InvalidOperationException("No data for media " + dto.Id); - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); + var deserializedMedia = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); var p = new ContentData { @@ -343,8 +361,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.CreatorId, // what-else? - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData + Properties = deserializedMedia.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedMedia.CultureData }; var n = new ContentNode(dto.Id, dto.Uid, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs new file mode 100644 index 0000000000..87ac5af91e --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -0,0 +1,18 @@ +using Umbraco.Core.PropertyEditors; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + + /// + /// Serializes/Deserializes document to the SQL Database as a string + /// + /// + /// Resolved from the . This cannot be resolved from DI. + /// + public interface IContentCacheDataSerializer + { + ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData); + ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model); + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs new file mode 100644 index 0000000000..14dfd7dc5b --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs @@ -0,0 +1,16 @@ +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public interface IContentCacheDataSerializerFactory + { + /// + /// Gets or creates a new instance of + /// + /// + /// + /// This method may return the same instance, however this depends on the state of the application and if any underlying data has changed. + /// This method may also be used to initialize anything before a serialization/deserialization session occurs. + /// + IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types); + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs deleted file mode 100644 index 09933d735d..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - // TODO: We need better names if possible, not sure why the class is called ContentNested in the first place - - /// - /// Serializes/Deserializes document to the SQL Database as bytes - /// - public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer - { - ContentNestedData DeserializeBytes(int contentTypeId, byte[] data); - byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData); - } - - /// - /// Serializes/Deserializes document to the SQL Database as a string - /// - public interface IContentNestedDataSerializer - { - ContentNestedData Deserialize(int contentTypeId, string data); - string Serialize(int contentTypeId, ContentNestedData nestedData); - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index d4f11591c1..2fa892a5e6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,18 +1,18 @@ using Newtonsoft.Json; +using System; using System.Collections.Generic; -using System.Linq; -using System.Reflection.Emit; -using System.Text; -using System.Threading.Tasks; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer + public class JsonContentNestedDataSerializer : IContentCacheDataSerializer { - public ContentNestedData Deserialize(int contentTypeId, string data) + public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { + if (byteData != null) + throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); + // by default JsonConvert will deserialize our numeric values as Int64 // which is bad, because they were Int32 in the database - take care @@ -24,18 +24,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateParseHandling = DateParseHandling.DateTime, DateFormatHandling = DateFormatHandling.IsoDateFormat, DateTimeZoneHandling = DateTimeZoneHandling.Utc, - DateFormatString = "o" + DateFormatString = "o" }; - return JsonConvert.DeserializeObject(data, settings); + return JsonConvert.DeserializeObject(stringData, settings); } - public string Serialize(int contentTypeId, ContentNestedData nestedData) + public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 - return JsonConvert.SerializeObject(nestedData); + var json = JsonConvert.SerializeObject(model); + return new ContentCacheDataSerializationResult(json, null); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs new file mode 100644 index 0000000000..e857eb8bf5 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs @@ -0,0 +1,10 @@ +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class JsonContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory + { + private Lazy _serializer = new Lazy(); + public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) => _serializer.Value; + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 4965935fbf..aad337b236 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,29 +1,27 @@ using K4os.Compression.LZ4; using MessagePack; -using MessagePack.Formatters; using MessagePack.Resolvers; -using NPoco.FluentMappings; using System; -using System.Collections.Generic; using System.Linq; using System.Text; using Umbraco.Core.PropertyEditors; -using Umbraco.Web.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// - /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack + /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack /// - internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer + public class MsgPackContentNestedDataSerializer : IContentCacheDataSerializer { - private MessagePackSerializerOptions _options; + private readonly MessagePackSerializerOptions _options; private readonly IPropertyCompressionOptions _propertyOptions; - public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions = null) + public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions) { - var defaultOptions = ContractlessStandardResolver.Options; + _propertyOptions = propertyOptions ?? throw new ArgumentNullException(nameof(propertyOptions)); + var defaultOptions = ContractlessStandardResolver.Options; var resolver = CompositeResolver.Create( // TODO: We want to be able to intern the strings for aliases when deserializing like we do for Newtonsoft but I'm unsure exactly how @@ -39,52 +37,51 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) - .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptions ?? new NoopPropertyCompressionOptions(); + .WithCompression(MessagePackCompression.Lz4BlockArray); } - public string ToJson(string serialized) + public string ToJson(byte[] bin) { - var bin = Convert.FromBase64String(serialized); var json = MessagePackSerializer.ConvertToJson(bin, _options); return json; } - public ContentNestedData Deserialize(int contentTypeId, string data) + public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { - var bin = Convert.FromBase64String(data); - var nestedData = MessagePackSerializer.Deserialize(bin, _options); - Expand(contentTypeId, nestedData); - return nestedData; + if (stringData != null) + { + // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) + var bin = Convert.FromBase64String(stringData); + var content = MessagePackSerializer.Deserialize(bin, _options); + Expand(contentTypeId, content); + return content; + } + else if (byteData != null) + { + var content = MessagePackSerializer.Deserialize(byteData, _options); + Expand(contentTypeId, content); + return content; + } + else + { + return null; + } } - public string Serialize(int contentTypeId, ContentNestedData nestedData) + public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) { - Compress(contentTypeId, nestedData); - var bin = MessagePackSerializer.Serialize(nestedData, _options); - return Convert.ToBase64String(bin); - } - - public ContentNestedData DeserializeBytes(int contentTypeId, byte[] data) - { - var nestedData = MessagePackSerializer.Deserialize(data, _options); - Expand(contentTypeId, nestedData); - return nestedData; - } - - public byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData) - { - Compress(contentTypeId, nestedData); - return MessagePackSerializer.Serialize(nestedData, _options); + Compress(contentTypeId, model); + var bytes = MessagePackSerializer.Serialize(model, _options); + return new ContentCacheDataSerializationResult(null, bytes); } /// /// Used during serialization to compress properties /// - /// - private void Compress(int contentTypeId, ContentNestedData nestedData) + /// + private void Compress(int contentTypeId, ContentCacheDataModel model) { - foreach(var propertyAliasToData in nestedData.PropertyData) + foreach(var propertyAliasToData in model.PropertyData) { if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) { @@ -100,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(int contentTypeId, ContentNestedData nestedData) + private void Expand(int contentTypeId, ContentCacheDataModel nestedData) { foreach (var propertyAliasToData in nestedData.PropertyData) { @@ -117,6 +114,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } + + //private class ContentNestedDataResolver : IFormatterResolver //{ // // GetFormatter's get cost should be minimized so use type cache. diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs new file mode 100644 index 0000000000..b509334604 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -0,0 +1,62 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using Umbraco.Core.Models; +using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Services; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class MsgPackContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory + { + private readonly IContentTypeService _contentTypeService; + private readonly IMediaTypeService _mediaTypeService; + private readonly IMemberTypeService _memberTypeService; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache = new ConcurrentDictionary<(int, string), CompressedStorageAttribute>(); + + public MsgPackContentNestedDataSerializerFactory(IContentTypeService contentTypeService, IMediaTypeService mediaTypeService, IMemberTypeService memberTypeService, PropertyEditorCollection propertyEditors) + { + _contentTypeService = contentTypeService; + _mediaTypeService = mediaTypeService; + _memberTypeService = memberTypeService; + _propertyEditors = propertyEditors; + } + + public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) + { + // Depending on which entity types are being requested, we need to look up those content types + // to initialize the compression options. + // We need to initialize these options now so that any data lookups required are completed and are not done while the content cache + // is performing DB queries which will result in errors since we'll be trying to query with open readers. + // NOTE: The calls to GetAll() below should be cached if the data has not been changed. + + var contentTypes = new Dictionary(); + if ((types & ContentCacheDataSerializerEntityType.Document) == ContentCacheDataSerializerEntityType.Document) + { + foreach(var ct in _contentTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + if ((types & ContentCacheDataSerializerEntityType.Media) == ContentCacheDataSerializerEntityType.Media) + { + foreach (var ct in _mediaTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + if ((types & ContentCacheDataSerializerEntityType.Member) == ContentCacheDataSerializerEntityType.Member) + { + foreach (var ct in _memberTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + + var options = new CompressedStoragePropertyEditorCompressionOptions(contentTypes, _propertyEditors, _compressedStoragePropertyEditorCache); + var serializer = new MsgPackContentNestedDataSerializer(options); + + return serializer; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index baa96a4a2e..faeb4f90b4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,10 +1,7 @@ -using System.Collections.Generic; -using System.Configuration; -using System.Linq; +using System.Configuration; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.PropertyEditors; -using Umbraco.Web.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Web.PublishedCache.NuCache @@ -19,13 +16,11 @@ namespace Umbraco.Web.PublishedCache.NuCache if (serializer != "MsgPack") { // TODO: This allows people to revert to the legacy serializer, by default it will be MessagePack - composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } else { - composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 41dfdd7a64..ad8705ef47 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1,14 +1,10 @@ using System; using System.Collections.Generic; using System.Configuration; -using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CSharpTest.Net.Collections; -using Newtonsoft.Json; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.Configuration; @@ -49,7 +45,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IPublishedModelFactory _publishedModelFactory; private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; - private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory; private readonly ContentDataSerializer _contentDataSerializer; // volatile because we read it with no lock @@ -84,7 +80,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer, ContentDataSerializer contentDataSerializer = null) + UrlSegmentProviderCollection urlSegmentProviders, IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -101,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _defaultCultureAccessor = defaultCultureAccessor; _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; - _contentNestedDataSerializer = contentNestedDataSerializer; + _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; _contentDataSerializer = contentDataSerializer; // we need an Xml serializer here so that the member cache can support XPath, @@ -1286,8 +1282,10 @@ namespace Umbraco.Web.PublishedCache.NuCache var db = args.Scope.Database; var content = (Content)args.Entity; + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // always refresh the edited data - OnRepositoryRefreshed(db, content, false); + OnRepositoryRefreshed(serializer, db, content, false); // if unpublishing, remove published data from table if (content.PublishedState == PublishedState.Unpublishing) @@ -1295,33 +1293,37 @@ namespace Umbraco.Web.PublishedCache.NuCache // if publishing, refresh the published data else if (content.PublishedState == PublishedState.Publishing) - OnRepositoryRefreshed(db, content, true); + OnRepositoryRefreshed(serializer, db, content, true); } private void OnMediaRefreshedEntity(MediaRepository sender, MediaRepository.ScopedEntityEventArgs args) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + var db = args.Scope.Database; var media = args.Entity; // refresh the edited data - OnRepositoryRefreshed(db, media, false); + OnRepositoryRefreshed(serializer, db, media, false); } private void OnMemberRefreshedEntity(MemberRepository sender, MemberRepository.ScopedEntityEventArgs args) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member); + var db = args.Scope.Database; var member = args.Entity; // refresh the edited data - OnRepositoryRefreshed(db, member, false); + OnRepositoryRefreshed(serializer, db, member, false); } - private void OnRepositoryRefreshed(IUmbracoDatabase db, IContentBase content, bool published) + private void OnRepositoryRefreshed(IContentCacheDataSerializer serializer, IUmbracoDatabase db, IContentBase content, bool published) { // use a custom SQL to update row version on each update //db.InsertOrUpdate(dto); - var dto = GetDto(content, published); + var dto = GetDto(content, published, serializer); db.InsertOrUpdate(dto, "SET data=@data, dataRaw=@dataRaw, rv=rv+1 WHERE nodeId=@id AND published=@published", new @@ -1375,7 +1377,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private ContentNuDto GetDto(IContentBase content, bool published) + private ContentNuDto GetDto(IContentBase content, bool published, IContentCacheDataSerializer serializer) { // should inject these in ctor // BUT for the time being we decide not to support ConvertDbToXml/String @@ -1447,19 +1449,21 @@ namespace Umbraco.Web.PublishedCache.NuCache } //the dictionary that will be serialized - var nestedData = new ContentNestedData + var contentCacheData = new ContentCacheDataModel { PropertyData = propertyData, CultureData = cultureData, UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; + var serialized = serializer.Serialize(content.ContentTypeId, contentCacheData); + var dto = new ContentNuDto { NodeId = content.Id, Published = published, - Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(content.ContentTypeId, nestedData) : null, - RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(content.ContentTypeId, nestedData) : null + Data = serialized.StringData, + RawData = serialized.ByteData }; //Core.Composing.Current.Logger.Debug(dto.Data); @@ -1482,30 +1486,32 @@ namespace Umbraco.Web.PublishedCache.NuCache public override void Rebuild() { _logger.Debug("Rebuilding..."); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document | ContentCacheDataSerializerEntityType.Media | ContentCacheDataSerializerEntityType.Member); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.ContentTree); scope.ReadLock(Constants.Locks.MediaTree); scope.ReadLock(Constants.Locks.MemberTree); - RebuildContentDbCacheLocked(scope, GetSqlPagingSize(), null); - RebuildMediaDbCacheLocked(scope, GetSqlPagingSize(), null); - RebuildMemberDbCacheLocked(scope, GetSqlPagingSize(), null); + RebuildContentDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); + RebuildMediaDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); + RebuildMemberDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); scope.Complete(); } } public void RebuildContentDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.ContentTree); - RebuildContentDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildContentDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes content tree lock - private void RebuildContentDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + private void RebuildContentDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var contentObjectType = Constants.ObjectTypes.Document; @@ -1552,11 +1558,11 @@ WHERE cmsContentNu.nodeId IN ( foreach (var c in descendants) { // always the edited version - items.Add(GetDto(c, false)); + items.Add(GetDto(c, false, serializer)); // and also the published version if it makes any sense if (c.Published) - items.Add(GetDto(c, true)); + items.Add(GetDto(c, true, serializer)); count++; } @@ -1568,16 +1574,17 @@ WHERE cmsContentNu.nodeId IN ( public void RebuildMediaDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.MediaTree); - RebuildMediaDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildMediaDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes media tree lock - public void RebuildMediaDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + public void RebuildMediaDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var mediaObjectType = Constants.ObjectTypes.Media; @@ -1619,7 +1626,7 @@ WHERE cmsContentNu.nodeId IN ( { // the tree is locked, counting and comparing to total is safe var descendants = _mediaRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path")); - var items = descendants.Select(m => GetDto(m, false)).ToList(); + var items = descendants.Select(m => GetDto(m, false, serializer)).ToList(); db.BulkInsertRecords(items); processed += items.Count; } while (processed < total); @@ -1627,16 +1634,17 @@ WHERE cmsContentNu.nodeId IN ( public void RebuildMemberDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.MemberTree); - RebuildMemberDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildMemberDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes member tree lock - public void RebuildMemberDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + public void RebuildMemberDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var memberObjectType = Constants.ObjectTypes.Member; @@ -1677,7 +1685,7 @@ WHERE cmsContentNu.nodeId IN ( do { var descendants = _memberRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path")); - var items = descendants.Select(m => GetDto(m, false)).ToArray(); + var items = descendants.Select(m => GetDto(m, false, serializer)).ToArray(); db.BulkInsertRecords(items); processed += items.Length; } while (processed < total); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 25478bf626..e93caaac66 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -262,12 +262,15 @@ - - + + + + + @@ -277,6 +280,7 @@ + @@ -588,7 +592,7 @@ - +