diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs index 15795bb61c..a99452a5b1 100644 --- a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs @@ -1,44 +1,47 @@ using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; -using Umbraco.Core.Services; +using Umbraco.Core.Models; namespace Umbraco.Core.PropertyEditors { /// - /// Ensures all property types that have an editor storing a complex value are compressed + /// Ensures all property types that have a property editor attributed with use data compression /// - public class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions + internal class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions { - private readonly IContentTypeService _contentTypeService; + private readonly IReadOnlyDictionary _contentTypes; private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); + private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache; - public CompressedStoragePropertyEditorCompressionOptions(PropertyEditorCollection propertyEditors) + public CompressedStoragePropertyEditorCompressionOptions( + IReadOnlyDictionary contentTypes, + PropertyEditorCollection propertyEditors, + ConcurrentDictionary<(int, string), CompressedStorageAttribute> compressedStoragePropertyEditorCache) { - _propertyEditors = propertyEditors; + _contentTypes = contentTypes ?? throw new System.ArgumentNullException(nameof(contentTypes)); + _propertyEditors = propertyEditors ?? throw new System.ArgumentNullException(nameof(propertyEditors)); + _compressedStoragePropertyEditorCache = compressedStoragePropertyEditorCache; } public bool IsCompressed(int contentTypeId, string alias) { - return false; - //var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => - //{ - // var ct = _contentTypeService.Get(contentTypeId); - // if (ct == null) return null; + var compressedStorage = _compressedStoragePropertyEditorCache.GetOrAdd((contentTypeId, alias), x => + { + if (!_contentTypes.TryGetValue(contentTypeId, out var ct)) + return null; - // var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); - // if (propertyType == null) return null; + var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + if (propertyType == null) return null; - // if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; + if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; - // var editor = propertyEditor.GetValueEditor(); - // if (editor == null) return null; + var attribute = propertyEditor.GetType().GetCustomAttribute(true); + return attribute; + }); - // return editor.ValueType; - //}); - - //return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; + return compressedStorage?.IsCompressed ?? false; } } } diff --git a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs index f5aaf3dc57..d1add38f19 100644 --- a/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs @@ -1,10 +1,12 @@ -namespace Umbraco.Core.PropertyEditors +using Umbraco.Core.Models; + +namespace Umbraco.Core.PropertyEditors { /// /// Determines if a property type's value should be compressed /// public interface IPropertyCompressionOptions - { - bool IsCompressed(int contentTypeId, string alias); + { + bool IsCompressed(int contentTypeId, string propertyTypeAlias); } } diff --git a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs index c85973f4b0..4be80083b8 100644 --- a/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs +++ b/src/Umbraco.Tests/PublishedContent/ContentSerializationTests.cs @@ -1,10 +1,8 @@ -using NUnit.Framework; +using Moq; +using NUnit.Framework; using System; -using System.Collections; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using Umbraco.Core.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Tests.PublishedContent @@ -16,10 +14,10 @@ namespace Umbraco.Tests.PublishedContent public void Ensure_Same_Results() { var jsonSerializer = new JsonContentNestedDataSerializer(); - var msgPackSerializer = new MsgPackContentNestedDataSerializer(); + var msgPackSerializer = new MsgPackContentNestedDataSerializer(Mock.Of()); var now = DateTime.Now; - var content = new ContentNestedData + var content = new ContentCacheDataModel { PropertyData = new Dictionary { @@ -55,14 +53,14 @@ namespace Umbraco.Tests.PublishedContent UrlSegment = "home" }; - var json = jsonSerializer.Serialize(1, content); - var msgPack = msgPackSerializer.Serialize(1, content); + var json = jsonSerializer.Serialize(1, content).StringData; + var msgPack = msgPackSerializer.Serialize(1, content).ByteData; Console.WriteLine(json); Console.WriteLine(msgPackSerializer.ToJson(msgPack)); - var jsonContent = jsonSerializer.Deserialize(1, json); - var msgPackContent = msgPackSerializer.Deserialize(1, msgPack); + var jsonContent = jsonSerializer.Deserialize(1, json, null); + var msgPackContent = msgPackSerializer.Deserialize(1, null, msgPack); CollectionAssert.AreEqual(jsonContent.CultureData.Keys, msgPackContent.CultureData.Keys); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index fef096498c..afba2dcc4f 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -37,7 +37,7 @@ namespace Umbraco.Tests.PublishedContent private ContentType _contentTypeInvariant; private ContentType _contentTypeVariant; private TestDataSource _source; - private IContentNestedDataSerializer _contentNestedDataSerializer; + private IContentCacheDataSerializerFactory _contentNestedDataSerializerFactory; [TearDown] public void Teardown() @@ -135,7 +135,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache _source = new TestDataSource(kits()); - _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); + _contentNestedDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); // at last, create the complete NuCache snapshot service! var options = new PublishedSnapshotServiceOptions { IgnoreLocalDb = true }; @@ -158,7 +158,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - _contentNestedDataSerializer); + _contentNestedDataSerializerFactory); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index 792ccc8529..eee3500495 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -33,7 +33,7 @@ namespace Umbraco.Tests.PublishedContent { private IPublishedSnapshotService _snapshotService; private IVariationContextAccessor _variationAccesor; - private IContentNestedDataSerializer _contentNestedDataSerializer; + private IContentCacheDataSerializerFactory _contentNestedDataSerializerFactory; private ContentType _contentType; private PropertyType _propertyType; @@ -115,7 +115,7 @@ namespace Umbraco.Tests.PublishedContent // create a data source for NuCache var dataSource = new TestDataSource(kit); - _contentNestedDataSerializer = new JsonContentNestedDataSerializer(); + _contentNestedDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); var runtime = Mock.Of(); Mock.Get(runtime).Setup(x => x.Level).Returns(RuntimeLevel.Run); @@ -204,7 +204,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - _contentNestedDataSerializer); + _contentNestedDataSerializerFactory); // invariant is the current default _variationAccesor.VariationContext = new VariationContext(); diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index 5f72947382..be10db3a9d 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -82,7 +82,7 @@ namespace Umbraco.Tests.Scoping var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); + var nestedContentDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); return new PublishedSnapshotService( options, null, @@ -96,12 +96,12 @@ namespace Umbraco.Tests.Scoping ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(nestedContentDataSerializer), + new DatabaseDataSource(nestedContentDataSerializerFactory), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - nestedContentDataSerializer); + nestedContentDataSerializerFactory); } protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable urlProviders = null) diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index 938b14c3a9..aaad60f7e9 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -53,7 +53,7 @@ namespace Umbraco.Tests.Services var mediaRepository = Mock.Of(); var memberRepository = Mock.Of(); - var nestedContentDataSerializer = new JsonContentNestedDataSerializer(); + var nestedContentDataSerializerFactory = new JsonContentNestedDataSerializerFactory(); return new PublishedSnapshotService( options, @@ -68,12 +68,12 @@ namespace Umbraco.Tests.Services ScopeProvider, documentRepository, mediaRepository, memberRepository, DefaultCultureAccessor, - new DatabaseDataSource(nestedContentDataSerializer), + new DatabaseDataSource(nestedContentDataSerializerFactory), Factory.GetInstance(), Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), - nestedContentDataSerializer); + nestedContentDataSerializerFactory); } public class LocalServerMessenger : ServerMessengerBase diff --git a/src/Umbraco.Web/Editors/NuCacheStatusController.cs b/src/Umbraco.Web/Editors/NuCacheStatusController.cs index 86dbdd4e01..589c763363 100644 --- a/src/Umbraco.Web/Editors/NuCacheStatusController.cs +++ b/src/Umbraco.Web/Editors/NuCacheStatusController.cs @@ -35,6 +35,11 @@ namespace Umbraco.Web.Editors service.RebuildContentDbCache(); service.RebuildMediaDbCache(); service.RebuildMemberDbCache(); + + // TODO: Shouldn't this just be ?? + // service.Rebuild(); + + return service.GetStatus(); } diff --git a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs index 42023382f1..2c1221e99e 100644 --- a/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/BlockListPropertyEditor.cs @@ -13,6 +13,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a block list property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.BlockList, "Block List", diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index 862837381a..7ce312c516 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -17,6 +17,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a grid property and parameter editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.Grid, "Grid layout", diff --git a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs index 2d66da5461..6ce66aaa00 100644 --- a/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/MarkdownPropertyEditor.cs @@ -7,6 +7,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a markdown editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.MarkdownEditor, "Markdown editor", diff --git a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs index 8f25449f99..ffe0051607 100644 --- a/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/NestedContentPropertyEditor.cs @@ -22,6 +22,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a nested content property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.NestedContent, "Nested Content", diff --git a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs deleted file mode 100644 index 6f626938bc..0000000000 --- a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Umbraco.Core.PropertyEditors; - -namespace Umbraco.Web.PropertyEditors -{ - /// - /// Disables all compression for all properties - /// - internal class NoopPropertyCompressionOptions : IPropertyCompressionOptions - { - public bool IsCompressed(int contentTypeId, string alias) => false; - } -} diff --git a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs index 42777f11ad..7c7a358bf3 100644 --- a/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/RichTextPropertyEditor.cs @@ -17,6 +17,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a rich text property editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TinyMce, "Rich Text Editor", diff --git a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs index c7bc2efbda..878330820a 100644 --- a/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/TextAreaPropertyEditor.cs @@ -7,6 +7,7 @@ namespace Umbraco.Web.PropertyEditors /// /// Represents a textarea property and parameter editor. /// + [CompressedStorage] [DataEditor( Constants.PropertyEditors.Aliases.TextArea, EditorType.PropertyValue | EditorType.MacroParameter, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs index 61fb5c12a3..43aa9a14d7 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentNodeKit.cs @@ -15,6 +15,7 @@ namespace Umbraco.Web.PublishedCache.NuCache public bool IsNull => ContentTypeId < 0; + public static ContentNodeKit Empty { get; } = new ContentNodeKit(); public static ContentNodeKit Null { get; } = new ContentNodeKit { ContentTypeId = -1 }; public void Build( diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs similarity index 93% rename from src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs rename to src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs index 1a49aaaf62..40acdfdb55 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentNestedData.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataModel.cs @@ -7,10 +7,10 @@ using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { /// - /// The content item 1:M data that is serialized to JSON + /// The content model stored in the content cache database table serialized as JSON /// [DataContract] // NOTE: Use DataContract annotations here to control how MessagePack serializes/deserializes the data to use INT keys - public class ContentNestedData + public class ContentCacheDataModel { // TODO: We don't want to allocate empty arrays //dont serialize empty properties diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs new file mode 100644 index 0000000000..7cd388d712 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializationResult.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public struct ContentCacheDataSerializationResult : IEquatable + { + public ContentCacheDataSerializationResult(string stringData, byte[] byteData) + { + StringData = stringData; + ByteData = byteData; + } + + public string StringData { get; } + public byte[] ByteData { get; } + + public override bool Equals(object obj) + { + return obj is ContentCacheDataSerializationResult result && Equals(result); + } + + public bool Equals(ContentCacheDataSerializationResult other) + { + return StringData == other.StringData && + EqualityComparer.Default.Equals(ByteData, other.ByteData); + } + + public override int GetHashCode() + { + var hashCode = 1910544615; + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(StringData); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(ByteData); + return hashCode; + } + + public static bool operator ==(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right) + { + return left.Equals(right); + } + + public static bool operator !=(ContentCacheDataSerializationResult left, ContentCacheDataSerializationResult right) + { + return !(left == right); + } + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs new file mode 100644 index 0000000000..e5b15f8dce --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/ContentCacheDataSerializerEntityType.cs @@ -0,0 +1,13 @@ +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + [Flags] + public enum ContentCacheDataSerializerEntityType + { + Document = 1, + Media = 2, + Member = 4 + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 0b3003d7ec..e60fd6623e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -2,14 +2,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using Newtonsoft.Json; using NPoco; using Umbraco.Core; using Umbraco.Core.Logging; using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Dtos; using Umbraco.Core.Scoping; -using Umbraco.Core.Serialization; using Umbraco.Web.Composing; using static Umbraco.Core.Persistence.NPocoSqlExtensions.Statics; @@ -21,11 +19,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource internal class DatabaseDataSource : IDataSource { private const int PageSize = 500; - private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory; - public DatabaseDataSource(IContentNestedDataSerializer contentNestedDataSerializer) + public DatabaseDataSource(IContentCacheDataSerializerFactory contentCacheDataSerializerFactory) { - _contentNestedDataSerializer = contentNestedDataSerializer; + _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; } // we want arrays, we want them all loaded, not an enumerable @@ -110,7 +108,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); - return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto); + + if (dto == null) return ContentNodeKit.Empty; + + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + return CreateContentNodeKit(dto, serializer); } public IEnumerable GetAllContentSources(IScope scope) @@ -125,12 +127,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed); var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -143,12 +147,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -161,12 +167,14 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) { - yield return CreateContentNodeKit(row); + yield return CreateContentNodeKit(row, serializer); } } @@ -201,7 +209,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); var dto = scope.Database.Fetch(sql).FirstOrDefault(); - return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto); + + if (dto == null) return ContentNodeKit.Empty; + + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + return CreateMediaNodeKit(dto, serializer); } public IEnumerable GetAllMediaSources(IScope scope) @@ -210,11 +222,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } public IEnumerable GetBranchMediaSources(IScope scope, int id) @@ -226,11 +242,15 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .Where(x => x.NodeId == id, "x") .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } public IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids) @@ -242,14 +262,18 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .WhereIn(x => x.ContentTypeId, ids) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) - yield return CreateMediaNodeKit(row); + { + yield return CreateMediaNodeKit(row, serializer); + } } - private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer) { ContentData d = null; ContentData p = null; @@ -264,9 +288,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); + var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); d = new ContentData { @@ -276,9 +298,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.EditWriterId, - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData, - UrlSegment = nested.UrlSegment + Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedContent.CultureData, + UrlSegment = deserializedContent.UrlSegment }; } } @@ -293,21 +315,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } else { - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.PubDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.PubData); + var deserializedContent = serializer.Deserialize(dto.ContentTypeId, dto.PubData, dto.PubDataRaw); p = new ContentData { Name = dto.PubName, - UrlSegment = nested.UrlSegment, + UrlSegment = deserializedContent.UrlSegment, Published = true, TemplateId = dto.PubTemplateId, VersionId = dto.VersionId, VersionDate = dto.PubVersionDate, WriterId = dto.PubWriterId, - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData + Properties = deserializedContent.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedContent.CultureData }; } } @@ -326,14 +346,12 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return s; } - private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto) + private ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto, IContentCacheDataSerializer serializer) { if (dto.EditData == null && dto.EditDataRaw == null) throw new InvalidOperationException("No data for media " + dto.Id); - var nested = _contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer - ? byteSerializer.DeserializeBytes(dto.ContentTypeId, dto.EditDataRaw) - : _contentNestedDataSerializer.Deserialize(dto.ContentTypeId, dto.EditData); + var deserializedMedia = serializer.Deserialize(dto.ContentTypeId, dto.EditData, dto.EditDataRaw); var p = new ContentData { @@ -343,8 +361,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource VersionId = dto.VersionId, VersionDate = dto.EditVersionDate, WriterId = dto.CreatorId, // what-else? - Properties = nested.PropertyData, // TODO: We don't want to allocate empty arrays - CultureInfos = nested.CultureData + Properties = deserializedMedia.PropertyData, // TODO: We don't want to allocate empty arrays + CultureInfos = deserializedMedia.CultureData }; var n = new ContentNode(dto.Id, dto.Uid, diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs new file mode 100644 index 0000000000..87ac5af91e --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializer.cs @@ -0,0 +1,18 @@ +using Umbraco.Core.PropertyEditors; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + + /// + /// Serializes/Deserializes document to the SQL Database as a string + /// + /// + /// Resolved from the . This cannot be resolved from DI. + /// + public interface IContentCacheDataSerializer + { + ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData); + ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model); + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs new file mode 100644 index 0000000000..14dfd7dc5b --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentCacheDataSerializerFactory.cs @@ -0,0 +1,16 @@ +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + public interface IContentCacheDataSerializerFactory + { + /// + /// Gets or creates a new instance of + /// + /// + /// + /// This method may return the same instance, however this depends on the state of the application and if any underlying data has changed. + /// This method may also be used to initialize anything before a serialization/deserialization session occurs. + /// + IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types); + } + +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs deleted file mode 100644 index 09933d735d..0000000000 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/IContentNestedDataSerializer.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Umbraco.Web.PublishedCache.NuCache.DataSource -{ - // TODO: We need better names if possible, not sure why the class is called ContentNested in the first place - - /// - /// Serializes/Deserializes document to the SQL Database as bytes - /// - public interface IContentNestedDataByteSerializer : IContentNestedDataSerializer - { - ContentNestedData DeserializeBytes(int contentTypeId, byte[] data); - byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData); - } - - /// - /// Serializes/Deserializes document to the SQL Database as a string - /// - public interface IContentNestedDataSerializer - { - ContentNestedData Deserialize(int contentTypeId, string data); - string Serialize(int contentTypeId, ContentNestedData nestedData); - } -} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index d4f11591c1..2fa892a5e6 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,18 +1,18 @@ using Newtonsoft.Json; +using System; using System.Collections.Generic; -using System.Linq; -using System.Reflection.Emit; -using System.Text; -using System.Threading.Tasks; using Umbraco.Core.Serialization; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { - internal class JsonContentNestedDataSerializer : IContentNestedDataSerializer + public class JsonContentNestedDataSerializer : IContentCacheDataSerializer { - public ContentNestedData Deserialize(int contentTypeId, string data) + public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { + if (byteData != null) + throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); + // by default JsonConvert will deserialize our numeric values as Int64 // which is bad, because they were Int32 in the database - take care @@ -24,18 +24,19 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateParseHandling = DateParseHandling.DateTime, DateFormatHandling = DateFormatHandling.IsoDateFormat, DateTimeZoneHandling = DateTimeZoneHandling.Utc, - DateFormatString = "o" + DateFormatString = "o" }; - return JsonConvert.DeserializeObject(data, settings); + return JsonConvert.DeserializeObject(stringData, settings); } - public string Serialize(int contentTypeId, ContentNestedData nestedData) + public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) { // note that numeric values (which are Int32) are serialized without their // type (eg "value":1234) and JsonConvert by default deserializes them as Int64 - return JsonConvert.SerializeObject(nestedData); + var json = JsonConvert.SerializeObject(model); + return new ContentCacheDataSerializationResult(json, null); } } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs new file mode 100644 index 0000000000..e857eb8bf5 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializerFactory.cs @@ -0,0 +1,10 @@ +using System; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class JsonContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory + { + private Lazy _serializer = new Lazy(); + public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) => _serializer.Value; + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 4965935fbf..aad337b236 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -1,29 +1,27 @@ using K4os.Compression.LZ4; using MessagePack; -using MessagePack.Formatters; using MessagePack.Resolvers; -using NPoco.FluentMappings; using System; -using System.Collections.Generic; using System.Linq; using System.Text; using Umbraco.Core.PropertyEditors; -using Umbraco.Web.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource { + /// - /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack + /// Serializes/Deserializes document to the SQL Database as bytes using MessagePack /// - internal class MsgPackContentNestedDataSerializer : IContentNestedDataByteSerializer + public class MsgPackContentNestedDataSerializer : IContentCacheDataSerializer { - private MessagePackSerializerOptions _options; + private readonly MessagePackSerializerOptions _options; private readonly IPropertyCompressionOptions _propertyOptions; - public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions = null) + public MsgPackContentNestedDataSerializer(IPropertyCompressionOptions propertyOptions) { - var defaultOptions = ContractlessStandardResolver.Options; + _propertyOptions = propertyOptions ?? throw new ArgumentNullException(nameof(propertyOptions)); + var defaultOptions = ContractlessStandardResolver.Options; var resolver = CompositeResolver.Create( // TODO: We want to be able to intern the strings for aliases when deserializing like we do for Newtonsoft but I'm unsure exactly how @@ -39,52 +37,51 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource _options = defaultOptions .WithResolver(resolver) - .WithCompression(MessagePackCompression.Lz4BlockArray); - _propertyOptions = propertyOptions ?? new NoopPropertyCompressionOptions(); + .WithCompression(MessagePackCompression.Lz4BlockArray); } - public string ToJson(string serialized) + public string ToJson(byte[] bin) { - var bin = Convert.FromBase64String(serialized); var json = MessagePackSerializer.ConvertToJson(bin, _options); return json; } - public ContentNestedData Deserialize(int contentTypeId, string data) + public ContentCacheDataModel Deserialize(int contentTypeId, string stringData, byte[] byteData) { - var bin = Convert.FromBase64String(data); - var nestedData = MessagePackSerializer.Deserialize(bin, _options); - Expand(contentTypeId, nestedData); - return nestedData; + if (stringData != null) + { + // NOTE: We don't really support strings but it's possible if manually used (i.e. tests) + var bin = Convert.FromBase64String(stringData); + var content = MessagePackSerializer.Deserialize(bin, _options); + Expand(contentTypeId, content); + return content; + } + else if (byteData != null) + { + var content = MessagePackSerializer.Deserialize(byteData, _options); + Expand(contentTypeId, content); + return content; + } + else + { + return null; + } } - public string Serialize(int contentTypeId, ContentNestedData nestedData) + public ContentCacheDataSerializationResult Serialize(int contentTypeId, ContentCacheDataModel model) { - Compress(contentTypeId, nestedData); - var bin = MessagePackSerializer.Serialize(nestedData, _options); - return Convert.ToBase64String(bin); - } - - public ContentNestedData DeserializeBytes(int contentTypeId, byte[] data) - { - var nestedData = MessagePackSerializer.Deserialize(data, _options); - Expand(contentTypeId, nestedData); - return nestedData; - } - - public byte[] SerializeBytes(int contentTypeId, ContentNestedData nestedData) - { - Compress(contentTypeId, nestedData); - return MessagePackSerializer.Serialize(nestedData, _options); + Compress(contentTypeId, model); + var bytes = MessagePackSerializer.Serialize(model, _options); + return new ContentCacheDataSerializationResult(null, bytes); } /// /// Used during serialization to compress properties /// - /// - private void Compress(int contentTypeId, ContentNestedData nestedData) + /// + private void Compress(int contentTypeId, ContentCacheDataModel model) { - foreach(var propertyAliasToData in nestedData.PropertyData) + foreach(var propertyAliasToData in model.PropertyData) { if (_propertyOptions.IsCompressed(contentTypeId, propertyAliasToData.Key)) { @@ -100,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource /// Used during deserialization to map the property data as lazy or expand the value /// /// - private void Expand(int contentTypeId, ContentNestedData nestedData) + private void Expand(int contentTypeId, ContentCacheDataModel nestedData) { foreach (var propertyAliasToData in nestedData.PropertyData) { @@ -117,6 +114,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource } } + + //private class ContentNestedDataResolver : IFormatterResolver //{ // // GetFormatter's get cost should be minimized so use type cache. diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs new file mode 100644 index 0000000000..b509334604 --- /dev/null +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializerFactory.cs @@ -0,0 +1,62 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using Umbraco.Core.Models; +using Umbraco.Core.PropertyEditors; +using Umbraco.Core.Services; + +namespace Umbraco.Web.PublishedCache.NuCache.DataSource +{ + internal class MsgPackContentNestedDataSerializerFactory : IContentCacheDataSerializerFactory + { + private readonly IContentTypeService _contentTypeService; + private readonly IMediaTypeService _mediaTypeService; + private readonly IMemberTypeService _memberTypeService; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int, string), CompressedStorageAttribute> _compressedStoragePropertyEditorCache = new ConcurrentDictionary<(int, string), CompressedStorageAttribute>(); + + public MsgPackContentNestedDataSerializerFactory(IContentTypeService contentTypeService, IMediaTypeService mediaTypeService, IMemberTypeService memberTypeService, PropertyEditorCollection propertyEditors) + { + _contentTypeService = contentTypeService; + _mediaTypeService = mediaTypeService; + _memberTypeService = memberTypeService; + _propertyEditors = propertyEditors; + } + + public IContentCacheDataSerializer Create(ContentCacheDataSerializerEntityType types) + { + // Depending on which entity types are being requested, we need to look up those content types + // to initialize the compression options. + // We need to initialize these options now so that any data lookups required are completed and are not done while the content cache + // is performing DB queries which will result in errors since we'll be trying to query with open readers. + // NOTE: The calls to GetAll() below should be cached if the data has not been changed. + + var contentTypes = new Dictionary(); + if ((types & ContentCacheDataSerializerEntityType.Document) == ContentCacheDataSerializerEntityType.Document) + { + foreach(var ct in _contentTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + if ((types & ContentCacheDataSerializerEntityType.Media) == ContentCacheDataSerializerEntityType.Media) + { + foreach (var ct in _mediaTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + if ((types & ContentCacheDataSerializerEntityType.Member) == ContentCacheDataSerializerEntityType.Member) + { + foreach (var ct in _memberTypeService.GetAll()) + { + contentTypes[ct.Id] = ct; + } + } + + var options = new CompressedStoragePropertyEditorCompressionOptions(contentTypes, _propertyEditors, _compressedStoragePropertyEditorCache); + var serializer = new MsgPackContentNestedDataSerializer(options); + + return serializer; + } + } +} diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index baa96a4a2e..faeb4f90b4 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,10 +1,7 @@ -using System.Collections.Generic; -using System.Configuration; -using System.Linq; +using System.Configuration; using Umbraco.Core; using Umbraco.Core.Composing; using Umbraco.Core.PropertyEditors; -using Umbraco.Web.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; namespace Umbraco.Web.PublishedCache.NuCache @@ -19,13 +16,11 @@ namespace Umbraco.Web.PublishedCache.NuCache if (serializer != "MsgPack") { // TODO: This allows people to revert to the legacy serializer, by default it will be MessagePack - composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } else { - composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index 41dfdd7a64..ad8705ef47 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -1,14 +1,10 @@ using System; using System.Collections.Generic; using System.Configuration; -using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using CSharpTest.Net.Collections; -using Newtonsoft.Json; using Umbraco.Core; using Umbraco.Core.Cache; using Umbraco.Core.Configuration; @@ -49,7 +45,7 @@ namespace Umbraco.Web.PublishedCache.NuCache private readonly IPublishedModelFactory _publishedModelFactory; private readonly IDefaultCultureAccessor _defaultCultureAccessor; private readonly UrlSegmentProviderCollection _urlSegmentProviders; - private readonly IContentNestedDataSerializer _contentNestedDataSerializer; + private readonly IContentCacheDataSerializerFactory _contentCacheDataSerializerFactory; private readonly ContentDataSerializer _contentDataSerializer; // volatile because we read it with no lock @@ -84,7 +80,7 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentNestedDataSerializer contentNestedDataSerializer, ContentDataSerializer contentDataSerializer = null) + UrlSegmentProviderCollection urlSegmentProviders, IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -101,7 +97,7 @@ namespace Umbraco.Web.PublishedCache.NuCache _defaultCultureAccessor = defaultCultureAccessor; _globalSettings = globalSettings; _urlSegmentProviders = urlSegmentProviders; - _contentNestedDataSerializer = contentNestedDataSerializer; + _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; _contentDataSerializer = contentDataSerializer; // we need an Xml serializer here so that the member cache can support XPath, @@ -1286,8 +1282,10 @@ namespace Umbraco.Web.PublishedCache.NuCache var db = args.Scope.Database; var content = (Content)args.Entity; + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); + // always refresh the edited data - OnRepositoryRefreshed(db, content, false); + OnRepositoryRefreshed(serializer, db, content, false); // if unpublishing, remove published data from table if (content.PublishedState == PublishedState.Unpublishing) @@ -1295,33 +1293,37 @@ namespace Umbraco.Web.PublishedCache.NuCache // if publishing, refresh the published data else if (content.PublishedState == PublishedState.Publishing) - OnRepositoryRefreshed(db, content, true); + OnRepositoryRefreshed(serializer, db, content, true); } private void OnMediaRefreshedEntity(MediaRepository sender, MediaRepository.ScopedEntityEventArgs args) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); + var db = args.Scope.Database; var media = args.Entity; // refresh the edited data - OnRepositoryRefreshed(db, media, false); + OnRepositoryRefreshed(serializer, db, media, false); } private void OnMemberRefreshedEntity(MemberRepository sender, MemberRepository.ScopedEntityEventArgs args) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member); + var db = args.Scope.Database; var member = args.Entity; // refresh the edited data - OnRepositoryRefreshed(db, member, false); + OnRepositoryRefreshed(serializer, db, member, false); } - private void OnRepositoryRefreshed(IUmbracoDatabase db, IContentBase content, bool published) + private void OnRepositoryRefreshed(IContentCacheDataSerializer serializer, IUmbracoDatabase db, IContentBase content, bool published) { // use a custom SQL to update row version on each update //db.InsertOrUpdate(dto); - var dto = GetDto(content, published); + var dto = GetDto(content, published, serializer); db.InsertOrUpdate(dto, "SET data=@data, dataRaw=@dataRaw, rv=rv+1 WHERE nodeId=@id AND published=@published", new @@ -1375,7 +1377,7 @@ namespace Umbraco.Web.PublishedCache.NuCache } } - private ContentNuDto GetDto(IContentBase content, bool published) + private ContentNuDto GetDto(IContentBase content, bool published, IContentCacheDataSerializer serializer) { // should inject these in ctor // BUT for the time being we decide not to support ConvertDbToXml/String @@ -1447,19 +1449,21 @@ namespace Umbraco.Web.PublishedCache.NuCache } //the dictionary that will be serialized - var nestedData = new ContentNestedData + var contentCacheData = new ContentCacheDataModel { PropertyData = propertyData, CultureData = cultureData, UrlSegment = content.GetUrlSegment(_urlSegmentProviders) }; + var serialized = serializer.Serialize(content.ContentTypeId, contentCacheData); + var dto = new ContentNuDto { NodeId = content.Id, Published = published, - Data = !(_contentNestedDataSerializer is IContentNestedDataByteSerializer) ? _contentNestedDataSerializer.Serialize(content.ContentTypeId, nestedData) : null, - RawData = (_contentNestedDataSerializer is IContentNestedDataByteSerializer byteSerializer) ? byteSerializer.SerializeBytes(content.ContentTypeId, nestedData) : null + Data = serialized.StringData, + RawData = serialized.ByteData }; //Core.Composing.Current.Logger.Debug(dto.Data); @@ -1482,30 +1486,32 @@ namespace Umbraco.Web.PublishedCache.NuCache public override void Rebuild() { _logger.Debug("Rebuilding..."); + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document | ContentCacheDataSerializerEntityType.Media | ContentCacheDataSerializerEntityType.Member); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.ContentTree); scope.ReadLock(Constants.Locks.MediaTree); scope.ReadLock(Constants.Locks.MemberTree); - RebuildContentDbCacheLocked(scope, GetSqlPagingSize(), null); - RebuildMediaDbCacheLocked(scope, GetSqlPagingSize(), null); - RebuildMemberDbCacheLocked(scope, GetSqlPagingSize(), null); + RebuildContentDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); + RebuildMediaDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); + RebuildMemberDbCacheLocked(serializer, scope, GetSqlPagingSize(), null); scope.Complete(); } } public void RebuildContentDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.ContentTree); - RebuildContentDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildContentDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes content tree lock - private void RebuildContentDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + private void RebuildContentDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var contentObjectType = Constants.ObjectTypes.Document; @@ -1552,11 +1558,11 @@ WHERE cmsContentNu.nodeId IN ( foreach (var c in descendants) { // always the edited version - items.Add(GetDto(c, false)); + items.Add(GetDto(c, false, serializer)); // and also the published version if it makes any sense if (c.Published) - items.Add(GetDto(c, true)); + items.Add(GetDto(c, true, serializer)); count++; } @@ -1568,16 +1574,17 @@ WHERE cmsContentNu.nodeId IN ( public void RebuildMediaDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.MediaTree); - RebuildMediaDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildMediaDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes media tree lock - public void RebuildMediaDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + public void RebuildMediaDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var mediaObjectType = Constants.ObjectTypes.Media; @@ -1619,7 +1626,7 @@ WHERE cmsContentNu.nodeId IN ( { // the tree is locked, counting and comparing to total is safe var descendants = _mediaRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path")); - var items = descendants.Select(m => GetDto(m, false)).ToList(); + var items = descendants.Select(m => GetDto(m, false, serializer)).ToList(); db.BulkInsertRecords(items); processed += items.Count; } while (processed < total); @@ -1627,16 +1634,17 @@ WHERE cmsContentNu.nodeId IN ( public void RebuildMemberDbCache(int groupSize = DefaultSqlPagingSize, IEnumerable contentTypeIds = null) { + var serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Member); using (var scope = _scopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped)) { scope.ReadLock(Constants.Locks.MemberTree); - RebuildMemberDbCacheLocked(scope, groupSize, contentTypeIds); + RebuildMemberDbCacheLocked(serializer, scope, groupSize, contentTypeIds); scope.Complete(); } } // assumes member tree lock - public void RebuildMemberDbCacheLocked(IScope scope, int groupSize, IEnumerable contentTypeIds) + public void RebuildMemberDbCacheLocked(IContentCacheDataSerializer serializer, IScope scope, int groupSize, IEnumerable contentTypeIds) { var contentTypeIdsA = contentTypeIds?.ToArray(); var memberObjectType = Constants.ObjectTypes.Member; @@ -1677,7 +1685,7 @@ WHERE cmsContentNu.nodeId IN ( do { var descendants = _memberRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path")); - var items = descendants.Select(m => GetDto(m, false)).ToArray(); + var items = descendants.Select(m => GetDto(m, false, serializer)).ToArray(); db.BulkInsertRecords(items); processed += items.Length; } while (processed < total); diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 25478bf626..e93caaac66 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -262,12 +262,15 @@ - - + + + + + @@ -277,6 +280,7 @@ + @@ -588,7 +592,7 @@ - +