From 6bd6d97bdaccacaf34fe83261e7fe434a11a64e7 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 24 Sep 2020 18:37:24 +1000 Subject: [PATCH] WIP (fixes migrations) --- .../Migrations/Upgrade/UmbracoPlan.cs | 6 +- .../Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs | 50 -------------- .../AddCmsContentNuByteColumn.cs | 2 +- .../Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs | 69 +++++++++++++++++++ .../CompressedStorageAttribute.cs | 21 ++++++ ...StoragePropertyEditorCompressionOptions.cs | 44 ++++++++++++ .../PropertyEditors/DataEditorAttribute.cs | 1 + .../IPropertyCompressionOptions.cs | 2 +- src/Umbraco.Core/Umbraco.Core.csproj | 7 +- ...ComplexEditorPropertyCompressionOptions.cs | 48 ------------- .../NoopPropertyCompressionOptions.cs | 4 +- .../NuCache/DataSource/DatabaseDataSource.cs | 6 ++ .../MsgPackContentNestedDataSerializer.cs | 1 + .../PublishedCache/NuCache/NuCacheComposer.cs | 3 +- .../NuCache/PublishedSnapshotService.cs | 5 +- src/Umbraco.Web/Umbraco.Web.csproj | 2 - 16 files changed, 159 insertions(+), 112 deletions(-) delete mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs rename src/Umbraco.Core/Migrations/Upgrade/{V_8_7_0 => V_8_9_0}/AddCmsContentNuByteColumn.cs (90%) create mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs create mode 100644 src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs create mode 100644 src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs rename src/{Umbraco.Web => Umbraco.Core}/PropertyEditors/IPropertyCompressionOptions.cs (84%) delete mode 100644 src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index a15dcfb23b..cb35783ffa 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -7,7 +7,7 @@ using Umbraco.Core.Migrations.Upgrade.V_8_0_0; using Umbraco.Core.Migrations.Upgrade.V_8_0_1; using Umbraco.Core.Migrations.Upgrade.V_8_1_0; using Umbraco.Core.Migrations.Upgrade.V_8_6_0; -using Umbraco.Core.Migrations.Upgrade.V_8_8_0; +using Umbraco.Core.Migrations.Upgrade.V_8_9_0; namespace Umbraco.Core.Migrations.Upgrade { @@ -194,9 +194,9 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.7.0... To("{a78e3369-8ea3-40ec-ad3f-5f76929d2b20}"); - To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); - // to 8.8.0... + // to 8.9.0... + To("{8DDDCD0B-D7D5-4C97-BD6A-6B38CA65752F}"); To("{4695D0C9-0729-4976-985B-048D503665D8}"); //FINAL diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs deleted file mode 100644 index d4dd783876..0000000000 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_8_0/UpgradedIncludeIndexes.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Linq; -using Umbraco.Core.Migrations.Expressions.Execute.Expressions; -using Umbraco.Core.Persistence.DatabaseModelDefinitions; -using Umbraco.Core.Persistence.Dtos; - -namespace Umbraco.Core.Migrations.Upgrade.V_8_8_0 -{ - public class UpgradedIncludeIndexes : MigrationBase - { - public UpgradedIncludeIndexes(IMigrationContext context) - : base(context) - { - - } - - public override void Migrate() - { - var indexesToReplace = new[] { $"IX_{NodeDto.TableName}_UniqueId", $"IX_{NodeDto.TableName}_ObjectType" }; - DeleteIndexes(indexesToReplace); // delete existing ones - CreateIndexes(indexesToReplace); // replace - CreateIndexes($"IX_{NodeDto.TableName}_Level"); // add the new definitions - - var contentVersionNodeIdIndex = $"IX_{ContentVersionDto.TableName}_NodeId"; - DeleteIndexes(contentVersionNodeIdIndex); // delete existing ones - CreateIndexes(contentVersionNodeIdIndex); // replace - CreateIndexes($"IX_{ContentVersionDto.TableName}_Current"); // add the new definitions - } - - private void DeleteIndexes(params string[] toDelete) - { - var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); - - foreach (var i in toDelete) - Delete.Index(i).OnTable(tableDef.Name).Do(); - } - - private void CreateIndexes(params string[] toCreate) - { - var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); - - foreach(var c in toCreate) - { - // get the definition by name - var index = tableDef.Indexes.First(x => x.Name == c); - new ExecuteSqlStatementExpression(Context) { SqlStatement = Context.SqlContext.SqlSyntax.Format(index) }.Execute(); - } - - } - } -} diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs similarity index 90% rename from src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs rename to src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs index 3cb51afb99..c295e13051 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_7_0/AddCmsContentNuByteColumn.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/AddCmsContentNuByteColumn.cs @@ -1,7 +1,7 @@ using System.Linq; using Umbraco.Core.Persistence.Dtos; -namespace Umbraco.Core.Migrations.Upgrade.V_8_6_0 +namespace Umbraco.Core.Migrations.Upgrade.V_8_9_0 { public class AddCmsContentNuByteColumn : MigrationBase { diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs new file mode 100644 index 0000000000..083294d1a5 --- /dev/null +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_9_0/UpgradedIncludeIndexes.cs @@ -0,0 +1,69 @@ +using System.Linq; +using Umbraco.Core.Migrations.Expressions.Execute.Expressions; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; +using Umbraco.Core.Persistence.Dtos; + +namespace Umbraco.Core.Migrations.Upgrade.V_8_9_0 +{ + public class UpgradedIncludeIndexes : MigrationBase + { + public UpgradedIncludeIndexes(IMigrationContext context) + : base(context) + { + + } + + public override void Migrate() + { + // Need to drop the FK for the redirect table before modifying the unique id index + Delete.ForeignKey() + .FromTable(Constants.DatabaseSchema.Tables.RedirectUrl) + .ForeignColumn("contentKey") + .ToTable(NodeDto.TableName) + .PrimaryColumn("uniqueID") + .Do(); + var nodeDtoIndexes = new[] { $"IX_{NodeDto.TableName}_UniqueId", $"IX_{NodeDto.TableName}_ObjectType", $"IX_{NodeDto.TableName}_Level" }; + DeleteIndexes(nodeDtoIndexes); // delete existing ones + CreateIndexes(nodeDtoIndexes); // update/add + // Now re-create the FK for the redirect table + Create.ForeignKey() + .FromTable(Constants.DatabaseSchema.Tables.RedirectUrl) + .ForeignColumn("contentKey") + .ToTable(NodeDto.TableName) + .PrimaryColumn("uniqueID") + .Do(); + + + var contentVersionIndexes = new[] { $"IX_{ContentVersionDto.TableName}_NodeId", $"IX_{ContentVersionDto.TableName}_Current" }; + DeleteIndexes(contentVersionIndexes); // delete existing ones + CreateIndexes(contentVersionIndexes); // update/add + } + + private void DeleteIndexes(params string[] toDelete) + { + var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); + + foreach (var i in toDelete) + { + if (IndexExists(i)) + { + Delete.Index(i).OnTable(tableDef.Name).Do(); + } + } + + } + + private void CreateIndexes(params string[] toCreate) + { + var tableDef = DefinitionFactory.GetTableDefinition(typeof(T), Context.SqlContext.SqlSyntax); + + foreach (var c in toCreate) + { + // get the definition by name + var index = tableDef.Indexes.First(x => x.Name == c); + new ExecuteSqlStatementExpression(Context) { SqlStatement = Context.SqlContext.SqlSyntax.Format(index) }.Execute(); + } + + } + } +} diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs b/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs new file mode 100644 index 0000000000..31689c4ee9 --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/CompressedStorageAttribute.cs @@ -0,0 +1,21 @@ +using System; + +namespace Umbraco.Core.PropertyEditors +{ + /// + /// When assigned to a DataEditor it indicates that the values it generates can be compressed + /// + /// + /// Used in conjunction with + /// + [AttributeUsage(AttributeTargets.Class)] + public sealed class CompressedStorageAttribute : Attribute + { + public CompressedStorageAttribute(bool isCompressed = true) + { + IsCompressed = isCompressed; + } + + public bool IsCompressed { get; } + } +} diff --git a/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs new file mode 100644 index 0000000000..15795bb61c --- /dev/null +++ b/src/Umbraco.Core/PropertyEditors/CompressedStoragePropertyEditorCompressionOptions.cs @@ -0,0 +1,44 @@ +using System.Collections.Concurrent; +using System.Linq; +using Umbraco.Core.Services; + +namespace Umbraco.Core.PropertyEditors +{ + + /// + /// Ensures all property types that have an editor storing a complex value are compressed + /// + public class CompressedStoragePropertyEditorCompressionOptions : IPropertyCompressionOptions + { + private readonly IContentTypeService _contentTypeService; + private readonly PropertyEditorCollection _propertyEditors; + private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); + + public CompressedStoragePropertyEditorCompressionOptions(PropertyEditorCollection propertyEditors) + { + _propertyEditors = propertyEditors; + } + + public bool IsCompressed(int contentTypeId, string alias) + { + return false; + //var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => + //{ + // var ct = _contentTypeService.Get(contentTypeId); + // if (ct == null) return null; + + // var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); + // if (propertyType == null) return null; + + // if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; + + // var editor = propertyEditor.GetValueEditor(); + // if (editor == null) return null; + + // return editor.ValueType; + //}); + + //return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; + } + } +} diff --git a/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs b/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs index 7b3be7ea5f..0375509797 100644 --- a/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs +++ b/src/Umbraco.Core/PropertyEditors/DataEditorAttribute.cs @@ -2,6 +2,7 @@ namespace Umbraco.Core.PropertyEditors { + /// /// Marks a class that represents a data editor. /// diff --git a/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs similarity index 84% rename from src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs rename to src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs index 70f3e7c6f0..f5aaf3dc57 100644 --- a/src/Umbraco.Web/PropertyEditors/IPropertyCompressionOptions.cs +++ b/src/Umbraco.Core/PropertyEditors/IPropertyCompressionOptions.cs @@ -1,4 +1,4 @@ -namespace Umbraco.Web.PropertyEditors +namespace Umbraco.Core.PropertyEditors { /// /// Determines if a property type's value should be compressed diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 4a25863158..444ef1a464 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -132,8 +132,8 @@ - - + + @@ -152,6 +152,9 @@ + + + diff --git a/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs deleted file mode 100644 index f4776f652d..0000000000 --- a/src/Umbraco.Web/PropertyEditors/ComplexEditorPropertyCompressionOptions.cs +++ /dev/null @@ -1,48 +0,0 @@ -using K4os.Compression.LZ4; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using Umbraco.Core.PropertyEditors; -using Umbraco.Core.Services; - -namespace Umbraco.Web.PropertyEditors -{ - - /// - /// Ensures all property types that have an editor storing a complex value are compressed - /// - public class ComplexEditorPropertyCompressionOptions : IPropertyCompressionOptions - { - private readonly IContentTypeService _contentTypeService; - private readonly PropertyEditorCollection _propertyEditors; - private readonly ConcurrentDictionary<(int, string), string> _editorValueTypes = new ConcurrentDictionary<(int, string), string>(); - - public ComplexEditorPropertyCompressionOptions(IContentTypeService contentTypeService, PropertyEditorCollection propertyEditors) - { - _contentTypeService = contentTypeService; - _propertyEditors = propertyEditors; - } - - public bool IsCompressed(int contentTypeId, string alias) - { - var valueType = _editorValueTypes.GetOrAdd((contentTypeId, alias), x => - { - var ct = _contentTypeService.Get(contentTypeId); - if (ct == null) return null; - - var propertyType = ct.CompositionPropertyTypes.FirstOrDefault(x => x.Alias == alias); - if (propertyType == null) return null; - - if (!_propertyEditors.TryGet(propertyType.PropertyEditorAlias, out var propertyEditor)) return null; - - var editor = propertyEditor.GetValueEditor(); - if (editor == null) return null; - - return editor.ValueType; - }); - - return valueType == ValueTypes.Json || valueType == ValueTypes.Xml || valueType == ValueTypes.Text; - } - } -} diff --git a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs index 638306d7ca..6f626938bc 100644 --- a/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs +++ b/src/Umbraco.Web/PropertyEditors/NoopPropertyCompressionOptions.cs @@ -1,4 +1,6 @@ -namespace Umbraco.Web.PropertyEditors +using Umbraco.Core.PropertyEditors; + +namespace Umbraco.Web.PropertyEditors { /// /// Disables all compression for all properties diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index a52fde2f0a..0b3003d7ec 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -129,7 +129,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) + { yield return CreateContentNodeKit(row); + } } public IEnumerable GetBranchContentSources(IScope scope, int id) @@ -145,7 +147,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + { yield return CreateContentNodeKit(row); + } } public IEnumerable GetTypeContentSources(IScope scope, IEnumerable ids) @@ -161,7 +165,9 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + { yield return CreateContentNodeKit(row); + } } private Sql MediaSourcesSelect(IScope scope, Func, Sql> joins = null) diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs index 0ea2b96fbe..4965935fbf 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/MsgPackContentNestedDataSerializer.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text; +using Umbraco.Core.PropertyEditors; using Umbraco.Web.PropertyEditors; namespace Umbraco.Web.PublishedCache.NuCache.DataSource diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index c5109db027..baa96a4a2e 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -3,6 +3,7 @@ using System.Configuration; using System.Linq; using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Core.PropertyEditors; using Umbraco.Web.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -24,7 +25,7 @@ namespace Umbraco.Web.PublishedCache.NuCache else { composition.RegisterUnique(); - composition.RegisterUnique(); + composition.RegisterUnique(); } composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs index e670cb75f6..41dfdd7a64 100755 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -387,10 +387,9 @@ namespace Umbraco.Web.PublishedCache.NuCache // contentStore is wlocked (1 thread) // content (and types) are read-locked - var contentTypes = _serviceContext.ContentTypeService.GetAll() - .Select(x => _publishedContentTypeFactory.CreateContentType(x)); + var contentTypes = _serviceContext.ContentTypeService.GetAll().ToList(); - _contentStore.SetAllContentTypesLocked(contentTypes); + _contentStore.SetAllContentTypesLocked(contentTypes.Select(x => _publishedContentTypeFactory.CreateContentType(x))); using (_logger.TraceDuration("Loading content from database")) { diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 51f4d838af..25478bf626 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -262,7 +262,6 @@ - @@ -271,7 +270,6 @@ -