From fba537928a44a95a994c9bdf209e589fcfb124da Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 11 Dec 2019 14:29:50 +0100 Subject: [PATCH 1/3] Adds a couple missing indexes --- .../Migrations/Upgrade/UmbracoPlan.cs | 1 + ...AddPropertyTypeValidationMessageColumns.cs | 1 + .../V_8_6_0/MissingContentVersionsIndexes.cs | 30 +++++++++++++++++++ .../Persistence/Dtos/ContentVersionDto.cs | 3 +- src/Umbraco.Core/Umbraco.Core.csproj | 1 + 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs diff --git a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs index 223603be14..3b2005bef6 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/UmbracoPlan.cs @@ -185,6 +185,7 @@ namespace Umbraco.Core.Migrations.Upgrade // to 8.6.0 To("{3D67D2C8-5E65-47D0-A9E1-DC2EE0779D6B}"); + To("{EE288A91-531B-4995-8179-1D62D9AA3E2E}"); //FINAL } diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/AddPropertyTypeValidationMessageColumns.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/AddPropertyTypeValidationMessageColumns.cs index 30eb30109e..f44695da69 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/AddPropertyTypeValidationMessageColumns.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/AddPropertyTypeValidationMessageColumns.cs @@ -3,6 +3,7 @@ using Umbraco.Core.Persistence.Dtos; namespace Umbraco.Core.Migrations.Upgrade.V_8_6_0 { + public class AddPropertyTypeValidationMessageColumns : MigrationBase { public AddPropertyTypeValidationMessageColumns(IMigrationContext context) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs new file mode 100644 index 0000000000..8d80f4f0da --- /dev/null +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs @@ -0,0 +1,30 @@ +using Umbraco.Core.Persistence.Dtos; + +namespace Umbraco.Core.Migrations.Upgrade.V_8_6_0 +{ + public class MissingContentVersionsIndexes : MigrationBase + { + public MissingContentVersionsIndexes(IMigrationContext context) : base(context) + { + } + + public override void Migrate() + { + Create + .Index("IX_" + ContentVersionDto.TableName + "_NodeId") + .OnTable(ContentVersionDto.TableName) + .OnColumn("nodeId") + .Ascending() + .WithOptions().NonClustered() + .Do(); + + Create + .Index("IX_" + ContentVersionDto.TableName + "_Current") + .OnTable(ContentVersionDto.TableName) + .OnColumn("current") + .Ascending() + .WithOptions().NonClustered() + .Do(); + } + } +} diff --git a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs index 3c2c3deda4..c31a8aefdc 100644 --- a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs @@ -19,6 +19,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("nodeId")] [ForeignKey(typeof(ContentDto))] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_NodeId")] public int NodeId { get; set; } [Column("versionDate")] // TODO: db rename to 'updateDate' @@ -30,8 +31,8 @@ namespace Umbraco.Core.Persistence.Dtos [NullSetting(NullSetting = NullSettings.Null)] public int? UserId { get => _userId == 0 ? null : _userId; set => _userId = value; } //return null if zero - // TODO: we need an index on this it is used almost always in querying and sorting [Column("current")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_Current")] public bool Current { get; set; } // about current: diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 6f9ff1e783..25eaa39430 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -128,6 +128,7 @@ --> + From 2b14a40e45929e6506bdf4085f43a2a4fc4bac9d Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 11 Dec 2019 16:30:27 +0100 Subject: [PATCH 2/3] Changes index to be a combined index since that is what is required --- .../Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs | 7 +------ src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs b/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs index 8d80f4f0da..b69ac8f9dc 100644 --- a/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs +++ b/src/Umbraco.Core/Migrations/Upgrade/V_8_6_0/MissingContentVersionsIndexes.cs @@ -15,16 +15,11 @@ namespace Umbraco.Core.Migrations.Upgrade.V_8_6_0 .OnTable(ContentVersionDto.TableName) .OnColumn("nodeId") .Ascending() - .WithOptions().NonClustered() - .Do(); - - Create - .Index("IX_" + ContentVersionDto.TableName + "_Current") - .OnTable(ContentVersionDto.TableName) .OnColumn("current") .Ascending() .WithOptions().NonClustered() .Do(); + } } } diff --git a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs index c31a8aefdc..4b203c128f 100644 --- a/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs +++ b/src/Umbraco.Core/Persistence/Dtos/ContentVersionDto.cs @@ -19,7 +19,7 @@ namespace Umbraco.Core.Persistence.Dtos [Column("nodeId")] [ForeignKey(typeof(ContentDto))] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_NodeId")] + [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_NodeId", ForColumns = "nodeId,current")] public int NodeId { get; set; } [Column("versionDate")] // TODO: db rename to 'updateDate' @@ -32,7 +32,6 @@ namespace Umbraco.Core.Persistence.Dtos public int? UserId { get => _userId == 0 ? null : _userId; set => _userId = value; } //return null if zero [Column("current")] - [Index(IndexTypes.NonClustered, Name = "IX_" + TableName + "_Current")] public bool Current { get; set; } // about current: From df84ec4fae4c9526eeeb8dc0d1710070ea0107d4 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 11 Dec 2019 16:31:03 +0100 Subject: [PATCH 3/3] Changes the DocumentRepository to be more granular in what data it loads and optimizes a few methods to not load erroneous data --- .../Implement/DocumentRepository.cs | 93 +++++++++++++------ 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs index dd9c7c93e5..9a66af6039 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs @@ -75,7 +75,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement if (ids.Any()) sql.WhereIn(x => x.NodeId, ids); - return MapDtosToContent(Database.Fetch(sql)); + return MapDtosToContent(Database.Fetch(sql), false, + // load everything + true, true, true, true); } protected override IEnumerable PerformGetByQuery(IQuery query) @@ -87,7 +89,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement AddGetByQueryOrderBy(sql); - return MapDtosToContent(Database.Fetch(sql)); + return MapDtosToContent(Database.Fetch(sql), false, + // load everything + true, true, true, true); } private void AddGetByQueryOrderBy(Sql sql) @@ -226,7 +230,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement .OrderByDescending(x => x.Current) .AndByDescending(x => x.VersionDate); - return MapDtosToContent(Database.Fetch(sql), true); + return MapDtosToContent(Database.Fetch(sql), true, true, true, true, true); } public override IEnumerable GetAllVersionsSlim(int nodeId, int skip, int take) @@ -236,7 +240,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement .OrderByDescending(x => x.Current) .AndByDescending(x => x.VersionDate); - return MapDtosToContent(Database.Fetch(sql), true, true).Skip(skip).Take(take); + return MapDtosToContent(Database.Fetch(sql), true, + // load bare minimum + false, false, false, false).Skip(skip).Take(take); } public override IContent GetVersion(int versionId) @@ -832,7 +838,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement } return GetPage(query, pageIndex, pageSize, out totalRecords, - x => MapDtosToContent(x), + x => MapDtosToContent(x, false, + // load properties but nothing else + true, false, false, false), filterSql, ordering); } @@ -919,7 +927,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement if (ids.Length > 0) sql.WhereIn(x => x.UniqueId, ids); - return _outerRepo.MapDtosToContent(Database.Fetch(sql)); + return _outerRepo.MapDtosToContent(Database.Fetch(sql), false, + // load everything + true, true, true, true); } protected override IEnumerable PerformGetByQuery(IQuery query) @@ -977,7 +987,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement AddGetByQueryOrderBy(sql); - return MapDtosToContent(Database.Fetch(sql)); + return MapDtosToContent(Database.Fetch(sql), + // load the bare minimum + false, false, false, false, false); } /// @@ -993,7 +1005,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement AddGetByQueryOrderBy(sql); - return MapDtosToContent(Database.Fetch(sql)); + return MapDtosToContent(Database.Fetch(sql), + // load the bare minimum + false, false, false, false, false); } #endregion @@ -1056,7 +1070,12 @@ namespace Umbraco.Core.Persistence.Repositories.Implement return base.ApplySystemOrdering(ref sql, ordering); } - private IEnumerable MapDtosToContent(List dtos, bool withCache = false, bool slim = false) + private IEnumerable MapDtosToContent(List dtos, + bool withCache, + bool loadProperties, + bool loadTemplates, + bool loadSchedule, + bool loadVariants) { var temps = new List>(); var contentTypes = new Dictionary(); @@ -1089,7 +1108,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement var c = content[i] = ContentBaseFactory.BuildEntity(dto, contentType); - if (!slim) + if (loadTemplates) { // need templates var templateId = dto.DocumentVersionDto.TemplateId; @@ -1114,49 +1133,71 @@ namespace Umbraco.Core.Persistence.Repositories.Implement temps.Add(temp); } - if (!slim) + Dictionary templates = null; + if (loadTemplates) { // load all required templates in 1 query, and index - var templates = _templateRepository.GetMany(templateIds.ToArray()) + templates = _templateRepository.GetMany(templateIds.ToArray()) .ToDictionary(x => x.Id, x => x); + } + IDictionary properties = null; + if (loadProperties) + { // load all properties for all documents from database in 1 query - indexed by version id - var properties = GetPropertyCollections(temps); - var schedule = GetContentSchedule(temps.Select(x => x.Content.Id).ToArray()); + properties = GetPropertyCollections(temps); + } - // assign templates and properties - foreach (var temp in temps) + var schedule = GetContentSchedule(temps.Select(x => x.Content.Id).ToArray()); + + // assign templates and properties + foreach (var temp in temps) + { + if (loadTemplates) { // set the template ID if it matches an existing template if (temp.Template1Id.HasValue && templates.ContainsKey(temp.Template1Id.Value)) temp.Content.TemplateId = temp.Template1Id; if (temp.Template2Id.HasValue && templates.ContainsKey(temp.Template2Id.Value)) temp.Content.PublishTemplateId = temp.Template2Id; + } + - // set properties + // set properties + if (loadProperties) + { if (properties.ContainsKey(temp.VersionId)) temp.Content.Properties = properties[temp.VersionId]; else throw new InvalidOperationException($"No property data found for version: '{temp.VersionId}'."); + } + if (loadSchedule) + { // load in the schedule if (schedule.TryGetValue(temp.Content.Id, out var s)) temp.Content.ContentSchedule = s; } + } - // set variations, if varying - temps = temps.Where(x => x.ContentType.VariesByCulture()).ToList(); - if (temps.Count > 0) + if (loadVariants) { - // load all variations for all documents from database, in one query - var contentVariations = GetContentVariations(temps); - var documentVariations = GetDocumentVariations(temps); - foreach (var temp in temps) - SetVariations(temp.Content, contentVariations, documentVariations); + // set variations, if varying + temps = temps.Where(x => x.ContentType.VariesByCulture()).ToList(); + if (temps.Count > 0) + { + // load all variations for all documents from database, in one query + var contentVariations = GetContentVariations(temps); + var documentVariations = GetDocumentVariations(temps); + foreach (var temp in temps) + SetVariations(temp.Content, contentVariations, documentVariations); + } } + - foreach(var c in content) + + foreach (var c in content) c.ResetDirtyProperties(false); // reset dirty initial properties (U4-1946) return content;