From bb654360bf47c0d0b586564fe746ae583a1c8bc0 Mon Sep 17 00:00:00 2001 From: Sven Geusens Date: Wed, 31 Jul 2024 10:35:28 +0200 Subject: [PATCH 1/2] Improve migration performance (#16784) * Improve migration performance * Fix PR review comments * Revert tags migration for sql lite as the optimized sql doesn't work properly and sqlLite datasets should not be large anyway --- .../MigrateTagsFromNVarcharToNText.cs | 25 +++++++++++++++++++ .../V_12_0_0/UseNvarcharInsteadOfNText.cs | 16 ++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_10_7_0/MigrateTagsFromNVarcharToNText.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_10_7_0/MigrateTagsFromNVarcharToNText.cs index 9bc9856fe7..318fead2d6 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_10_7_0/MigrateTagsFromNVarcharToNText.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_10_7_0/MigrateTagsFromNVarcharToNText.cs @@ -25,6 +25,18 @@ public class MigrateTagsFromNVarcharToNText : MigrationBase Database.Execute(updateDbTypeForTagsQuery); // Then migrate the data from "varcharValue" column to "textValue" + if (Database.DatabaseType == DatabaseType.SQLite) + { + MigrateSqlLiteData(); + } + else + { + MigrateSqlServerData(); + } + } + + private void MigrateSqlLiteData() + { Sql tagsDataTypeIdQuery = Database.SqlContext.Sql() .Select(dt => dt.NodeId) .From() @@ -44,4 +56,17 @@ public class MigrateTagsFromNVarcharToNText : MigrationBase Database.Execute(updatePropertyDataColumnsQuery); } + + private void MigrateSqlServerData() + { + Sql updateTagsValues = Database.SqlContext.Sql() + .Update() + .Append("SET textValue = COALESCE([textValue], [varCharValue]), varcharValue = null") + .From() + .InnerJoin().On((dt, pt) => dt.NodeId == pt.DataTypeId) + .InnerJoin().On((pt, pd) => pt.Id == pd.PropertyTypeId) + .Where(dt => dt.EditorAlias == Constants.PropertyEditors.Aliases.Tags); + + Database.Execute(updateTagsValues); + } } diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_12_0_0/UseNvarcharInsteadOfNText.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_12_0_0/UseNvarcharInsteadOfNText.cs index e761ccf8c4..aaa2137c61 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_12_0_0/UseNvarcharInsteadOfNText.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_12_0_0/UseNvarcharInsteadOfNText.cs @@ -32,6 +32,15 @@ public class UseNvarcharInsteadOfNText : MigrationBase MigrateNtextColumn("textValue", Constants.DatabaseSchema.Tables.PropertyData, x => x.TextValue); } + private void RawMigrateNtextColumn(string columnName, string tableName) + { + var updateTypeSql = @$" +ALTER TABLE {tableName} +ALTER COLUMN {columnName} nvarchar(max)"; + Sql copyDataQuery = Database.SqlContext.Sql(updateTypeSql); + Database.Execute(copyDataQuery); + } + private void MigrateNtextColumn(string columnName, string tableName, Expression> fieldSelector, bool nullable = true) { var columnType = ColumnType(tableName, columnName); @@ -40,6 +49,13 @@ public class UseNvarcharInsteadOfNText : MigrationBase return; } + // since it's ntext to nvarchar(max) we should be able to just raw query this without having issues with fallback values on sql server + if (Database.DatabaseType != DatabaseType.SQLite) + { + RawMigrateNtextColumn(columnName, tableName); + return; + } + var oldColumnName = $"Old{columnName}"; // Rename the column so we can create the new one and copy over the data. From 2736a513a3dd0a29aeac61ace614ff2effdf7135 Mon Sep 17 00:00:00 2001 From: Wojciech Tengler Date: Fri, 2 Aug 2024 11:21:48 +0200 Subject: [PATCH 2/2] Fixing regression bug introduced in #14806 and found during investigation of #16803 - well tested on db with 300k+ nodes (#16837) Co-authored-by: Wojciech Tengler (cherry picked from commit 688790e90ab28322d1e836e2b6542efabadc9149) --- .../Persistence/NuCacheContentRepository.cs | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs b/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs index f76b176d5e..0d67d2a8e3 100644 --- a/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs +++ b/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs @@ -221,10 +221,17 @@ AND cmsContentNu.nodeId IS NULL .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); + // Use a more efficient COUNT query + Sql? sqlCountQuery = SqlContentSourcesCount() + .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)); + + Sql? sqlCount = + SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); - IEnumerable dtos = GetContentNodeDtos(sql); + IEnumerable dtos = GetContentNodeDtos(sql, sqlCount); foreach (ContentSourceDto row in dtos) { @@ -239,10 +246,18 @@ AND cmsContentNu.nodeId IS NULL .Append(SqlWhereNodeIdX(SqlContext, id)) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); + // Use a more efficient COUNT query + Sql? sqlCountQuery = SqlContentSourcesCount(SqlContentSourcesSelectUmbracoNodeJoin) + .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)) + .Append(SqlWhereNodeIdX(SqlContext, id)); + + Sql? sqlCount = + SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); - IEnumerable dtos = GetContentNodeDtos(sql); + IEnumerable dtos = GetContentNodeDtos(sql, sqlCount); foreach (ContentSourceDto row in dtos) { @@ -262,10 +277,18 @@ AND cmsContentNu.nodeId IS NULL .WhereIn(x => x.ContentTypeId, ids) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); + // Use a more efficient COUNT query + Sql sqlCountQuery = SqlContentSourcesCount() + .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)) + .WhereIn(x => x.ContentTypeId, ids); + + Sql? sqlCount = + SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); - IEnumerable dtos = GetContentNodeDtos(sql); + IEnumerable dtos = GetContentNodeDtos(sql, sqlCount); foreach (ContentSourceDto row in dtos) { @@ -1015,27 +1038,14 @@ WHERE cmsContentNu.nodeId IN ( return dtos; } - private IEnumerable GetContentNodeDtos(Sql sql) + private IEnumerable GetContentNodeDtos(Sql sql, Sql sqlCount) { // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. // QueryPaged is very slow on large sites however, so use fetch if UsePagedSqlQuery is disabled. - IEnumerable dtos; - if (_nucacheSettings.Value.UsePagedSqlQuery) - { - // Use a more efficient COUNT query - Sql? sqlCountQuery = SqlContentSourcesCount() - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)); - - Sql? sqlCount = - SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - - dtos = Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount); - } - else - { - dtos = Database.Fetch(sql); - } + IEnumerable dtos = _nucacheSettings.Value.UsePagedSqlQuery ? + Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount) : + Database.Fetch(sql); return dtos; }