From 8ccc2faee075693a05737b2470c9a71da4dbd651 Mon Sep 17 00:00:00 2001 From: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Date: Thu, 14 Sep 2023 20:11:08 +0200 Subject: [PATCH] Add UsePagedSqlQuery setting to NucacheSettings (#14806) * imlement UsePagedSqlQuery * Remove unneccesary spacing --- .../Configuration/Models/NuCacheSettings.cs | 4 + .../Persistence/NuCacheContentRepository.cs | 102 ++++++++---------- 2 files changed, 47 insertions(+), 59 deletions(-) diff --git a/src/Umbraco.Core/Configuration/Models/NuCacheSettings.cs b/src/Umbraco.Core/Configuration/Models/NuCacheSettings.cs index b88dbb5d0d..cdf2557763 100644 --- a/src/Umbraco.Core/Configuration/Models/NuCacheSettings.cs +++ b/src/Umbraco.Core/Configuration/Models/NuCacheSettings.cs @@ -14,6 +14,7 @@ public class NuCacheSettings internal const string StaticNuCacheSerializerType = "MessagePack"; internal const int StaticSqlPageSize = 1000; internal const int StaticKitBatchSize = 1; + internal const bool StaticUsePagedSqlQuery = true; /// /// Gets or sets a value defining the BTree block size. @@ -40,4 +41,7 @@ public class NuCacheSettings public int KitBatchSize { get; set; } = StaticKitBatchSize; public bool UnPublishedContentCompression { get; set; } = false; + + [DefaultValue(StaticUsePagedSqlQuery)] + public bool UsePagedSqlQuery { get; set; } = true; } diff --git a/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs b/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs index 75208b2b16..af8c46821f 100644 --- a/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs +++ b/src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs @@ -221,19 +221,12 @@ AND cmsContentNu.nodeId IS NULL .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - // Use a more efficient COUNT query - Sql? sqlCountQuery = SqlContentSourcesCount() - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)); - - Sql? sqlCount = - SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); - // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. - // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (ContentSourceDto row in Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount)) + IEnumerable dtos = GetContentNodeDtos(sql); + + foreach (ContentSourceDto row in dtos) { yield return CreateContentNodeKit(row, serializer); } @@ -246,19 +239,12 @@ AND cmsContentNu.nodeId IS NULL .Append(SqlWhereNodeIdX(SqlContext, id)) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - // Use a more efficient COUNT query - Sql? sqlCountQuery = SqlContentSourcesCount(SqlContentSourcesSelectUmbracoNodeJoin) - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)) - .Append(SqlWhereNodeIdX(SqlContext, id)); - Sql? sqlCount = - SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); - // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. - // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (ContentSourceDto row in Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount)) + IEnumerable dtos = GetContentNodeDtos(sql); + + foreach (ContentSourceDto row in dtos) { yield return CreateContentNodeKit(row, serializer); } @@ -276,19 +262,12 @@ AND cmsContentNu.nodeId IS NULL .WhereIn(x => x.ContentTypeId, ids) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - // Use a more efficient COUNT query - Sql sqlCountQuery = SqlContentSourcesCount() - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)) - .WhereIn(x => x.ContentTypeId, ids); - Sql? sqlCount = - SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document); - // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. - // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (ContentSourceDto row in Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount)) + IEnumerable dtos = GetContentNodeDtos(sql); + + foreach (ContentSourceDto row in dtos) { yield return CreateContentNodeKit(row, serializer); } @@ -319,18 +298,12 @@ AND cmsContentNu.nodeId IS NULL .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media)) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - // Use a more efficient COUNT query - Sql? sqlCountQuery = SqlMediaSourcesCount() - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media)); - Sql? sqlCount = - SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); - // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. - // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (ContentSourceDto row in Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount)) + IEnumerable dtos = GetContentNodeDtos(sql); + + foreach (ContentSourceDto row in dtos) { yield return CreateMediaNodeKit(row, serializer); } @@ -343,19 +316,12 @@ AND cmsContentNu.nodeId IS NULL .Append(SqlWhereNodeIdX(SqlContext, id)) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - // Use a more efficient COUNT query - Sql? sqlCountQuery = SqlMediaSourcesCount(SqlContentSourcesSelectUmbracoNodeJoin) - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media)) - .Append(SqlWhereNodeIdX(SqlContext, id)); - Sql? sqlCount = - SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); - // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. - // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (ContentSourceDto row in Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount)) + IEnumerable dtos = GetContentNodeDtos(sql); + + foreach (ContentSourceDto row in dtos) { yield return CreateMediaNodeKit(row, serializer); } @@ -373,19 +339,12 @@ AND cmsContentNu.nodeId IS NULL .WhereIn(x => x.ContentTypeId, ids) .Append(SqlOrderByLevelIdSortOrder(SqlContext)); - // Use a more efficient COUNT query - Sql sqlCountQuery = SqlMediaSourcesCount() - .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media)) - .WhereIn(x => x.ContentTypeId, ids); - Sql? sqlCount = - SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); - IContentCacheDataSerializer serializer = _contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media); - // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. - // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (ContentSourceDto row in Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount)) + IEnumerable dtos = GetContentNodeDtos(sql); + + foreach (ContentSourceDto row in dtos) { yield return CreateMediaNodeKit(row, serializer); } @@ -1030,4 +989,29 @@ WHERE cmsContentNu.nodeId IN ( return s; } + + private IEnumerable GetContentNodeDtos(Sql sql) + { + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. + // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. + // QueryPaged is very slow on large sites however, so use fetch if UsePagedSqlQuery is disabled. + IEnumerable dtos; + if (_nucacheSettings.Value.UsePagedSqlQuery) + { + // Use a more efficient COUNT query + Sql? sqlCountQuery = SqlContentSourcesCount() + .Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document)); + + Sql? sqlCount = + SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + + dtos = Database.QueryPaged(_nucacheSettings.Value.SqlPageSize, sql, sqlCount); + } + else + { + dtos = Database.Fetch(sql); + } + + return dtos; + } }