yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add UsePagedSqlQuery setting to NucacheSettings (#14806)

Browse Source 
* imlement UsePagedSqlQuery

* Remove unneccesary spacing
This commit is contained in:
Nikolaj Geisle
2023-09-14 20:11:08 +02:00
committed by GitHub
parent 0a4cc27a5d
commit 8ccc2faee0
2 changed files with 47 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/Umbraco.Core/Configuration/Models/NuCacheSettings.cs
View File

@@ -14,6 +14,7 @@ public class NuCacheSettings
internal const string StaticNuCacheSerializerType = "MessagePack";
internal const int StaticSqlPageSize = 1000;
internal const int StaticKitBatchSize = 1;
internal const bool StaticUsePagedSqlQuery = true;
/// <summary>
/// Gets or sets a value defining the BTree block size.
@@ -40,4 +41,7 @@ public class NuCacheSettings
public int KitBatchSize { get; set; } = StaticKitBatchSize;
public bool UnPublishedContentCompression { get; set; } = false;
[DefaultValue(StaticUsePagedSqlQuery)]
public bool UsePagedSqlQuery { get; set; } = true;
}

102
src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs
View File

@@ -221,19 +221,12 @@ AND cmsContentNu.nodeId IS NULL
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document))
.Append(SqlOrderByLevelIdSortOrder(SqlContext));
// Use a more efficient COUNT query
Sql<ISqlContext>? sqlCountQuery = SqlContentSourcesCount()
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document));
Sql<ISqlContext>? sqlCount =
SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
IContentCacheDataSerializer serializer =
_contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (ContentSourceDto row in Database.QueryPaged<ContentSourceDto>(_nucacheSettings.Value.SqlPageSize, sql, sqlCount))
IEnumerable<ContentSourceDto> dtos = GetContentNodeDtos(sql);
foreach (ContentSourceDto row in dtos)
{
yield return CreateContentNodeKit(row, serializer);
}
@@ -246,19 +239,12 @@ AND cmsContentNu.nodeId IS NULL
.Append(SqlWhereNodeIdX(SqlContext, id))
.Append(SqlOrderByLevelIdSortOrder(SqlContext));
// Use a more efficient COUNT query
Sql<ISqlContext>? sqlCountQuery = SqlContentSourcesCount(SqlContentSourcesSelectUmbracoNodeJoin)
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document))
.Append(SqlWhereNodeIdX(SqlContext, id));
Sql<ISqlContext>? sqlCount =
SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
IContentCacheDataSerializer serializer =
_contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (ContentSourceDto row in Database.QueryPaged<ContentSourceDto>(_nucacheSettings.Value.SqlPageSize, sql, sqlCount))
IEnumerable<ContentSourceDto> dtos = GetContentNodeDtos(sql);
foreach (ContentSourceDto row in dtos)
{
yield return CreateContentNodeKit(row, serializer);
}
@@ -276,19 +262,12 @@ AND cmsContentNu.nodeId IS NULL
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.Append(SqlOrderByLevelIdSortOrder(SqlContext));
// Use a more efficient COUNT query
Sql<ISqlContext> sqlCountQuery = SqlContentSourcesCount()
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document))
.WhereIn<ContentDto>(x => x.ContentTypeId, ids);
Sql<ISqlContext>? sqlCount =
SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
IContentCacheDataSerializer serializer =
_contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Document);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (ContentSourceDto row in Database.QueryPaged<ContentSourceDto>(_nucacheSettings.Value.SqlPageSize, sql, sqlCount))
IEnumerable<ContentSourceDto> dtos = GetContentNodeDtos(sql);
foreach (ContentSourceDto row in dtos)
{
yield return CreateContentNodeKit(row, serializer);
}
@@ -319,18 +298,12 @@ AND cmsContentNu.nodeId IS NULL
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media))
.Append(SqlOrderByLevelIdSortOrder(SqlContext));
// Use a more efficient COUNT query
Sql<ISqlContext>? sqlCountQuery = SqlMediaSourcesCount()
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media));
Sql<ISqlContext>? sqlCount =
SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
IContentCacheDataSerializer serializer =
_contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (ContentSourceDto row in Database.QueryPaged<ContentSourceDto>(_nucacheSettings.Value.SqlPageSize, sql, sqlCount))
IEnumerable<ContentSourceDto> dtos = GetContentNodeDtos(sql);
foreach (ContentSourceDto row in dtos)
{
yield return CreateMediaNodeKit(row, serializer);
}
@@ -343,19 +316,12 @@ AND cmsContentNu.nodeId IS NULL
.Append(SqlWhereNodeIdX(SqlContext, id))
.Append(SqlOrderByLevelIdSortOrder(SqlContext));
// Use a more efficient COUNT query
Sql<ISqlContext>? sqlCountQuery = SqlMediaSourcesCount(SqlContentSourcesSelectUmbracoNodeJoin)
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media))
.Append(SqlWhereNodeIdX(SqlContext, id));
Sql<ISqlContext>? sqlCount =
SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
IContentCacheDataSerializer serializer =
_contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (ContentSourceDto row in Database.QueryPaged<ContentSourceDto>(_nucacheSettings.Value.SqlPageSize, sql, sqlCount))
IEnumerable<ContentSourceDto> dtos = GetContentNodeDtos(sql);
foreach (ContentSourceDto row in dtos)
{
yield return CreateMediaNodeKit(row, serializer);
}
@@ -373,19 +339,12 @@ AND cmsContentNu.nodeId IS NULL
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.Append(SqlOrderByLevelIdSortOrder(SqlContext));
// Use a more efficient COUNT query
Sql<ISqlContext> sqlCountQuery = SqlMediaSourcesCount()
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Media))
.WhereIn<ContentDto>(x => x.ContentTypeId, ids);
Sql<ISqlContext>? sqlCount =
SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
IContentCacheDataSerializer serializer =
_contentCacheDataSerializerFactory.Create(ContentCacheDataSerializerEntityType.Media);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (ContentSourceDto row in Database.QueryPaged<ContentSourceDto>(_nucacheSettings.Value.SqlPageSize, sql, sqlCount))
IEnumerable<ContentSourceDto> dtos = GetContentNodeDtos(sql);
foreach (ContentSourceDto row in dtos)
{
yield return CreateMediaNodeKit(row, serializer);
}
@@ -1030,4 +989,29 @@ WHERE cmsContentNu.nodeId IN (
return s;
}
private IEnumerable<ContentSourceDto> GetContentNodeDtos(Sql<ISqlContext> sql)
{
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
// QueryPaged is very slow on large sites however, so use fetch if UsePagedSqlQuery is disabled.
IEnumerable<ContentSourceDto> dtos;
if (_nucacheSettings.Value.UsePagedSqlQuery)
{
// Use a more efficient COUNT query
Sql<ISqlContext>? sqlCountQuery = SqlContentSourcesCount()
.Append(SqlObjectTypeNotTrashed(SqlContext, Constants.ObjectTypes.Document));
Sql<ISqlContext>? sqlCount =
SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
dtos = Database.QueryPaged<ContentSourceDto>(_nucacheSettings.Value.SqlPageSize, sql, sqlCount);
}
else
{
dtos = Database.Fetch<ContentSourceDto>(sql);
}
return dtos;
}
}