yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uses a more optimized COUNT query when rebuilding the in memory cache

Browse Source
This commit is contained in:
Shannon
2020-08-25 10:14:36 +10:00
parent 666d67e562
commit 6f2dccc2e2
3 changed files with 67 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Umbraco.Core/Constants-SqlTemplates.cs
View File

@@ -15,6 +15,12 @@
public const string GetReservedId = "Umbraco.Core.VersionableRepository.GetReservedId";
}
internal static class NuCacheDatabaseDataSource
{
public const string ContentSourcesSelect1 = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelect1";
public const string ContentSourcesCount = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesCount";
}
}
}
}

23
src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs
View File

@@ -27,12 +27,13 @@ namespace Umbraco.Core.Persistence
/// The number of rows to load per page
/// </param>
/// <param name="sql"></param>
/// <param name="sqlCount">Specify a custom Sql command to get the total count, if null is specified than the auto-generated sql count will be used</param>
/// <returns></returns>
/// <remarks>
/// NPoco's normal Page returns a List{T} but sometimes we don't want all that in memory and instead want to
/// iterate over each row with a reader using Query vs Fetch.
/// </remarks>
internal static IEnumerable<T> QueryPaged<T>(this IDatabase database, long pageSize, Sql sql)
internal static IEnumerable<T> QueryPaged<T>(this IDatabase database, long pageSize, Sql sql, Sql sqlCount)
{
var sqlString = sql.SQL;
var sqlArgs = sql.Arguments;
@@ -42,12 +43,12 @@ namespace Umbraco.Core.Persistence
do
{
// Get the paged queries
database.BuildPageQueries<T>(pageIndex * pageSize, pageSize, sqlString, ref sqlArgs, out var sqlCount, out var sqlPage);
database.BuildPageQueries<T>(pageIndex * pageSize, pageSize, sqlString, ref sqlArgs, out var generatedSqlCount, out var sqlPage);
// get the item count once
if (itemCount == null)
{
itemCount = database.ExecuteScalar<int>(sqlCount, sqlArgs);
itemCount = database.ExecuteScalar<int>(sqlCount?.SQL ?? generatedSqlCount, sqlCount?.Arguments ?? sqlArgs);
}
pageIndex++;
@@ -60,6 +61,22 @@ namespace Umbraco.Core.Persistence
} while ((pageIndex * pageSize) < itemCount);
}
/// <summary>
/// Iterates over the result of a paged data set with a db reader
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="database"></param>
/// <param name="pageSize">
/// The number of rows to load per page
/// </param>
/// <param name="sql"></param>
/// <returns></returns>
/// <remarks>
/// NPoco's normal Page returns a List{T} but sometimes we don't want all that in memory and instead want to
/// iterate over each row with a reader using Query vs Fetch.
/// </remarks>
internal static IEnumerable<T> QueryPaged<T>(this IDatabase database, long pageSize, Sql sql) => database.QueryPaged<T>(pageSize, sql, null);
// NOTE
//
// proper way to do it with TSQL and SQLCE

47
src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
View File

@@ -32,9 +32,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
private Sql<ISqlContext> ContentSourcesSelect(IScope scope, Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
{
var sql = scope.SqlContext.Sql()
.Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelect1, tsql =>
tsql.Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
.AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
@@ -52,7 +51,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.AndSelect<ContentNuDto>("nuEdit", x => Alias(x.RawData, "EditDataRaw"))
.AndSelect<ContentNuDto>("nuPub", x => Alias(x.RawData, "PubDataRaw"))
.From<NodeDto>();
.From<NodeDto>());
var sql = sqlTemplate.Sql();
// TODO: I'm unsure how we can format the below into SQL templates also because right.Current and right.Published end up being parameters
if (joins != null)
sql = joins(sql);
@@ -74,6 +77,32 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
return sql;
}
/// <summary>
/// Returns a slightly more optimized query to use for the document counting when paging over the content sources
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
private Sql<ISqlContext> ContentSourcesCount(IScope scope)
{
var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesCount, tsql =>
tsql.Select<NodeDto>(x => Alias(x.NodeId, "Id"))
.From<NodeDto>()
.InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<DocumentDto>().On<NodeDto, DocumentDto>((left, right) => left.NodeId == right.NodeId));
var sql = sqlTemplate.Sql();
// TODO: We can't use a template with this one because of the 'right.Current' and 'right.Published' ends up being a parameter so not sure how we can do that
sql = sql
.InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
.InnerJoin<DocumentVersionDto>().On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id)
.LeftJoin<ContentVersionDto>(j =>
j.InnerJoin<DocumentVersionDto>("pdver").On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id && right.Published, "pcver", "pdver"), "pcver")
.On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId, aliasRight: "pcver");
return sql;
}
public ContentNodeKit GetContentSource(IScope scope, int id)
{
var sql = ContentSourcesSelect(scope)
@@ -86,14 +115,20 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
public IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope)
{
// Create a different query for the SQL vs the COUNT Sql since the auto-generated COUNT Sql will be inneficient
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// create a more efficient COUNT query without the join on the cmsContentNu table
var sqlCountQuery = ContentSourcesCount(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed);
var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl");
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql, sqlCount))
yield return CreateContentNodeKit(row);
}
@@ -319,6 +354,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
return s;
}
}
}