From 6f2dccc2e2d4278fcf748986f0f7beefffbd95a3 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 25 Aug 2020 10:14:36 +1000 Subject: [PATCH] Uses a more optimized COUNT query when rebuilding the in memory cache --- src/Umbraco.Core/Constants-SqlTemplates.cs | 6 +++ .../Persistence/NPocoDatabaseExtensions.cs | 23 +++++++-- .../NuCache/DataSource/DatabaseDataSource.cs | 47 ++++++++++++++++--- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Core/Constants-SqlTemplates.cs b/src/Umbraco.Core/Constants-SqlTemplates.cs index 984bc495b0..6940539cb6 100644 --- a/src/Umbraco.Core/Constants-SqlTemplates.cs +++ b/src/Umbraco.Core/Constants-SqlTemplates.cs @@ -15,6 +15,12 @@ public const string GetReservedId = "Umbraco.Core.VersionableRepository.GetReservedId"; } + + internal static class NuCacheDatabaseDataSource + { + public const string ContentSourcesSelect1 = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesSelect1"; + public const string ContentSourcesCount = "Umbraco.Web.PublishedCache.NuCache.DataSource.ContentSourcesCount"; + } } } } diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs index 152dcbe6d3..c2100d97ad 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions.cs @@ -27,12 +27,13 @@ namespace Umbraco.Core.Persistence /// The number of rows to load per page /// /// + /// Specify a custom Sql command to get the total count, if null is specified than the auto-generated sql count will be used /// /// /// NPoco's normal Page returns a List{T} but sometimes we don't want all that in memory and instead want to /// iterate over each row with a reader using Query vs Fetch. /// - internal static IEnumerable QueryPaged(this IDatabase database, long pageSize, Sql sql) + internal static IEnumerable QueryPaged(this IDatabase database, long pageSize, Sql sql, Sql sqlCount) { var sqlString = sql.SQL; var sqlArgs = sql.Arguments; @@ -42,12 +43,12 @@ namespace Umbraco.Core.Persistence do { // Get the paged queries - database.BuildPageQueries(pageIndex * pageSize, pageSize, sqlString, ref sqlArgs, out var sqlCount, out var sqlPage); + database.BuildPageQueries(pageIndex * pageSize, pageSize, sqlString, ref sqlArgs, out var generatedSqlCount, out var sqlPage); // get the item count once if (itemCount == null) { - itemCount = database.ExecuteScalar(sqlCount, sqlArgs); + itemCount = database.ExecuteScalar(sqlCount?.SQL ?? generatedSqlCount, sqlCount?.Arguments ?? sqlArgs); } pageIndex++; @@ -60,6 +61,22 @@ namespace Umbraco.Core.Persistence } while ((pageIndex * pageSize) < itemCount); } + /// + /// Iterates over the result of a paged data set with a db reader + /// + /// + /// + /// + /// The number of rows to load per page + /// + /// + /// + /// + /// NPoco's normal Page returns a List{T} but sometimes we don't want all that in memory and instead want to + /// iterate over each row with a reader using Query vs Fetch. + /// + internal static IEnumerable QueryPaged(this IDatabase database, long pageSize, Sql sql) => database.QueryPaged(pageSize, sql, null); + // NOTE // // proper way to do it with TSQL and SQLCE diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs index 19998c7956..77cb82b226 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs @@ -32,9 +32,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource private Sql ContentSourcesSelect(IScope scope, Func, Sql> joins = null) { - var sql = scope.SqlContext.Sql() - - .Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), + var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesSelect1, tsql => + tsql.Select(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"), x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"), x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId")) .AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId")) @@ -52,7 +51,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource .AndSelect("nuEdit", x => Alias(x.RawData, "EditDataRaw")) .AndSelect("nuPub", x => Alias(x.RawData, "PubDataRaw")) - .From(); + .From()); + + var sql = sqlTemplate.Sql(); + + // TODO: I'm unsure how we can format the below into SQL templates also because right.Current and right.Published end up being parameters if (joins != null) sql = joins(sql); @@ -74,6 +77,32 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return sql; } + /// + /// Returns a slightly more optimized query to use for the document counting when paging over the content sources + /// + /// + /// + private Sql ContentSourcesCount(IScope scope) + { + var sqlTemplate = scope.SqlContext.Templates.Get(Constants.SqlTemplates.NuCacheDatabaseDataSource.ContentSourcesCount, tsql => + tsql.Select(x => Alias(x.NodeId, "Id")) + .From() + .InnerJoin().On((left, right) => left.NodeId == right.NodeId) + .InnerJoin().On((left, right) => left.NodeId == right.NodeId)); + + var sql = sqlTemplate.Sql(); + + // TODO: We can't use a template with this one because of the 'right.Current' and 'right.Published' ends up being a parameter so not sure how we can do that + sql = sql + .InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current) + .InnerJoin().On((left, right) => left.Id == right.Id) + .LeftJoin(j => + j.InnerJoin("pdver").On((left, right) => left.Id == right.Id && right.Published, "pcver", "pdver"), "pcver") + .On((left, right) => left.NodeId == right.NodeId, aliasRight: "pcver"); + + return sql; + } + public ContentNodeKit GetContentSource(IScope scope, int id) { var sql = ContentSourcesSelect(scope) @@ -86,14 +115,20 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource public IEnumerable GetAllContentSources(IScope scope) { + // Create a different query for the SQL vs the COUNT Sql since the auto-generated COUNT Sql will be inneficient var sql = ContentSourcesSelect(scope) .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed) .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder); + // create a more efficient COUNT query without the join on the cmsContentNu table + var sqlCountQuery = ContentSourcesCount(scope) + .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed); + var sqlCount = scope.SqlContext.Sql("SELECT COUNT(*) FROM (").Append(sqlCountQuery).Append(") npoco_tbl"); + // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout. // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that. - foreach (var row in scope.Database.QueryPaged(PageSize, sql)) + foreach (var row in scope.Database.QueryPaged(PageSize, sql, sqlCount)) yield return CreateContentNodeKit(row); } @@ -319,6 +354,6 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return s; } - + } }