From c08f7f211c4ddac76b01389c4771ff33f841e1da Mon Sep 17 00:00:00 2001 From: Bjarke Berg Date: Tue, 31 May 2022 15:55:29 +0200 Subject: [PATCH] Query optimization in TrackedReferencesRepository (#12488) * Rewrote query https://github.com/umbraco/Umbraco-CMS/issues/12308 * Reuse new query * Updated the other queries in TrackedReferencesRepository * Fix GetPagedRelationsForItem to not include self * Update src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs Co-authored-by: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> Co-authored-by: Nikolaj Geisle <70372949+Zeegaan@users.noreply.github.com> --- .../Persistence/NPocoSqlExtensions.cs | 16 ++ .../Implement/TrackedReferencesRepository.cs | 188 ++++++++++++------ 2 files changed, 138 insertions(+), 66 deletions(-) diff --git a/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs b/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs index 47cca58ce2..86091c3133 100644 --- a/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs +++ b/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs @@ -88,6 +88,17 @@ namespace Umbraco.Extensions return sql; } + public static Sql Union(this Sql sql, Sql sql2) + { + return sql.Append( " UNION ").Append(sql2); + } + + public static Sql.SqlJoinClause InnerJoinNested(this Sql sql, Sql nestedQuery, string alias) + { + return new Sql.SqlJoinClause(sql.Append("INNER JOIN (").Append(nestedQuery) + .Append($") [{alias}]")); + } + public static Sql WhereLike(this Sql sql, Expression> fieldSelector, string likeValue) { var fieldName = sql.SqlContext.SqlSyntax.GetFieldName(fieldSelector); @@ -252,6 +263,11 @@ namespace Umbraco.Extensions return sql.OrderBy("(" + sql.SqlContext.SqlSyntax.GetFieldName(field) + ")"); } + public static Sql OrderBy(this Sql sql, Expression> field, string alias) + { + return sql.OrderBy("(" + sql.SqlContext.SqlSyntax.GetFieldName(field, alias) + ")"); + } + /// /// Appends an ORDER BY clause to the Sql statement. /// diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/TrackedReferencesRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/TrackedReferencesRepository.cs index 478018ed96..ea357134d4 100644 --- a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/TrackedReferencesRepository.cs +++ b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/TrackedReferencesRepository.cs @@ -24,38 +24,41 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement /// public IEnumerable GetPagedItemsWithRelations(int[] ids, long pageIndex, int pageSize, bool filterMustBeIsDependency, out long totalRecords) { + Sql innerUnionSql = GetInnerUnionSql(); var sql = _scopeAccessor.AmbientScope.Database.SqlContext.Sql().SelectDistinct( - "[pn].[id] as nodeId", - "[pn].[uniqueId] as nodeKey", - "[pn].[text] as nodeName", - "[pn].[nodeObjectType] as nodeObjectType", - "[ct].[icon] as contentTypeIcon", - "[ct].[alias] as contentTypeAlias", - "[ctn].[text] as contentTypeName", - "[umbracoRelationType].[alias] as relationTypeAlias", - "[umbracoRelationType].[name] as relationTypeName", - "[umbracoRelationType].[isDependency] as relationTypeIsDependency", - "[umbracoRelationType].[dual] as relationTypeIsBidirectional") - .From("r") - .InnerJoin("umbracoRelationType").On((left, right) => left.RelationType == right.Id, aliasLeft: "r", aliasRight: "umbracoRelationType") - .InnerJoin("cn").On((r, cn, rt) => (!rt.Dual && r.ParentId == cn.NodeId) || (rt.Dual && (r.ChildId == cn.NodeId || r.ParentId == cn.NodeId)), aliasLeft: "r", aliasRight: "cn", aliasOther: "umbracoRelationType") - .InnerJoin("pn").On((r, pn, cn) => (pn.NodeId == r.ChildId && cn.NodeId == r.ParentId) || (pn.NodeId == r.ParentId && cn.NodeId == r.ChildId), aliasLeft: "r", aliasRight: "pn", aliasOther: "cn") - .LeftJoin("c").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "pn", aliasRight: "c") - .LeftJoin("ct").On((left, right) => left.ContentTypeId == right.NodeId, aliasLeft: "c", aliasRight: "ct") - .LeftJoin("ctn").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "ct", aliasRight: "ctn"); - + "[x].[id] as nodeId", + "[n].[uniqueId] as nodeKey", + "[n].[text] as nodeName", + "[n].[nodeObjectType] as nodeObjectType", + "[ct].[icon] as contentTypeIcon", + "[ct].[alias] as contentTypeAlias", + "[ctn].[text] as contentTypeName", + "[x].[alias] as relationTypeAlias", + "[x].[name] as relationTypeName", + "[x].[isDependency] as relationTypeIsDependency", + "[x].[dual] as relationTypeIsBidirectional") + .From("n") + .InnerJoinNested(innerUnionSql, "x") + .On((n, x) => n.NodeId == x.Id, "n", "x") + .LeftJoin("c").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "n", + aliasRight: "c") + .LeftJoin("ct") + .On((left, right) => left.ContentTypeId == right.NodeId, aliasLeft: "c", + aliasRight: "ct") + .LeftJoin("ctn").On((left, right) => left.NodeId == right.NodeId, + aliasLeft: "ct", aliasRight: "ctn"); if (ids.Any()) { - sql = sql.Where(x => ids.Contains(x.NodeId), "pn"); + sql = sql.Where(x => ids.Contains(x.NodeId), "n"); } if (filterMustBeIsDependency) { - sql = sql.Where(rt => rt.IsDependency, "umbracoRelationType"); + sql = sql.Where(rt => rt.IsDependency, "x"); } // Ordering is required for paging - sql = sql.OrderBy(x => x.Alias); + sql = sql.OrderBy(x => x.Alias, "x"); var pagedResult = _scopeAccessor.AmbientScope.Database.Page(pageIndex + 1, pageSize, sql); totalRecords = pagedResult.TotalItems; @@ -63,6 +66,31 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement return pagedResult.Items.Select(MapDtoToEntity); } + private Sql GetInnerUnionSql() + { + var innerUnionSqlChild = _scopeAccessor.AmbientScope.Database.SqlContext.Sql().Select( + "[cr].childId as id", "[cr].parentId as otherId", "[rt].[alias]", "[rt].[name]", "[rt].[isDependency]", "[rt].[dual]") + .From("cr").InnerJoin("rt") + .On((cr, rt) => rt.Dual == false && rt.Id == cr.RelationType, "cr", "rt"); + + var innerUnionSqlDualParent = _scopeAccessor.AmbientScope.Database.SqlContext.Sql().Select( + "[dpr].parentId as id", "[dpr].childId as otherId", "[dprt].[alias]", "[dprt].[name]", "[dprt].[isDependency]", "[dprt].[dual]") + .From("dpr").InnerJoin("dprt") + .On((dpr, dprt) => dprt.Dual == true && dprt.Id == dpr.RelationType, "dpr", + "dprt"); + + var innerUnionSql3 = _scopeAccessor.AmbientScope.Database.SqlContext.Sql().Select( + "[dcr].childId as id", "[dcr].parentId as otherId", "[dcrt].[alias]", "[dcrt].[name]", "[dcrt].[isDependency]", "[dcrt].[dual]") + .From("dcr").InnerJoin("dcrt") + .On((dcr, dcrt) => dcrt.Dual == true && dcrt.Id == dcr.RelationType, "dcr", + "dcrt"); + + + var innerUnionSql = innerUnionSqlChild.Union(innerUnionSqlDualParent).Union(innerUnionSql3); + + return innerUnionSql; + } + /// /// Gets a page of the descending items that have any references, given a parent id. /// @@ -98,35 +126,38 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement .WhereLike(x => x.Path, subsubQuery); } - // Get all relations where parent is in the sub query + Sql innerUnionSql = GetInnerUnionSql(); var sql = _scopeAccessor.AmbientScope.Database.SqlContext.Sql().SelectDistinct( - "[pn].[id] as nodeId", - "[pn].[uniqueId] as nodeKey", - "[pn].[text] as nodeName", - "[pn].[nodeObjectType] as nodeObjectType", - "[ct].[icon] as contentTypeIcon", - "[ct].[alias] as contentTypeAlias", - "[ctn].[text] as contentTypeName", - "[umbracoRelationType].[alias] as relationTypeAlias", - "[umbracoRelationType].[name] as relationTypeName", - "[umbracoRelationType].[isDependency] as relationTypeIsDependency", - "[umbracoRelationType].[dual] as relationTypeIsBidirectional") - .From("r") - .InnerJoin("umbracoRelationType").On((left, right) => left.RelationType == right.Id, aliasLeft: "r", aliasRight: "umbracoRelationType") - .InnerJoin("cn").On((r, cn, rt) => (!rt.Dual && r.ParentId == cn.NodeId) || (rt.Dual && (r.ChildId == cn.NodeId || r.ParentId == cn.NodeId)), aliasLeft: "r", aliasRight: "cn", aliasOther: "umbracoRelationType") - .InnerJoin("pn").On((r, pn, cn) => (pn.NodeId == r.ChildId && cn.NodeId == r.ParentId) || (pn.NodeId == r.ParentId && cn.NodeId == r.ChildId), aliasLeft: "r", aliasRight: "pn", aliasOther: "cn") - .LeftJoin("c").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "pn", aliasRight: "c") - .LeftJoin("ct").On((left, right) => left.ContentTypeId == right.NodeId, aliasLeft: "c", aliasRight: "ct") - .LeftJoin("ctn").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "ct", aliasRight: "ctn") - .WhereIn((System.Linq.Expressions.Expression>)(x => x.NodeId), subQuery, "pn"); + "[x].[id] as nodeId", + "[n].[uniqueId] as nodeKey", + "[n].[text] as nodeName", + "[n].[nodeObjectType] as nodeObjectType", + "[ct].[icon] as contentTypeIcon", + "[ct].[alias] as contentTypeAlias", + "[ctn].[text] as contentTypeName", + "[x].[alias] as relationTypeAlias", + "[x].[name] as relationTypeName", + "[x].[isDependency] as relationTypeIsDependency", + "[x].[dual] as relationTypeIsBidirectional") + .From("n") + .InnerJoinNested(innerUnionSql, "x") + .On((n, x) => n.NodeId == x.Id, "n", "x") + .LeftJoin("c").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "n", + aliasRight: "c") + .LeftJoin("ct") + .On((left, right) => left.ContentTypeId == right.NodeId, aliasLeft: "c", + aliasRight: "ct") + .LeftJoin("ctn").On((left, right) => left.NodeId == right.NodeId, + aliasLeft: "ct", aliasRight: "ctn"); + sql = sql.WhereIn((System.Linq.Expressions.Expression>)(x => x.NodeId), subQuery, "n"); if (filterMustBeIsDependency) { - sql = sql.Where(rt => rt.IsDependency, "umbracoRelationType"); + sql = sql.Where(rt => rt.IsDependency, "x"); } // Ordering is required for paging - sql = sql.OrderBy(x => x.Alias); + sql = sql.OrderBy(x => x.Alias, "x"); var pagedResult = _scopeAccessor.AmbientScope.Database.Page(pageIndex + 1, pageSize, sql); totalRecords = pagedResult.TotalItems; @@ -134,41 +165,45 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement return pagedResult.Items.Select(MapDtoToEntity); } + /// /// Gets a page of items which are in relation with the current item. /// Basically, shows the items which depend on the current item. /// public IEnumerable GetPagedRelationsForItem(int id, long pageIndex, int pageSize, bool filterMustBeIsDependency, out long totalRecords) { + Sql innerUnionSql = GetInnerUnionSql(); var sql = _scopeAccessor.AmbientScope.Database.SqlContext.Sql().SelectDistinct( - "[cn].[id] as nodeId", - "[cn].[uniqueId] as nodeKey", - "[cn].[text] as nodeName", - "[cn].[nodeObjectType] as nodeObjectType", - "[ct].[icon] as contentTypeIcon", - "[ct].[alias] as contentTypeAlias", - "[ctn].[text] as contentTypeName", - "[umbracoRelationType].[alias] as relationTypeAlias", - "[umbracoRelationType].[name] as relationTypeName", - "[umbracoRelationType].[isDependency] as relationTypeIsDependency", - "[umbracoRelationType].[dual] as relationTypeIsBidirectional") - .From("r") - .InnerJoin("umbracoRelationType").On((left, right) => left.RelationType == right.Id, aliasLeft: "r", aliasRight: "umbracoRelationType") - .InnerJoin("cn").On((r, cn, rt) => (!rt.Dual && r.ParentId == cn.NodeId) || (rt.Dual && (r.ChildId == cn.NodeId || r.ParentId == cn.NodeId)), aliasLeft: "r", aliasRight: "cn", aliasOther: "umbracoRelationType") - .InnerJoin("pn").On((r, pn, cn) => (pn.NodeId == r.ChildId && cn.NodeId == r.ParentId) || (pn.NodeId == r.ParentId && cn.NodeId == r.ChildId), aliasLeft: "r", aliasRight: "pn", aliasOther: "cn") - .LeftJoin("c").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "cn", aliasRight: "c") - .LeftJoin("ct").On((left, right) => left.ContentTypeId == right.NodeId, aliasLeft: "c", aliasRight: "ct") - .LeftJoin("ctn").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "ct", aliasRight: "ctn") - .Where(x => x.NodeId == id, "pn") - .Where(x => x.ChildId == id || x.ParentId == id, "r"); // This last Where is purely to help SqlServer make a smarter query plan. More info https://github.com/umbraco/Umbraco-CMS/issues/12190 + "[x].[otherId] as nodeId", + "[n].[uniqueId] as nodeKey", + "[n].[text] as nodeName", + "[n].[nodeObjectType] as nodeObjectType", + "[ct].[icon] as contentTypeIcon", + "[ct].[alias] as contentTypeAlias", + "[ctn].[text] as contentTypeName", + "[x].[alias] as relationTypeAlias", + "[x].[name] as relationTypeName", + "[x].[isDependency] as relationTypeIsDependency", + "[x].[dual] as relationTypeIsBidirectional") + .From("n") + .InnerJoinNested(innerUnionSql, "x") + .On((n, x) => n.NodeId == x.OtherId, "n", "x") + .LeftJoin("c").On((left, right) => left.NodeId == right.NodeId, aliasLeft: "n", + aliasRight: "c") + .LeftJoin("ct") + .On((left, right) => left.ContentTypeId == right.NodeId, aliasLeft: "c", + aliasRight: "ct") + .LeftJoin("ctn").On((left, right) => left.NodeId == right.NodeId, + aliasLeft: "ct", aliasRight: "ctn") + .Where(x => x.OtherId != id, "x"); if (filterMustBeIsDependency) { - sql = sql.Where(rt => rt.IsDependency, "umbracoRelationType"); + sql = sql.Where(rt => rt.IsDependency, "x"); } // Ordering is required for paging - sql = sql.OrderBy(x => x.Alias); + sql = sql.OrderBy(x => x.Alias, "x"); var pagedResult = _scopeAccessor.AmbientScope.Database.Page(pageIndex + 1, pageSize, sql); totalRecords = pagedResult.TotalItems; @@ -176,6 +211,27 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement return pagedResult.Items.Select(MapDtoToEntity); } + private class UnionHelperDto + { + [Column("id")] + public int Id { get; set; } + + [Column("otherId")] + public int OtherId { get; set; } + + [Column("alias")] + public string Alias { get; set; } + + [Column("name")] + public string Name { get; set; } + + [Column("isDependency")] + public bool IsDependency { get; set; } + + [Column("dual")] + public bool Dual { get; set; } + } + private RelationItem MapDtoToEntity(RelationItemDto dto) { return new RelationItem()