From 86b2dac231e771c1e0639ae2d2d9fff0d7121c35 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 31 Jan 2017 17:20:44 +1100 Subject: [PATCH] Needed to change the 2nd query to use a new PagingSqlQuery object due to the way that < SQL 2012 formats it's paging query which is just not compatible with how we were parsing the queries for properties... i don't actually think it was working for a long tme. --- .../Repositories/ContentRepository.cs | 21 ++- .../Repositories/MediaRepository.cs | 19 +-- .../Repositories/MemberRepository.cs | 25 ++- .../Repositories/VersionableRepositoryBase.cs | 159 ++++++++++++++---- 4 files changed, 156 insertions(+), 68 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index f7b88ef46b..aedf701524 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -84,7 +84,7 @@ namespace Umbraco.Core.Persistence.Repositories var sqlBaseFull = GetBaseQuery(BaseQueryType.Full); var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids); - return ProcessQuery(translate(sqlBaseFull), translate(sqlBaseIds)); + return ProcessQuery(translate(sqlBaseFull), new PagingSqlQuery(translate(sqlBaseIds))); } protected override IEnumerable PerformGetByQuery(IQuery query) @@ -103,7 +103,7 @@ namespace Umbraco.Core.Persistence.Repositories var translatorFull = new SqlTranslator(sqlBaseFull, query); var translatorIds = new SqlTranslator(sqlBaseIds, query); - return ProcessQuery(translate(translatorFull), translate(translatorIds)); + return ProcessQuery(translate(translatorFull), new PagingSqlQuery(translate(translatorIds))); } #endregion @@ -225,7 +225,7 @@ namespace Umbraco.Core.Persistence.Repositories var sqlFull = translate(baseId, GetBaseQuery(BaseQueryType.Full)); var sqlIds = translate(baseId, GetBaseQuery(BaseQueryType.Ids)); - var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), SqlSyntax.SelectTop(sqlIds, groupSize)) + var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), new PagingSqlQuery(SqlSyntax.SelectTop(sqlIds, groupSize))) .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() }) .ToList(); @@ -260,7 +260,7 @@ namespace Umbraco.Core.Persistence.Repositories var sqlFull = translate(GetBaseQuery(BaseQueryType.Full)); var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids)); - return ProcessQuery(sqlFull, sqlIds, true); + return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true); } public override IContent GetByVersion(Guid versionId) @@ -679,7 +679,7 @@ namespace Umbraco.Core.Persistence.Repositories var sqlIds = GetBaseQuery(BaseQueryType.Ids); var translatorIds = new SqlTranslator(sqlIds, query); - return ProcessQuery(translate(translatorFull), translate(translatorIds), true); + return ProcessQuery(translate(translatorFull), new PagingSqlQuery(translate(translatorIds)), true); } /// @@ -859,7 +859,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, new Tuple("cmsDocument", "nodeId"), - (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds, isPaged:true), orderBy, orderDirection, orderBySystemField, + (sqlFull, pagingSqlQuery) => ProcessQuery(sqlFull, pagingSqlQuery), orderBy, orderDirection, orderBySystemField, filterCallback); } @@ -896,13 +896,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", /// /// The full SQL with the outer join to return all data required to create an IContent /// - /// + /// /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item /// - /// True if this is a paged query /// /// - private IEnumerable ProcessQuery(Sql sqlFull, Sql sqlIds, bool withCache = false, bool isPaged = false) + private IEnumerable ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false) { // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); @@ -968,7 +967,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", .ToDictionary(x => x.Id, x => x); // load all properties for all documents from database in 1 query - var propertyData = GetPropertyCollection(sqlIds, defs, isPaged); + var propertyData = GetPropertyCollection(pagingSqlQuery, defs); // assign var dtoIndex = 0; @@ -1015,7 +1014,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType); - var properties = GetPropertyCollection(docSql, new[] { docDef }, false); + var properties = GetPropertyCollection(docSql, new[] { docDef }); content.Properties = properties[dto.NodeId]; diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index c4957ca9fe..56b5e2ed9d 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -68,7 +68,7 @@ namespace Umbraco.Core.Persistence.Repositories sql.Where("umbracoNode.id in (@ids)", new { ids = ids }); } - return ProcessQuery(sql, sql); + return ProcessQuery(sql, new PagingSqlQuery(sql)); } protected override IEnumerable PerformGetByQuery(IQuery query) @@ -78,7 +78,7 @@ namespace Umbraco.Core.Persistence.Repositories var sql = translator.Translate() .OrderBy(x => x.SortOrder, SqlSyntax); - return ProcessQuery(sql, sql); + return ProcessQuery(sql, new PagingSqlQuery(sql)); } #endregion @@ -143,7 +143,7 @@ namespace Umbraco.Core.Persistence.Repositories var sql = GetBaseQuery(false) .Where(GetBaseWhereClause(), new { Id = id }) .OrderByDescending(x => x.VersionDate, SqlSyntax); - return ProcessQuery(sql, sql, true); + return ProcessQuery(sql, new PagingSqlQuery(sql), true); } /// @@ -152,13 +152,12 @@ namespace Umbraco.Core.Persistence.Repositories /// /// The full SQL to select all media data /// - /// + /// /// The Id SQL to just return all media ids - used to process the properties for the media item /// - /// True if this is a paged query /// /// - private IEnumerable ProcessQuery(Sql sqlFull, Sql sqlIds, bool withCache = false, bool isPaged = false) + private IEnumerable ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false) { // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); @@ -212,7 +211,7 @@ namespace Umbraco.Core.Persistence.Repositories } // load all properties for all documents from database in 1 query - var propertyData = GetPropertyCollection(sqlIds, defs, isPaged); + var propertyData = GetPropertyCollection(pagingSqlQuery, defs); // assign var dtoIndex = 0; @@ -270,7 +269,7 @@ namespace Umbraco.Core.Persistence.Repositories .Where(x => x.NodeId > baseId, SqlSyntax) .OrderBy(x => x.NodeId, SqlSyntax); var sql = SqlSyntax.SelectTop(query, groupSize); - var xmlItems = ProcessQuery(sql, sql) + var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql)) .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() }) .ToList(); @@ -520,7 +519,7 @@ namespace Umbraco.Core.Persistence.Repositories return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, new Tuple("cmsContentVersion", "contentId"), - (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds, isPaged: true), orderBy, orderDirection, orderBySystemField, + (sqlFull, pagingSqlQuery) => ProcessQuery(sqlFull, pagingSqlQuery), orderBy, orderDirection, orderBySystemField, filterCallback); } @@ -540,7 +539,7 @@ namespace Umbraco.Core.Persistence.Repositories var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType); - var properties = GetPropertyCollection(docSql, new[] { docDef }, false); + var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef }); media.Properties = properties[dto.NodeId]; diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index 0cb5be4ce0..9fdee650b3 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -68,7 +68,7 @@ namespace Umbraco.Core.Persistence.Repositories sql.Where("umbracoNode.id in (@ids)", new { ids = ids }); } - return ProcessQuery(sql, sql); + return ProcessQuery(sql, new PagingSqlQuery(sql)); } @@ -90,7 +90,7 @@ namespace Umbraco.Core.Persistence.Repositories baseQuery.Append(new Sql("WHERE umbracoNode.id IN (" + sql.SQL + ")", sql.Arguments)) .OrderBy(x => x.SortOrder); - return ProcessQuery(baseQuery, baseQuery); + return ProcessQuery(baseQuery, new PagingSqlQuery(baseQuery)); } else { @@ -98,7 +98,7 @@ namespace Umbraco.Core.Persistence.Repositories var sql = translator.Translate() .OrderBy(x => x.SortOrder); - return ProcessQuery(sql, sql); + return ProcessQuery(sql, new PagingSqlQuery(sql)); } } @@ -385,7 +385,7 @@ namespace Umbraco.Core.Persistence.Repositories var sql = GetBaseQuery(false) .Where(GetBaseWhereClause(), new { Id = id }) .OrderByDescending(x => x.VersionDate, SqlSyntax); - return ProcessQuery(sql, sql, true); + return ProcessQuery(sql, new PagingSqlQuery(sql), true); } public void RebuildXmlStructures(Func serializer, int groupSize = 200, IEnumerable contentTypeIds = null) @@ -409,7 +409,7 @@ namespace Umbraco.Core.Persistence.Repositories .Where(x => x.NodeId > baseId) .OrderBy(x => x.NodeId, SqlSyntax); var sql = SqlSyntax.SelectTop(query, groupSize); - var xmlItems = ProcessQuery(sql, sql) + var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql)) .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() }) .ToList(); @@ -450,7 +450,7 @@ namespace Umbraco.Core.Persistence.Repositories var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId); var media = factory.BuildEntity(dto); - var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) }, false); + var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) }); media.Properties = properties[dto.NodeId]; @@ -541,7 +541,7 @@ namespace Umbraco.Core.Persistence.Repositories .OrderByDescending(x => x.VersionDate) .OrderBy(x => x.SortOrder); - return ProcessQuery(sql, sql); + return ProcessQuery(sql, new PagingSqlQuery(sql)); } @@ -625,7 +625,7 @@ namespace Umbraco.Core.Persistence.Repositories return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, new Tuple("cmsMember", "nodeId"), - (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds, isPaged:true), orderBy, orderDirection, orderBySystemField, + (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds), orderBy, orderDirection, orderBySystemField, filterCallback); } @@ -671,13 +671,12 @@ namespace Umbraco.Core.Persistence.Repositories /// /// The full SQL to select all member data /// - /// + /// /// The Id SQL to just return all member ids - used to process the properties for the member item /// - /// True if this is a paged query /// /// - private IEnumerable ProcessQuery(Sql sqlFull, Sql sqlIds, bool withCache = false, bool isPaged = false) + private IEnumerable ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false) { // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); @@ -717,7 +716,7 @@ namespace Umbraco.Core.Persistence.Repositories } // load all properties for all documents from database in 1 query - var propertyData = GetPropertyCollection(sqlIds, defs, isPaged); + var propertyData = GetPropertyCollection(pagingSqlQuery, defs); // assign var dtoIndex = 0; @@ -754,7 +753,7 @@ namespace Umbraco.Core.Persistence.Repositories var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType); - var properties = GetPropertyCollection(docSql, new[] { docDef }, false); + var properties = GetPropertyCollection(docSql, new[] { docDef }); member.Properties = properties[dto.ContentVersionDto.NodeId]; diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index 26c53a091f..6b18fc70b4 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -431,7 +431,7 @@ namespace Umbraco.Core.Persistence.Repositories /// orderBy protected IEnumerable GetPagedResultsByQuery(IQuery query, long pageIndex, int pageSize, out long totalRecords, Tuple nodeIdSelect, - Func> processQuery, + Func, IEnumerable> processQuery, string orderBy, Direction orderDirection, bool orderBySystemField, @@ -485,11 +485,8 @@ namespace Umbraco.Core.Persistence.Repositories var fullQuery = GetSortedSqlForPagedResults( GetFilteredSqlForPagedResults(fullQueryWithPagedInnerJoin, defaultFilter), orderDirection, orderBy, orderBySystemField, nodeIdSelect); - - //get the id query in the paged format - var idPagedQuery = new Sql(sqlStringPage, args); - - return processQuery(fullQuery, idPagedQuery); + + return processQuery(fullQuery, new PagingSqlQuery(Database, sqlNodeIdsWithSort, pageIndex, pageSize)); } else { @@ -499,20 +496,47 @@ namespace Umbraco.Core.Persistence.Repositories return result; } + /// + /// Gets the property collection for a non-paged query + /// + /// + /// + /// protected IDictionary GetPropertyCollection( - Sql docSql, - IReadOnlyCollection documentDefs, - bool isPaged) + Sql sql, + IReadOnlyCollection documentDefs) + { + return GetPropertyCollection(new PagingSqlQuery(sql), documentDefs); + } + + /// + /// Gets the property collection for a query + /// + /// + /// + /// + protected IDictionary GetPropertyCollection( + PagingSqlQuery pagingSqlQuery, + IReadOnlyCollection documentDefs) { if (documentDefs.Count == 0) return new Dictionary(); + //initialize to the query passed in + var docSql = pagingSqlQuery.PrePagedSql; + //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use // the statement to go get the property data for all of the items by using an inner join var parsedOriginalSql = "SELECT {0} " + docSql.SQL.Substring(docSql.SQL.IndexOf("FROM", StringComparison.Ordinal)); - - //now remove everything from an Orderby clause and beyond if this is unpaged data - if (isPaged == false && parsedOriginalSql.InvariantContains("ORDER BY ")) + + if (pagingSqlQuery.HasPaging) { + //if this is a paged query, build the paged query with the custom column substitution, then re-assign + docSql = pagingSqlQuery.BuildPagedQuery("{0}"); + parsedOriginalSql = docSql.SQL; + } + else if (parsedOriginalSql.InvariantContains("ORDER BY ")) + { + //now remove everything from an Orderby clause and beyond if this is unpaged data parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal)); } @@ -652,28 +676,7 @@ ORDER BY contentNodeId, propertytypeid return result; } - - public class DocumentDefinition - { - /// - /// Initializes a new instance of the class. - /// - public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition) - { - Id = id; - Version = version; - VersionDate = versionDate; - CreateDate = createDate; - Composition = composition; - } - - public int Id { get; set; } - public Guid Version { get; set; } - public DateTime VersionDate { get; set; } - public DateTime CreateDate { get; set; } - public IContentTypeComposition Composition { get; set; } - } - + protected virtual string GetDatabaseFieldNameForOrderBy(string orderBy) { // Translate the passed order by field (which were originally defined for in-memory object sorting @@ -769,5 +772,93 @@ ORDER BY contentNodeId, propertytypeid /// /// protected abstract Sql GetBaseQuery(BaseQueryType queryType); + + + internal class DocumentDefinition + { + /// + /// Initializes a new instance of the class. + /// + public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition) + { + Id = id; + Version = version; + VersionDate = versionDate; + CreateDate = createDate; + Composition = composition; + } + + public int Id { get; set; } + public Guid Version { get; set; } + public DateTime VersionDate { get; set; } + public DateTime CreateDate { get; set; } + public IContentTypeComposition Composition { get; set; } + } + + /// + /// An object representing a query that may contain paging information + /// + internal class PagingSqlQuery + { + public Sql PrePagedSql { get; private set; } + + public PagingSqlQuery(Sql prePagedSql) + { + PrePagedSql = prePagedSql; + } + + public virtual bool HasPaging + { + get { return false; } + } + + public virtual Sql BuildPagedQuery(string selectColumns) + { + throw new InvalidOperationException("This query has no paging information"); + } + } + + /// + /// An object representing a query that contains paging information + /// + /// + internal class PagingSqlQuery : PagingSqlQuery + { + private readonly Database _db; + private readonly long _pageIndex; + private readonly int _pageSize; + + public PagingSqlQuery(Database db, Sql prePagedSql, long pageIndex, int pageSize) : base(prePagedSql) + { + _db = db; + _pageIndex = pageIndex; + _pageSize = pageSize; + } + + public override bool HasPaging + { + get { return _pageSize > 0; } + } + + /// + /// Creates a paged query based on the original query and subtitutes the selectColumns specified + /// + /// + /// + public override Sql BuildPagedQuery(string selectColumns) + { + if (HasPaging == false) throw new InvalidOperationException("This query has no paging information"); + + var resultSql = string.Format("SELECT {0} {1}", selectColumns, PrePagedSql.SQL.Substring(PrePagedSql.SQL.IndexOf("FROM", StringComparison.Ordinal))); + + //this query is meant to be paged so we need to generate the paging syntax + //Create the inner paged query that was used above to get the paged result, we'll use that as the inner sub query + var args = PrePagedSql.Arguments; + string sqlStringCount, sqlStringPage; + _db.BuildPageQueries(_pageIndex * _pageSize, _pageSize, resultSql, ref args, out sqlStringCount, out sqlStringPage); + + return new Sql(sqlStringPage, args); + } + } } } \ No newline at end of file