yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1715 from umbraco/temp-U4-9450

Browse Source 
U4-9450 Paged data queries return all property data in the entire dat…
This commit is contained in:
Sebastiaan Janssen
2017-02-09 21:02:15 +01:00
committed by GitHub
parent ade3c4ee6c 32d757b13c
commit a7e28a97d9
6 changed files with 314 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/Umbraco.Core/Persistence/Factories/ContentFactory.cs
View File

@@ -27,7 +27,20 @@ namespace Umbraco.Core.Persistence.Factories
#region Implementation of IEntityFactory<IContent,DocumentDto>
public static IContent BuildEntity(DocumentDto dto, IContentType contentType)
/// <summary>
/// Builds a IContent item from the dto(s) and content type
/// </summary>
/// <param name="dto">
/// This DTO can contain all of the information to build an IContent item, however in cases where multiple entities are being built,
/// a separate <see cref="DocumentPublishedReadOnlyDto"/> publishedDto entity will be supplied in place of the <see cref="DocumentDto"/>'s own
/// ResultColumn DocumentPublishedReadOnlyDto
/// </param>
/// <param name="contentType"></param>
/// <param name="publishedDto">
/// When querying for multiple content items the main DTO will not contain the ResultColumn DocumentPublishedReadOnlyDto and a separate publishedDto instance will be supplied
/// </param>
/// <returns></returns>
public static IContent BuildEntity(DocumentDto dto, IContentType contentType, DocumentPublishedReadOnlyDto publishedDto = null)
{
var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, contentType);
@@ -52,8 +65,13 @@ namespace Umbraco.Core.Persistence.Factories
content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?)null;
content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?)null;
content.Version = dto.ContentVersionDto.VersionId;
content.PublishedState = dto.Published ? PublishedState.Published : PublishedState.Unpublished;
content.PublishedVersionGuid = dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId;
//Check if the publishedDto has been supplied, if not the use the dto's own DocumentPublishedReadOnlyDto value
content.PublishedVersionGuid = publishedDto == null
? (dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId)
: publishedDto.VersionId;
//on initial construction we don't want to have dirty properties tracked
// http://issues.umbraco.org/issue/U4-1946

24
src/Umbraco.Core/Persistence/Repositories/BaseQueryType.cs
View File

@@ -2,8 +2,30 @@ namespace Umbraco.Core.Persistence.Repositories
{
internal enum BaseQueryType
{
Full,
/// <summary>
/// A query to return all information for a single item
/// </summary>
/// <remarks>
/// In some cases this will be the same as <see cref="FullMultiple"/>
/// </remarks>
FullSingle,
/// <summary>
/// A query to return all information for multiple items
/// </summary>
/// <remarks>
/// In some cases this will be the same as <see cref="FullSingle"/>
/// </remarks>
FullMultiple,
/// <summary>
/// A query to return the ids for items
/// </summary>
Ids,
/// <summary>
/// A query to return the count for items
/// </summary>
Count
}
}

125
src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Globalization;
using System.Linq;
using System.Xml;
@@ -53,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories
protected override IContent PerformGet(int id)
{
var sql = GetBaseQuery(false)
var sql = GetBaseQuery(BaseQueryType.FullSingle)
.Where(GetBaseWhereClause(), new { Id = id })
.Where<DocumentDto>(x => x.Newest, SqlSyntax)
.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
@@ -81,15 +82,15 @@ namespace Umbraco.Core.Persistence.Repositories
return s;
};
var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
var sqlBaseFull = GetBaseQuery(BaseQueryType.FullMultiple);
var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
return ProcessQuery(translate(sqlBaseFull), translate(sqlBaseIds));
return ProcessQuery(translate(sqlBaseFull), new PagingSqlQuery(translate(sqlBaseIds)));
}
protected override IEnumerable<IContent> PerformGetByQuery(IQuery<IContent> query)
{
var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
var sqlBaseFull = GetBaseQuery(BaseQueryType.FullMultiple);
var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
Func<SqlTranslator<IContent>, Sql> translate = (translator) =>
@@ -103,13 +104,25 @@ namespace Umbraco.Core.Persistence.Repositories
var translatorFull = new SqlTranslator<IContent>(sqlBaseFull, query);
var translatorIds = new SqlTranslator<IContent>(sqlBaseIds, query);
return ProcessQuery(translate(translatorFull), translate(translatorIds));
return ProcessQuery(translate(translatorFull), new PagingSqlQuery(translate(translatorIds)));
}
#endregion
#region Overrides of PetaPocoRepositoryBase<IContent>
/// <summary>
/// Returns the base query to return Content
/// </summary>
/// <param name="queryType"></param>
/// <returns></returns>
/// <remarks>
/// Content queries will differ depending on what needs to be returned:
/// * FullSingle: When querying for a single document, this will include the Outer join to fetch the content item's published version info
/// * FullMultiple: When querying for multiple documents, this will exclude the Outer join to fetch the content item's published version info - this info would need to be fetched separately
/// * Ids: This would essentially be the same as FullMultiple however the columns specified will only return the Ids for the documents
/// * Count: A query to return the count for documents
/// </remarks>
protected override Sql GetBaseQuery(BaseQueryType queryType)
{
var sql = new Sql();
@@ -122,14 +135,14 @@ namespace Umbraco.Core.Persistence.Repositories
.InnerJoin<NodeDto>(SqlSyntax)
.On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId);
if (queryType == BaseQueryType.Full)
if (queryType == BaseQueryType.FullSingle)
{
//The only reason we apply this left outer join is to be able to pull back the DocumentPublishedReadOnlyDto
//information with the entire data set, so basically this will get both the latest document and also it's published
//version if it has one. When performing a count or when just retrieving Ids like in paging, this is unecessary
//version if it has one. When performing a count or when retrieving Ids like in paging, this is unecessary
//and causes huge performance overhead for the SQL server, especially when sorting the result.
//To fix this perf overhead we'd need another index on :
// CREATE NON CLUSTERED INDEX ON cmsDocument.node + cmsDocument.published
//We also don't include this outer join when querying for multiple entities since it is much faster to fetch this information
//in a separate query. For a single entity this is ok.
var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)",
SqlSyntax.GetQuotedTableName("cmsDocument"),
@@ -151,7 +164,7 @@ namespace Umbraco.Core.Persistence.Repositories
protected override Sql GetBaseQuery(bool isCount)
{
return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.FullSingle);
}
protected override string GetBaseWhereClause()
@@ -222,10 +235,10 @@ namespace Umbraco.Core.Persistence.Repositories
while (true)
{
// get the next group of nodes
var sqlFull = translate(baseId, GetBaseQuery(BaseQueryType.Full));
var sqlFull = translate(baseId, GetBaseQuery(BaseQueryType.FullMultiple));
var sqlIds = translate(baseId, GetBaseQuery(BaseQueryType.Ids));
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), SqlSyntax.SelectTop(sqlIds, groupSize))
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), new PagingSqlQuery(SqlSyntax.SelectTop(sqlIds, groupSize)))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
@@ -245,7 +258,7 @@ namespace Umbraco.Core.Persistence.Repositories
Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
}
}
baseId = xmlItems.Last().NodeId;
baseId = xmlItems[xmlItems.Count - 1].NodeId;
}
}
@@ -257,15 +270,15 @@ namespace Umbraco.Core.Persistence.Repositories
.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
};
var sqlFull = translate(GetBaseQuery(BaseQueryType.Full));
var sqlFull = translate(GetBaseQuery(BaseQueryType.FullMultiple));
var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids));
return ProcessQuery(sqlFull, sqlIds, true);
return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true);
}
public override IContent GetByVersion(Guid versionId)
{
var sql = GetBaseQuery(false);
var sql = GetBaseQuery(BaseQueryType.FullSingle);
sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
sql.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
@@ -674,12 +687,12 @@ namespace Umbraco.Core.Persistence.Repositories
// ORDER BY substring(path, 1, len(path) - charindex(',', reverse(path))), sortOrder
// but that's probably an overkill - sorting by level,sortOrder should be enough
var sqlFull = GetBaseQuery(BaseQueryType.Full);
var sqlFull = GetBaseQuery(BaseQueryType.FullMultiple);
var translatorFull = new SqlTranslator<IContent>(sqlFull, query);
var sqlIds = GetBaseQuery(BaseQueryType.Ids);
var translatorIds = new SqlTranslator<IContent>(sqlIds, query);
return ProcessQuery(translate(translatorFull), translate(translatorIds), true);
return ProcessQuery(translate(translatorFull), new PagingSqlQuery(translate(translatorIds)), true);
}
/// <summary>
@@ -859,7 +872,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
return GetPagedResultsByQuery<DocumentDto>(query, pageIndex, pageSize, out totalRecords,
new Tuple<string, string>("cmsDocument", "nodeId"),
(sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds), orderBy, orderDirection, orderBySystemField,
(sqlFull, pagingSqlQuery) => ProcessQuery(sqlFull, pagingSqlQuery), orderBy, orderDirection, orderBySystemField,
filterCallback);
}
@@ -889,23 +902,58 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
return base.GetDatabaseFieldNameForOrderBy(orderBy);
}
/// <summary>
/// This is the underlying method that processes most queries for this repository
/// </summary>
/// <param name="sqlFull">
/// The full SQL with the outer join to return all data required to create an IContent
/// The FullMultiple SQL without the outer join to return all data required to create an IContent excluding it's published state data which this will query separately
/// </param>
/// <param name="sqlIds">
/// <param name="pagingSqlQuery">
/// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item
/// </param>
/// <param name="withCache"></param>
/// <returns></returns>
private IEnumerable<IContent> ProcessQuery(Sql sqlFull, Sql sqlIds, bool withCache = false)
private IEnumerable<IContent> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
{
// fetch returns a list so it's ok to iterate it in this method
var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto, DocumentPublishedReadOnlyDto>(sqlFull);
var dtos = Database.Fetch<DocumentDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);
if (dtos.Count == 0) return Enumerable.Empty<IContent>();
//Go and get all of the published version data separately for this data, this is because when we are querying
//for multiple content items we don't include the outer join to fetch this data in the same query because
//it is insanely slow. Instead we just fetch the published version data separately in one query.
//we need to parse the original SQL statement and reduce the columns to just cmsDocument.nodeId so that we can use
// the statement to go get the published data for all of the items by using an inner join
var parsedOriginalSql = "SELECT cmsDocument.nodeId " + sqlFull.SQL.Substring(sqlFull.SQL.IndexOf("FROM", StringComparison.Ordinal));
//now remove everything from an Orderby clause and beyond
if (parsedOriginalSql.InvariantContains("ORDER BY "))
{
parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
}
var publishedSql = new Sql(@"SELECT *
FROM cmsDocument AS doc2
INNER JOIN
(" + parsedOriginalSql + @") as docData
ON doc2.nodeId = docData.nodeId
WHERE doc2.published = 1
ORDER BY doc2.nodeId
", sqlFull.Arguments);
//go and get the published version data, we do a Query here and not a Fetch so we are
//not allocating a whole list to memory just to allocate another list in memory since
//we are assigning this data to a keyed collection for fast lookup below
var publishedData = Database.Query<DocumentPublishedReadOnlyDto>(publishedSql);
var publishedDataCollection = new DocumentPublishedReadOnlyDtoCollection();
foreach (var publishedDto in publishedData)
{
//double check that there's no corrupt db data, there should only be a single published item
if (publishedDataCollection.Contains(publishedDto.NodeId) == false)
publishedDataCollection.Add(publishedDto);
}
var content = new IContent[dtos.Count];
var defs = new List<DocumentDefinition>();
@@ -919,6 +967,8 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
for (var i = 0; i < dtos.Count; i++)
{
var dto = dtos[i];
DocumentPublishedReadOnlyDto publishedDto;
publishedDataCollection.TryGetValue(dto.NodeId, out publishedDto);
// if the cache contains the published version, use it
if (withCache)
@@ -946,7 +996,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
}
content[i] = ContentFactory.BuildEntity(dto, contentType);
content[i] = ContentFactory.BuildEntity(dto, contentType, publishedDto);
// need template
if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
@@ -967,7 +1017,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
.ToDictionary(x => x.Id, x => x);
// load all properties for all documents from database in 1 query
var propertyData = GetPropertyCollection(sqlIds, defs);
var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
// assign
var dtoIndex = 0;
@@ -1055,7 +1105,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
return currentName;
}
/// <summary>
/// Dispose disposable properties
/// </summary>
@@ -1070,5 +1120,26 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
_contentPreviewRepository.Dispose();
_contentXmlRepository.Dispose();
}
/// <summary>
/// A keyed collection for fast lookup when retrieving a separate list of published version data
/// </summary>
private class DocumentPublishedReadOnlyDtoCollection : KeyedCollection<int, DocumentPublishedReadOnlyDto>
{
protected override int GetKeyForItem(DocumentPublishedReadOnlyDto item)
{
return item.NodeId;
}
public bool TryGetValue(int key, out DocumentPublishedReadOnlyDto val)
{
if (Dictionary == null)
{
val = null;
return false;
}
return Dictionary.TryGetValue(key, out val);
}
}
}
}

34
src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
View File

@@ -68,7 +68,7 @@ namespace Umbraco.Core.Persistence.Repositories
sql.Where("umbracoNode.id in (@ids)", new { ids = ids });
}
return ProcessQuery(sql);
return ProcessQuery(sql, new PagingSqlQuery(sql));
}
protected override IEnumerable<IMedia> PerformGetByQuery(IQuery<IMedia> query)
@@ -78,7 +78,7 @@ namespace Umbraco.Core.Persistence.Repositories
var sql = translator.Translate()
.OrderBy<NodeDto>(x => x.SortOrder, SqlSyntax);
return ProcessQuery(sql);
return ProcessQuery(sql, new PagingSqlQuery(sql));
}
#endregion
@@ -100,7 +100,7 @@ namespace Umbraco.Core.Persistence.Repositories
protected override Sql GetBaseQuery(bool isCount)
{
return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.FullSingle);
}
protected override string GetBaseWhereClause()
@@ -143,13 +143,24 @@ namespace Umbraco.Core.Persistence.Repositories
var sql = GetBaseQuery(false)
.Where(GetBaseWhereClause(), new { Id = id })
.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
return ProcessQuery(sql, true);
return ProcessQuery(sql, new PagingSqlQuery(sql), true);
}
private IEnumerable<IMedia> ProcessQuery(Sql sql, bool withCache = false)
/// <summary>
/// This is the underlying method that processes most queries for this repository
/// </summary>
/// <param name="sqlFull">
/// The full SQL to select all media data
/// </param>
/// <param name="pagingSqlQuery">
/// The Id SQL to just return all media ids - used to process the properties for the media item
/// </param>
/// <param name="withCache"></param>
/// <returns></returns>
private IEnumerable<IMedia> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
{
// fetch returns a list so it's ok to iterate it in this method
var dtos = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sql);
var dtos = Database.Fetch<ContentVersionDto, ContentDto, NodeDto>(sqlFull);
var content = new IMedia[dtos.Count];
var defs = new List<DocumentDefinition>();
@@ -200,7 +211,7 @@ namespace Umbraco.Core.Persistence.Repositories
}
// load all properties for all documents from database in 1 query
var propertyData = GetPropertyCollection(sql, defs);
var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
// assign
var dtoIndex = 0;
@@ -257,7 +268,8 @@ namespace Umbraco.Core.Persistence.Repositories
query = query
.Where<NodeDto>(x => x.NodeId > baseId, SqlSyntax)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
var sql = SqlSyntax.SelectTop(query, groupSize);
var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
@@ -507,7 +519,7 @@ namespace Umbraco.Core.Persistence.Repositories
return GetPagedResultsByQuery<ContentVersionDto>(query, pageIndex, pageSize, out totalRecords,
new Tuple<string, string>("cmsContentVersion", "contentId"),
(sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField,
(sqlFull, pagingSqlQuery) => ProcessQuery(sqlFull, pagingSqlQuery), orderBy, orderDirection, orderBySystemField,
filterCallback);
}
@@ -515,7 +527,7 @@ namespace Umbraco.Core.Persistence.Repositories
/// <summary>
/// Private method to create a media object from a ContentDto
/// </summary>
/// <param name="d"></param>
/// <param name="dto"></param>
/// <param name="versionId"></param>
/// <param name="docSql"></param>
/// <returns></returns>
@@ -527,7 +539,7 @@ namespace Umbraco.Core.Persistence.Repositories
var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
var properties = GetPropertyCollection(docSql, new[] { docDef });
var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef });
media.Properties = properties[dto.NodeId];

36
src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
View File

@@ -68,7 +68,7 @@ namespace Umbraco.Core.Persistence.Repositories
sql.Where("umbracoNode.id in (@ids)", new { ids = ids });
}
return ProcessQuery(sql);
return ProcessQuery(sql, new PagingSqlQuery(sql));
}
@@ -90,7 +90,7 @@ namespace Umbraco.Core.Persistence.Repositories
baseQuery.Append(new Sql("WHERE umbracoNode.id IN (" + sql.SQL + ")", sql.Arguments))
.OrderBy<NodeDto>(x => x.SortOrder);
return ProcessQuery(baseQuery);
return ProcessQuery(baseQuery, new PagingSqlQuery(baseQuery));
}
else
{
@@ -98,7 +98,7 @@ namespace Umbraco.Core.Persistence.Repositories
var sql = translator.Translate()
.OrderBy<NodeDto>(x => x.SortOrder);
return ProcessQuery(sql);
return ProcessQuery(sql, new PagingSqlQuery(sql));
}
}
@@ -129,7 +129,7 @@ namespace Umbraco.Core.Persistence.Repositories
protected override Sql GetBaseQuery(bool isCount)
{
return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full);
return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.FullSingle);
}
protected override string GetBaseWhereClause()
@@ -385,7 +385,7 @@ namespace Umbraco.Core.Persistence.Repositories
var sql = GetBaseQuery(false)
.Where(GetBaseWhereClause(), new { Id = id })
.OrderByDescending<ContentVersionDto>(x => x.VersionDate, SqlSyntax);
return ProcessQuery(sql, true);
return ProcessQuery(sql, new PagingSqlQuery(sql), true);
}
public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
@@ -408,7 +408,8 @@ namespace Umbraco.Core.Persistence.Repositories
query = query
.Where<NodeDto>(x => x.NodeId > baseId)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
var sql = SqlSyntax.SelectTop(query, groupSize);
var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
@@ -449,7 +450,7 @@ namespace Umbraco.Core.Persistence.Repositories
var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId);
var media = factory.BuildEntity(dto);
var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) });
var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) });
media.Properties = properties[dto.NodeId];
@@ -540,7 +541,7 @@ namespace Umbraco.Core.Persistence.Repositories
.OrderByDescending<ContentVersionDto>(x => x.VersionDate)
.OrderBy<NodeDto>(x => x.SortOrder);
return ProcessQuery(sql);
return ProcessQuery(sql, new PagingSqlQuery(sql));
}
@@ -624,7 +625,7 @@ namespace Umbraco.Core.Persistence.Repositories
return GetPagedResultsByQuery<MemberDto>(query, pageIndex, pageSize, out totalRecords,
new Tuple<string, string>("cmsMember", "nodeId"),
(sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField,
(sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds), orderBy, orderDirection, orderBySystemField,
filterCallback);
}
@@ -664,10 +665,21 @@ namespace Umbraco.Core.Persistence.Repositories
return base.GetEntityPropertyNameForOrderBy(orderBy);
}
private IEnumerable<IMember> ProcessQuery(Sql sql, bool withCache = false)
/// <summary>
/// This is the underlying method that processes most queries for this repository
/// </summary>
/// <param name="sqlFull">
/// The full SQL to select all member data
/// </param>
/// <param name="pagingSqlQuery">
/// The Id SQL to just return all member ids - used to process the properties for the member item
/// </param>
/// <param name="withCache"></param>
/// <returns></returns>
private IEnumerable<IMember> ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
{
// fetch returns a list so it's ok to iterate it in this method
var dtos = Database.Fetch<MemberDto, ContentVersionDto, ContentDto, NodeDto>(sql);
var dtos = Database.Fetch<MemberDto, ContentVersionDto, ContentDto, NodeDto>(sqlFull);
var content = new IMember[dtos.Count];
var defs = new List<DocumentDefinition>();
@@ -704,7 +716,7 @@ namespace Umbraco.Core.Persistence.Repositories
}
// load all properties for all documents from database in 1 query
var propertyData = GetPropertyCollection(sql, defs);
var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
// assign
var dtoIndex = 0;

157
src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
View File

@@ -431,7 +431,7 @@ namespace Umbraco.Core.Persistence.Repositories
/// <exception cref="System.ArgumentNullException">orderBy</exception>
protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto>(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
Tuple<string, string> nodeIdSelect,
Func<Sql, Sql, IEnumerable<TEntity>> processQuery,
Func<Sql, PagingSqlQuery<TDto>, IEnumerable<TEntity>> processQuery,
string orderBy,
Direction orderDirection,
bool orderBySystemField,
@@ -442,7 +442,7 @@ namespace Umbraco.Core.Persistence.Repositories
// Get base query for returning IDs
var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids);
// Get base query for returning all data
var sqlBaseFull = GetBaseQuery(BaseQueryType.Full);
var sqlBaseFull = GetBaseQuery(BaseQueryType.FullMultiple);
if (query == null) query = new Query<TEntity>();
var translatorIds = new SqlTranslator<TEntity>(sqlBaseIds, query);
@@ -465,7 +465,7 @@ namespace Umbraco.Core.Persistence.Repositories
// the pageResult, then the GetAll will actually return ALL records in the db.
if (pagedResult.Items.Any())
{
//Crete the inner paged query that was used above to get the paged result, we'll use that as the inner sub query
//Create the inner paged query that was used above to get the paged result, we'll use that as the inner sub query
var args = sqlNodeIdsWithSort.Arguments;
string sqlStringCount, sqlStringPage;
Database.BuildPageQueries<TDto>(pageIndex * pageSize, pageSize, sqlNodeIdsWithSort.SQL, ref args, out sqlStringCount, out sqlStringPage);
@@ -485,8 +485,8 @@ namespace Umbraco.Core.Persistence.Repositories
var fullQuery = GetSortedSqlForPagedResults(
GetFilteredSqlForPagedResults(fullQueryWithPagedInnerJoin, defaultFilter),
orderDirection, orderBy, orderBySystemField, nodeIdSelect);
return processQuery(fullQuery, sqlNodeIdsWithSort);
return processQuery(fullQuery, new PagingSqlQuery<TDto>(Database, sqlNodeIdsWithSort, pageIndex, pageSize));
}
else
{
@@ -496,18 +496,47 @@ namespace Umbraco.Core.Persistence.Repositories
return result;
}
/// <summary>
/// Gets the property collection for a non-paged query
/// </summary>
/// <param name="sql"></param>
/// <param name="documentDefs"></param>
/// <returns></returns>
protected IDictionary<int, PropertyCollection> GetPropertyCollection(
Sql docSql,
Sql sql,
IReadOnlyCollection<DocumentDefinition> documentDefs)
{
return GetPropertyCollection(new PagingSqlQuery(sql), documentDefs);
}
/// <summary>
/// Gets the property collection for a query
/// </summary>
/// <param name="pagingSqlQuery"></param>
/// <param name="documentDefs"></param>
/// <returns></returns>
protected IDictionary<int, PropertyCollection> GetPropertyCollection(
PagingSqlQuery pagingSqlQuery,
IReadOnlyCollection<DocumentDefinition> documentDefs)
{
if (documentDefs.Count == 0) return new Dictionary<int, PropertyCollection>();
//initialize to the query passed in
var docSql = pagingSqlQuery.PrePagedSql;
//we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use
// the statement to go get the property data for all of the items by using an inner join
var parsedOriginalSql = "SELECT {0} " + docSql.SQL.Substring(docSql.SQL.IndexOf("FROM", StringComparison.Ordinal));
//now remove everything from an Orderby clause and beyond
if (parsedOriginalSql.InvariantContains("ORDER BY "))
if (pagingSqlQuery.HasPaging)
{
//if this is a paged query, build the paged query with the custom column substitution, then re-assign
docSql = pagingSqlQuery.BuildPagedQuery("{0}");
parsedOriginalSql = docSql.SQL;
}
else if (parsedOriginalSql.InvariantContains("ORDER BY "))
{
//now remove everything from an Orderby clause and beyond if this is unpaged data
parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal));
}
@@ -524,7 +553,7 @@ WHERE EXISTS(
INNER JOIN cmsPropertyType
ON b.datatypeNodeId = cmsPropertyType.dataTypeId
INNER JOIN
(" + string.Format(parsedOriginalSql, "DISTINCT cmsContent.contentType") + @") as docData
(" + string.Format(parsedOriginalSql, "cmsContent.contentType") + @") as docData
ON cmsPropertyType.contentTypeId = docData.contentType
WHERE a.id = b.id)", docSql.Arguments);
@@ -647,28 +676,7 @@ ORDER BY contentNodeId, propertytypeid
return result;
}
public class DocumentDefinition
{
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition)
{
Id = id;
Version = version;
VersionDate = versionDate;
CreateDate = createDate;
Composition = composition;
}
public int Id { get; set; }
public Guid Version { get; set; }
public DateTime VersionDate { get; set; }
public DateTime CreateDate { get; set; }
public IContentTypeComposition Composition { get; set; }
}
protected virtual string GetDatabaseFieldNameForOrderBy(string orderBy)
{
// Translate the passed order by field (which were originally defined for in-memory object sorting
@@ -764,5 +772,92 @@ ORDER BY contentNodeId, propertytypeid
/// <param name="queryType"></param>
/// <returns></returns>
protected abstract Sql GetBaseQuery(BaseQueryType queryType);
internal class DocumentDefinition
{
/// <summary>
/// Initializes a new instance of the <see cref="T:System.Object"/> class.
/// </summary>
public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition)
{
Id = id;
Version = version;
VersionDate = versionDate;
CreateDate = createDate;
Composition = composition;
}
public int Id { get; set; }
public Guid Version { get; set; }
public DateTime VersionDate { get; set; }
public DateTime CreateDate { get; set; }
public IContentTypeComposition Composition { get; set; }
}
/// <summary>
/// An object representing a query that may contain paging information
/// </summary>
internal class PagingSqlQuery
{
public Sql PrePagedSql { get; private set; }
public PagingSqlQuery(Sql prePagedSql)
{
PrePagedSql = prePagedSql;
}
public virtual bool HasPaging
{
get { return false; }
}
public virtual Sql BuildPagedQuery(string selectColumns)
{
throw new InvalidOperationException("This query has no paging information");
}
}
/// <summary>
/// An object representing a query that contains paging information
/// </summary>
/// <typeparam name="T"></typeparam>
internal class PagingSqlQuery<T> : PagingSqlQuery
{
private readonly Database _db;
private readonly long _pageIndex;
private readonly int _pageSize;
public PagingSqlQuery(Database db, Sql prePagedSql, long pageIndex, int pageSize) : base(prePagedSql)
{
_db = db;
_pageIndex = pageIndex;
_pageSize = pageSize;
}
public override bool HasPaging
{
get { return _pageSize > 0; }
}
/// <summary>
/// Creates a paged query based on the original query and subtitutes the selectColumns specified
/// </summary>
/// <param name="selectColumns"></param>
/// <returns></returns>
public override Sql BuildPagedQuery(string selectColumns)
{
if (HasPaging == false) throw new InvalidOperationException("This query has no paging information");
var resultSql = string.Format("SELECT {0} {1}", selectColumns, PrePagedSql.SQL.Substring(PrePagedSql.SQL.IndexOf("FROM", StringComparison.Ordinal)));
//this query is meant to be paged so we need to generate the paging syntax
//Create the inner paged query that was used above to get the paged result, we'll use that as the inner sub query
var args = PrePagedSql.Arguments;
string sqlStringCount, sqlStringPage;
_db.BuildPageQueries<T>(_pageIndex * _pageSize, _pageSize, resultSql, ref args, out sqlStringCount, out sqlStringPage);
return new Sql(sqlStringPage, args);
}
}
}
}