yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improves the paging for member repository, no longer doing an IN clause and instead an inner join = far better. Created a helper method to do this on the base repository so i can share the logic with content and media repository.

Browse Source
This commit is contained in:
Shannon
2014-09-17 18:32:13 +10:00
parent 102b97fa82
commit 4828869713
5 changed files with 248 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
View File

@@ -796,7 +796,7 @@ namespace Umbraco.Core.Persistence.Repositories
var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType);
var properties = GetPropertyCollection(docSql, docDef);
var properties = GetPropertyCollection(docSql, new[] { docDef });
content.Properties = properties[dto.NodeId];

4
src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
View File

@@ -160,7 +160,7 @@ namespace Umbraco.Core.Persistence.Repositories
var factory = new MediaFactory(mediaType, NodeObjectTypeId, dto.NodeId);
var media = factory.BuildEntity(dto);
var properties = GetPropertyCollection(sql, new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType));
var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType) });
media.Properties = properties[dto.NodeId];
@@ -504,7 +504,7 @@ namespace Umbraco.Core.Persistence.Repositories
var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType);
var properties = GetPropertyCollection(docSql, docDef);
var properties = GetPropertyCollection(docSql, new[] { docDef });
media.Properties = properties[dto.NodeId];

255
src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
View File

@@ -497,7 +497,7 @@ namespace Umbraco.Core.Persistence.Repositories
var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId);
var media = factory.BuildEntity(dto);
var properties = GetPropertyCollection(sql, new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType));
var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) });
media.Properties = properties[dto.NodeId];
@@ -648,124 +648,99 @@ namespace Umbraco.Core.Persistence.Repositories
public IEnumerable<IMember> GetPagedResultsByQuery(IQuery<IMember> query, int pageIndex, int pageSize, out int totalRecords,
string orderBy, Direction orderDirection, string filter = "")
{
if (orderBy == null) throw new ArgumentNullException("orderBy");
// Get base query
var sqlClause = GetBaseQuery(false);
if (query == null) query = new Query<IMember>();
var translator = new SqlTranslator<IMember>(sqlClause, query);
var sql = translator.Translate();
// Apply filter
if (string.IsNullOrEmpty(filter) == false)
{
return GetPagedResultsByQuery<MemberDto>(query, pageIndex, pageSize, out totalRecords,
"SELECT cmsMember.nodeId",
//TODO: Maybe other filters?
sql = sql.Where("cmsMember.LoginName LIKE @0", "%" + filter + "%");
}
() => new Tuple<string, object[]>("AND (cmsMember.LoginName LIKE @0", new object[] { "%" + filter + "%)" })
, ProcessQuery, orderBy, orderDirection, filter);
// Apply order according to parameters
if (string.IsNullOrEmpty(orderBy) == false)
{
var orderByParams = new[] { GetDatabaseFieldNameForOrderBy( orderBy) };
if (orderDirection == Direction.Ascending)
{
sql = sql.OrderBy(orderByParams);
}
else
{
sql = sql.OrderByDescending(orderByParams);
}
}
//if (orderBy == null) throw new ArgumentNullException("orderBy");
// Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery,
// but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id.
// So we'll modify the SQL.
var modifiedSQL = sql.SQL.Replace("SELECT *", "SELECT cmsMember.nodeId");
//// Get base query
//var sqlBase = GetBaseQuery(false);
// Get page of results and total count
IEnumerable<IMember> result;
var pagedResult = Database.Page<MemberDto>(pageIndex + 1, pageSize, modifiedSQL, sql.Arguments);
totalRecords = Convert.ToInt32(pagedResult.TotalItems);
if (totalRecords > 0)
{
// Parse out node Ids and load content (we need the cast here in order to be able to call the IQueryable extension
// methods OrderBy or OrderByDescending)
var content = GetAll(pagedResult.Items
.DistinctBy(x => x.NodeId)
.Select(x => x.NodeId).ToArray())
.Cast<Member>()
.AsQueryable();
//if (query == null) query = new Query<IMember>();
//var translator = new SqlTranslator<IMember>(sqlBase, query);
//var sqlQuery = translator.Translate();
// Now we need to ensure this result is also ordered by the same order by clause
var orderByProperty = GetEntityPropertyNameForOrderBy(orderBy);
if (orderDirection == Direction.Ascending)
{
result = content.OrderBy(orderByProperty);
}
else
{
result = content.OrderByDescending(orderByProperty);
}
}
else
{
result = Enumerable.Empty<IMember>();
}
//Func<Sql, string, Sql> getFilteredSql = (sql, additionalFilter) =>
//{
// //copy to var so that the original isn't changed
// var filteredSql = new Sql(sql.SQL, sql.Arguments);
// // Apply filter
// if (string.IsNullOrEmpty(filter) == false)
// {
// //TODO: Maybe other filters?
// filteredSql.Append("AND (cmsMember.LoginName LIKE @0", "%" + filter + "%)");
// }
// if (string.IsNullOrEmpty(additionalFilter) == false)
// {
// filteredSql.Append("AND (" + additionalFilter + ")");
// }
// return filteredSql;
//};
return result;
//Func<Sql, Sql> getSortedSql = sql =>
//{
// //copy to var so that the original isn't changed
// var sortedSql = new Sql(sql.SQL, sql.Arguments);
// // Apply order according to parameters
// if (string.IsNullOrEmpty(orderBy) == false)
// {
// var orderByParams = new[] { GetDatabaseFieldNameForOrderBy(orderBy) };
// if (orderDirection == Direction.Ascending)
// {
// sortedSql.OrderBy(orderByParams);
// }
// else
// {
// sortedSql.OrderByDescending(orderByParams);
// }
// return sortedSql;
// }
// return sortedSql;
//};
//// Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery,
//// but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id.
//// So we'll modify the SQL.
//var sqlNodeIds = new Sql(sqlQuery.SQL.Replace("SELECT *", "SELECT cmsMember.nodeId"), sqlQuery.Arguments);
//var sqlNodeIdsWithSort = getSortedSql(
// getFilteredSql(sqlNodeIds, null));
//// Get page of results and total count
//IEnumerable<IMember> result;
//var pagedResult = Database.Page<MemberDto>(pageIndex + 1, pageSize, sqlNodeIdsWithSort);
//totalRecords = Convert.ToInt32(pagedResult.TotalItems);
//if (totalRecords > 0)
//{
// var fullQuery = getSortedSql(
// getFilteredSql(sqlQuery, string.Format("umbracoNode.id IN ({0})", sqlNodeIds.SQL)));
// var content = ProcessQuery(fullQuery)
// .Cast<Member>()
// .AsQueryable();
// // Now we need to ensure this result is also ordered by the same order by clause
// var orderByProperty = GetEntityPropertyNameForOrderBy(orderBy);
// if (orderDirection == Direction.Ascending)
// {
// result = content.OrderBy(orderByProperty);
// }
// else
// {
// result = content.OrderByDescending(orderByProperty);
// }
//}
//else
//{
// result = Enumerable.Empty<IMember>();
//}
//return result;
}
//public IEnumerable<IMember> GetPagedResultsByQuery<TDto>(
// Sql sql, int pageIndex, int pageSize, out int totalRecords,
// Func<IEnumerable<TDto>, int[]> resolveIds)
//{
// ////If a max size is passed in, just do a normal get all!
// //if (pageSize == int.MaxValue)
// //{
// // var result = GetAll().ToArray();
// // totalRecords = result.Count();
// // return result;
// //}
// //// Get base query
// //var sqlClause = GetBaseQuery(false);
// //var translator = new SqlTranslator<IContent>(sqlClause, query);
// //var sql = translator.Translate()
// // .Where<DocumentDto>(x => x.Newest);
// //// Apply filter
// //if (!string.IsNullOrEmpty(filter))
// //{
// // sql = sql.Where("cmsDocument.text LIKE @0", "%" + filter + "%");
// //}
// //// Apply order according to parameters
// //if (!string.IsNullOrEmpty(orderBy))
// //{
// // var orderByParams = new[] { GetDatabaseFieldNameForOrderBy(orderBy) };
// // if (orderDirection == Direction.Ascending)
// // {
// // sql = sql.OrderBy(orderByParams);
// // }
// // else
// // {
// // sql = sql.OrderByDescending(orderByParams);
// // }
// //}
// var pagedResult = Database.Page<TDto>(pageIndex + 1, pageSize, sql);
// totalRecords = Convert.ToInt32(pagedResult.TotalItems);
// //now that we have the member dto's we need to construct true members from the list.
// if (totalRecords == 0)
// {
// return Enumerable.Empty<IMember>();
// }
// return GetAll(resolveIds(pagedResult.Items)).ToArray();
//}
public void AddOrUpdateContentXml(IMember content, Func<IMember, XElement> xml)
{
var contentExists = Database.ExecuteScalar<int>("SELECT COUNT(nodeId) FROM cmsContentXml WHERE nodeId = @Id", new { Id = content.Id }) != 0;
@@ -823,8 +798,7 @@ namespace Umbraco.Core.Persistence.Repositories
dto.ContentVersionDto.VersionId,
dto.ContentVersionDto.VersionDate,
dto.ContentVersionDto.ContentDto.NodeDto.CreateDate,
contentTypes.First(ct => ct.Id == dto.ContentVersionDto.ContentDto.ContentTypeId)))
.ToArray();
contentTypes.First(ct => ct.Id == dto.ContentVersionDto.ContentDto.ContentTypeId)));
var propertyData = GetPropertyCollection(sql, docDefs);
@@ -872,7 +846,7 @@ namespace Umbraco.Core.Persistence.Repositories
var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType);
var properties = GetPropertyCollection(docSql, docDef);
var properties = GetPropertyCollection(docSql, new[] { docDef });
member.Properties = properties[dto.ContentVersionDto.NodeId];
@@ -881,56 +855,5 @@ namespace Umbraco.Core.Persistence.Repositories
((Entity)member).ResetDirtyProperties(false);
return member;
}
//private IMember BuildFromDto(List<MemberReadOnlyDto> dtos, Sql docSql)
//{
// if (dtos == null || dtos.Any() == false)
// return null;
// var dto = dtos.First();
// var memberTypes = new Dictionary<string, IMemberType>
// {
// {
// dto.ContentTypeAlias,
// _memberTypeRepository.Get(dto.ContentTypeId)
// }
// };
// var factory = new MemberReadOnlyFactory(memberTypes);
// var member = factory.BuildEntity(dto);
// var properties = GetPropertyCollection(docSql, new DocumentDefinition(dto.NodeId, dto.VersionId, dto.UpdateDate, dto.CreateDate, member.ContentType));
// member.Properties = properties[dto.NodeId];
// return member;
//}
//private IEnumerable<IMember> BuildFromDtos(List<MemberReadOnlyDto> dtos, Sql docSql)
//{
// if (dtos == null || dtos.Any() == false)
// return Enumerable.Empty<IMember>();
// //We assume that there won't exist a lot of MemberTypes, so the following should be fairly fast
// var memberTypes = new Dictionary<string, IMemberType>();
// //TODO: We should do an SQL 'IN' here
// var memberTypeList = _memberTypeRepository.GetAll();
// memberTypeList.ForEach(x => memberTypes.Add(x.Alias, x));
// var entities = new List<IMember>();
// var factory = new MemberReadOnlyFactory(memberTypes);
// foreach (var dto in dtos)
// {
// var entity = factory.BuildEntity(dto);
// var properties = GetPropertyCollection(docSql,new DocumentDefinition(dto.NodeId, dto.VersionId, dto.UpdateDate, dto.CreateDate, entity.ContentType));
// entity.Properties = properties[dto.NodeId];
// entities.Add(entity);
// }
// return entities;
//}
}
}

10
src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs
View File

@@ -133,11 +133,6 @@ namespace Umbraco.Core.Persistence.Repositories
/// <returns></returns>
public IEnumerable<TEntity> GetAll(params TId[] ids)
{
if (ids.Length > 2000)
{
throw new InvalidOperationException("Cannot perform a query with more than 2000 parameters");
}
//ensure they are de-duplicated, easy win if people don't do this as this can cause many excess queries
ids = ids.Distinct()
//don't query by anything that is a default of T (like a zero)
@@ -145,6 +140,11 @@ namespace Umbraco.Core.Persistence.Repositories
//.Where(x => Equals(x, default(TId)) == false)
.ToArray();
if (ids.Length > 2000)
{
throw new InvalidOperationException("Cannot perform a query with more than 2000 parameters");
}
if (ids.Any())
{
var entities = _cache.GetByIds(

190
src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
View File

@@ -1,15 +1,19 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Editors;
using Umbraco.Core.Models.EntityBase;
using Umbraco.Core.Models.Rdbms;
using Umbraco.Core.Persistence.Caching;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Factories;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Persistence.UnitOfWork;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Services;
using Umbraco.Core.Dynamics;
namespace Umbraco.Core.Persistence.Repositories
{
@@ -219,11 +223,106 @@ namespace Umbraco.Core.Persistence.Repositories
}
}
protected IEnumerable<TEntity> GetPagedResultsByQuery<TDto>(IQuery<TEntity> query, int pageIndex, int pageSize, out int totalRecords,
string nodeIdSelect,
Func<Tuple<string, object[]>> defaultFilter,
Func<Sql, IEnumerable<TEntity>> processQuery,
string orderBy,
Direction orderDirection,
string filter = "")
{
if (orderBy == null) throw new ArgumentNullException("orderBy");
// Get base query
var sqlBase = GetBaseQuery(false);
if (query == null) query = new Query<TEntity>();
var translator = new SqlTranslator<TEntity>(sqlBase, query);
var sqlQuery = translator.Translate();
Func<Sql, string, Sql> getFilteredSql = (sql, additionalFilter) =>
{
//copy to var so that the original isn't changed
var filteredSql = new Sql(sql.SQL, sql.Arguments);
// Apply filter
if (string.IsNullOrEmpty(filter) == false)
{
var f = defaultFilter();
filteredSql.Append(f.Item1, f.Item2);
}
if (string.IsNullOrEmpty(additionalFilter) == false)
{
filteredSql.Append("AND (" + additionalFilter + ")");
}
return filteredSql;
};
Func<Sql, Sql> getSortedSql = sql =>
{
//copy to var so that the original isn't changed
var sortedSql = new Sql(sql.SQL, sql.Arguments);
// Apply order according to parameters
if (string.IsNullOrEmpty(orderBy) == false)
{
var orderByParams = new[] { GetDatabaseFieldNameForOrderBy(orderBy) };
if (orderDirection == Direction.Ascending)
{
sortedSql.OrderBy(orderByParams);
}
else
{
sortedSql.OrderByDescending(orderByParams);
}
return sortedSql;
}
return sortedSql;
};
// Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery,
// but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id.
// So we'll modify the SQL.
var sqlNodeIds = new Sql(sqlQuery.SQL.Replace("SELECT *", nodeIdSelect), sqlQuery.Arguments);
var sqlNodeIdsWithSort = getSortedSql(
getFilteredSql(sqlNodeIds, null));
// Get page of results and total count
IEnumerable<TEntity> result;
var pagedResult = Database.Page<TDto>(pageIndex + 1, pageSize, sqlNodeIdsWithSort);
totalRecords = Convert.ToInt32(pagedResult.TotalItems);
if (totalRecords > 0)
{
var fullQuery = getSortedSql(
getFilteredSql(sqlQuery, string.Format("umbracoNode.id IN ({0})", sqlNodeIds.SQL)));
var content = processQuery(fullQuery)
//.Cast<Member>()
.AsQueryable();
// Now we need to ensure this result is also ordered by the same order by clause
var orderByProperty = GetEntityPropertyNameForOrderBy(orderBy);
if (orderDirection == Direction.Ascending)
{
result = content.OrderBy(orderByProperty);
}
else
{
result = content.OrderByDescending(orderByProperty);
}
}
else
{
result = Enumerable.Empty<TEntity>();
}
return result;
}
protected IDictionary<int, PropertyCollection> GetPropertyCollection(
Sql docSql,
params DocumentDefinition[] documentDefs)
IEnumerable<DocumentDefinition> documentDefs)
{
if (documentDefs.Length <= 0) return new Dictionary<int, PropertyCollection>();
if (documentDefs.Any() == false) return new Dictionary<int, PropertyCollection>();
//we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use
// the statement to go get the property data for all of the items by using an inner join
@@ -231,7 +330,7 @@ namespace Umbraco.Core.Persistence.Repositories
//now remove everything from an Orderby clause and beyond
if (parsedOriginalSql.InvariantContains("ORDER BY "))
{
parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.IndexOf("ORDER BY ", System.StringComparison.Ordinal));
parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.IndexOf("ORDER BY ", StringComparison.Ordinal));
}
var propSql = new Sql(@"SELECT cmsPropertyData.*
@@ -264,49 +363,62 @@ ON cmsPropertyType.contentTypeId = docData.contentType", docSql.Arguments);
var result = new Dictionary<int, PropertyCollection>();
foreach (var def in documentDefs)
var propertiesWithTagSupport = new Dictionary<string, SupportTagsAttribute>();
using (DisposableTimer.DebugDuration<VersionableRepositoryBase<TId, TEntity>>(
() => "Building properties for each document", () => "Finished building properties"))
{
var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct();
foreach (var def in documentDefs)
{
var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct();
var propertyFactory = new PropertyFactory(def.Composition, def.Version, def.Id, def.CreateDate, def.VersionDate);
var properties = propertyFactory.BuildEntity(propertyDataDtos).ToArray();
var propertyFactory = new PropertyFactory(def.Composition, def.Version, def.Id, def.CreateDate, def.VersionDate);
var properties = propertyFactory.BuildEntity(propertyDataDtos).ToArray();
var newProperties = properties.Where(x => x.HasIdentity == false && x.PropertyType.HasIdentity);
foreach (var property in newProperties)
{
var propertyDataDto = new PropertyDataDto { NodeId = def.Id, PropertyTypeId = property.PropertyTypeId, VersionId = def.Version };
int primaryKey = Convert.ToInt32(Database.Insert(propertyDataDto));
property.Version = def.Version;
property.Id = primaryKey;
}
foreach (var property in properties)
{
//NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck
var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias);
var tagSupport = TagExtractor.GetAttribute(editor);
if (tagSupport != null)
var newProperties = properties.Where(x => x.HasIdentity == false && x.PropertyType.HasIdentity);
foreach (var property in newProperties)
{
//this property has tags, so we need to extract them and for that we need the prevals which we've already looked up
var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
.Distinct()
.ToArray();
var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder));
var propertyDataDto = new PropertyDataDto { NodeId = def.Id, PropertyTypeId = property.PropertyTypeId, VersionId = def.Version };
int primaryKey = Convert.ToInt32(Database.Insert(propertyDataDto));
var preVals = new PreValueCollection(asDictionary);
var contentPropData = new ContentPropertyData(property.Value,
preVals,
new Dictionary<string, object>());
TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport);
property.Version = def.Version;
property.Id = primaryKey;
}
}
result.Add(def.Id, new PropertyCollection(properties));
foreach (var property in properties)
{
//NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck
var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias);
var tagSupport = propertiesWithTagSupport.ContainsKey(property.PropertyType.PropertyEditorAlias)
? propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias]
: TagExtractor.GetAttribute(editor);
if (tagSupport != null)
{
//add to local cache so we don't need to reflect next time for this property editor alias
propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport;
//this property has tags, so we need to extract them and for that we need the prevals which we've already looked up
var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId)
.Distinct()
.ToArray();
var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder));
var preVals = new PreValueCollection(asDictionary);
var contentPropData = new ContentPropertyData(property.Value,
preVals,
new Dictionary<string, object>());
TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport);
}
}
result.Add(def.Id, new PropertyCollection(properties));
}
}
return result;