diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 8ed8f2ba63..431337dae9 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -796,7 +796,7 @@ namespace Umbraco.Core.Persistence.Repositories var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType); - var properties = GetPropertyCollection(docSql, docDef); + var properties = GetPropertyCollection(docSql, new[] { docDef }); content.Properties = properties[dto.NodeId]; diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index 4612acb470..8d3686cc53 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -160,7 +160,7 @@ namespace Umbraco.Core.Persistence.Repositories var factory = new MediaFactory(mediaType, NodeObjectTypeId, dto.NodeId); var media = factory.BuildEntity(dto); - var properties = GetPropertyCollection(sql, new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType)); + var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType) }); media.Properties = properties[dto.NodeId]; @@ -504,7 +504,7 @@ namespace Umbraco.Core.Persistence.Repositories var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType); - var properties = GetPropertyCollection(docSql, docDef); + var properties = GetPropertyCollection(docSql, new[] { docDef }); media.Properties = properties[dto.NodeId]; diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index 059f6f0a8f..36e68eccbd 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -497,7 +497,7 @@ namespace Umbraco.Core.Persistence.Repositories var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId); var media = factory.BuildEntity(dto); - var properties = GetPropertyCollection(sql, new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType)); + var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) }); media.Properties = properties[dto.NodeId]; @@ -648,124 +648,99 @@ namespace Umbraco.Core.Persistence.Repositories public IEnumerable GetPagedResultsByQuery(IQuery query, int pageIndex, int pageSize, out int totalRecords, string orderBy, Direction orderDirection, string filter = "") { - if (orderBy == null) throw new ArgumentNullException("orderBy"); - - // Get base query - var sqlClause = GetBaseQuery(false); - - if (query == null) query = new Query(); - var translator = new SqlTranslator(sqlClause, query); - var sql = translator.Translate(); - - // Apply filter - if (string.IsNullOrEmpty(filter) == false) - { + return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, + "SELECT cmsMember.nodeId", //TODO: Maybe other filters? - sql = sql.Where("cmsMember.LoginName LIKE @0", "%" + filter + "%"); - } + () => new Tuple("AND (cmsMember.LoginName LIKE @0", new object[] { "%" + filter + "%)" }) + , ProcessQuery, orderBy, orderDirection, filter); - // Apply order according to parameters - if (string.IsNullOrEmpty(orderBy) == false) - { - var orderByParams = new[] { GetDatabaseFieldNameForOrderBy( orderBy) }; - if (orderDirection == Direction.Ascending) - { - sql = sql.OrderBy(orderByParams); - } - else - { - sql = sql.OrderByDescending(orderByParams); - } - } + //if (orderBy == null) throw new ArgumentNullException("orderBy"); - // Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery, - // but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id. - // So we'll modify the SQL. - var modifiedSQL = sql.SQL.Replace("SELECT *", "SELECT cmsMember.nodeId"); + //// Get base query + //var sqlBase = GetBaseQuery(false); - // Get page of results and total count - IEnumerable result; - var pagedResult = Database.Page(pageIndex + 1, pageSize, modifiedSQL, sql.Arguments); - totalRecords = Convert.ToInt32(pagedResult.TotalItems); - if (totalRecords > 0) - { - // Parse out node Ids and load content (we need the cast here in order to be able to call the IQueryable extension - // methods OrderBy or OrderByDescending) - var content = GetAll(pagedResult.Items - .DistinctBy(x => x.NodeId) - .Select(x => x.NodeId).ToArray()) - .Cast() - .AsQueryable(); + //if (query == null) query = new Query(); + //var translator = new SqlTranslator(sqlBase, query); + //var sqlQuery = translator.Translate(); - // Now we need to ensure this result is also ordered by the same order by clause - var orderByProperty = GetEntityPropertyNameForOrderBy(orderBy); - if (orderDirection == Direction.Ascending) - { - result = content.OrderBy(orderByProperty); - } - else - { - result = content.OrderByDescending(orderByProperty); - } - } - else - { - result = Enumerable.Empty(); - } + //Func getFilteredSql = (sql, additionalFilter) => + //{ + // //copy to var so that the original isn't changed + // var filteredSql = new Sql(sql.SQL, sql.Arguments); + // // Apply filter + // if (string.IsNullOrEmpty(filter) == false) + // { + // //TODO: Maybe other filters? + // filteredSql.Append("AND (cmsMember.LoginName LIKE @0", "%" + filter + "%)"); + // } + // if (string.IsNullOrEmpty(additionalFilter) == false) + // { + // filteredSql.Append("AND (" + additionalFilter + ")"); + // } + // return filteredSql; + //}; - return result; + //Func getSortedSql = sql => + //{ + // //copy to var so that the original isn't changed + // var sortedSql = new Sql(sql.SQL, sql.Arguments); + // // Apply order according to parameters + // if (string.IsNullOrEmpty(orderBy) == false) + // { + // var orderByParams = new[] { GetDatabaseFieldNameForOrderBy(orderBy) }; + // if (orderDirection == Direction.Ascending) + // { + // sortedSql.OrderBy(orderByParams); + // } + // else + // { + // sortedSql.OrderByDescending(orderByParams); + // } + // return sortedSql; + // } + // return sortedSql; + //}; + + //// Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery, + //// but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id. + //// So we'll modify the SQL. + //var sqlNodeIds = new Sql(sqlQuery.SQL.Replace("SELECT *", "SELECT cmsMember.nodeId"), sqlQuery.Arguments); + + //var sqlNodeIdsWithSort = getSortedSql( + // getFilteredSql(sqlNodeIds, null)); + + //// Get page of results and total count + //IEnumerable result; + //var pagedResult = Database.Page(pageIndex + 1, pageSize, sqlNodeIdsWithSort); + //totalRecords = Convert.ToInt32(pagedResult.TotalItems); + //if (totalRecords > 0) + //{ + // var fullQuery = getSortedSql( + // getFilteredSql(sqlQuery, string.Format("umbracoNode.id IN ({0})", sqlNodeIds.SQL))); + + // var content = ProcessQuery(fullQuery) + // .Cast() + // .AsQueryable(); + + // // Now we need to ensure this result is also ordered by the same order by clause + // var orderByProperty = GetEntityPropertyNameForOrderBy(orderBy); + // if (orderDirection == Direction.Ascending) + // { + // result = content.OrderBy(orderByProperty); + // } + // else + // { + // result = content.OrderByDescending(orderByProperty); + // } + //} + //else + //{ + // result = Enumerable.Empty(); + //} + + //return result; } - - //public IEnumerable GetPagedResultsByQuery( - // Sql sql, int pageIndex, int pageSize, out int totalRecords, - // Func, int[]> resolveIds) - //{ - // ////If a max size is passed in, just do a normal get all! - // //if (pageSize == int.MaxValue) - // //{ - // // var result = GetAll().ToArray(); - // // totalRecords = result.Count(); - // // return result; - // //} - - // //// Get base query - // //var sqlClause = GetBaseQuery(false); - // //var translator = new SqlTranslator(sqlClause, query); - // //var sql = translator.Translate() - // // .Where(x => x.Newest); - - // //// Apply filter - // //if (!string.IsNullOrEmpty(filter)) - // //{ - // // sql = sql.Where("cmsDocument.text LIKE @0", "%" + filter + "%"); - // //} - - // //// Apply order according to parameters - // //if (!string.IsNullOrEmpty(orderBy)) - // //{ - // // var orderByParams = new[] { GetDatabaseFieldNameForOrderBy(orderBy) }; - // // if (orderDirection == Direction.Ascending) - // // { - // // sql = sql.OrderBy(orderByParams); - // // } - // // else - // // { - // // sql = sql.OrderByDescending(orderByParams); - // // } - // //} - - // var pagedResult = Database.Page(pageIndex + 1, pageSize, sql); - - // totalRecords = Convert.ToInt32(pagedResult.TotalItems); - - // //now that we have the member dto's we need to construct true members from the list. - // if (totalRecords == 0) - // { - // return Enumerable.Empty(); - // } - // return GetAll(resolveIds(pagedResult.Items)).ToArray(); - //} - + public void AddOrUpdateContentXml(IMember content, Func xml) { var contentExists = Database.ExecuteScalar("SELECT COUNT(nodeId) FROM cmsContentXml WHERE nodeId = @Id", new { Id = content.Id }) != 0; @@ -823,8 +798,7 @@ namespace Umbraco.Core.Persistence.Repositories dto.ContentVersionDto.VersionId, dto.ContentVersionDto.VersionDate, dto.ContentVersionDto.ContentDto.NodeDto.CreateDate, - contentTypes.First(ct => ct.Id == dto.ContentVersionDto.ContentDto.ContentTypeId))) - .ToArray(); + contentTypes.First(ct => ct.Id == dto.ContentVersionDto.ContentDto.ContentTypeId))); var propertyData = GetPropertyCollection(sql, docDefs); @@ -872,7 +846,7 @@ namespace Umbraco.Core.Persistence.Repositories var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType); - var properties = GetPropertyCollection(docSql, docDef); + var properties = GetPropertyCollection(docSql, new[] { docDef }); member.Properties = properties[dto.ContentVersionDto.NodeId]; @@ -881,56 +855,5 @@ namespace Umbraco.Core.Persistence.Repositories ((Entity)member).ResetDirtyProperties(false); return member; } - - //private IMember BuildFromDto(List dtos, Sql docSql) - //{ - // if (dtos == null || dtos.Any() == false) - // return null; - // var dto = dtos.First(); - - // var memberTypes = new Dictionary - // { - // { - // dto.ContentTypeAlias, - // _memberTypeRepository.Get(dto.ContentTypeId) - // } - // }; - - // var factory = new MemberReadOnlyFactory(memberTypes); - // var member = factory.BuildEntity(dto); - - // var properties = GetPropertyCollection(docSql, new DocumentDefinition(dto.NodeId, dto.VersionId, dto.UpdateDate, dto.CreateDate, member.ContentType)); - - // member.Properties = properties[dto.NodeId]; - - // return member; - //} - - //private IEnumerable BuildFromDtos(List dtos, Sql docSql) - //{ - // if (dtos == null || dtos.Any() == false) - // return Enumerable.Empty(); - - // //We assume that there won't exist a lot of MemberTypes, so the following should be fairly fast - // var memberTypes = new Dictionary(); - - // //TODO: We should do an SQL 'IN' here - // var memberTypeList = _memberTypeRepository.GetAll(); - // memberTypeList.ForEach(x => memberTypes.Add(x.Alias, x)); - - // var entities = new List(); - // var factory = new MemberReadOnlyFactory(memberTypes); - // foreach (var dto in dtos) - // { - // var entity = factory.BuildEntity(dto); - - // var properties = GetPropertyCollection(docSql,new DocumentDefinition(dto.NodeId, dto.VersionId, dto.UpdateDate, dto.CreateDate, entity.ContentType)); - - // entity.Properties = properties[dto.NodeId]; - - // entities.Add(entity); - // } - // return entities; - //} } } diff --git a/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs index 1424861d52..4ebe4ad67e 100644 --- a/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/RepositoryBase.cs @@ -133,11 +133,6 @@ namespace Umbraco.Core.Persistence.Repositories /// public IEnumerable GetAll(params TId[] ids) { - if (ids.Length > 2000) - { - throw new InvalidOperationException("Cannot perform a query with more than 2000 parameters"); - } - //ensure they are de-duplicated, easy win if people don't do this as this can cause many excess queries ids = ids.Distinct() //don't query by anything that is a default of T (like a zero) @@ -145,6 +140,11 @@ namespace Umbraco.Core.Persistence.Repositories //.Where(x => Equals(x, default(TId)) == false) .ToArray(); + if (ids.Length > 2000) + { + throw new InvalidOperationException("Cannot perform a query with more than 2000 parameters"); + } + if (ids.Any()) { var entities = _cache.GetByIds( diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index 72a54a337f..fc96d4642e 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -1,15 +1,19 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using Umbraco.Core.Models; using Umbraco.Core.Models.Editors; using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence.Caching; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Factories; +using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.UnitOfWork; using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; +using Umbraco.Core.Dynamics; namespace Umbraco.Core.Persistence.Repositories { @@ -219,11 +223,106 @@ namespace Umbraco.Core.Persistence.Repositories } } + protected IEnumerable GetPagedResultsByQuery(IQuery query, int pageIndex, int pageSize, out int totalRecords, + string nodeIdSelect, + Func> defaultFilter, + Func> processQuery, + string orderBy, + Direction orderDirection, + string filter = "") + { + if (orderBy == null) throw new ArgumentNullException("orderBy"); + + // Get base query + var sqlBase = GetBaseQuery(false); + + if (query == null) query = new Query(); + var translator = new SqlTranslator(sqlBase, query); + var sqlQuery = translator.Translate(); + + Func getFilteredSql = (sql, additionalFilter) => + { + //copy to var so that the original isn't changed + var filteredSql = new Sql(sql.SQL, sql.Arguments); + // Apply filter + if (string.IsNullOrEmpty(filter) == false) + { + var f = defaultFilter(); + filteredSql.Append(f.Item1, f.Item2); + } + if (string.IsNullOrEmpty(additionalFilter) == false) + { + filteredSql.Append("AND (" + additionalFilter + ")"); + } + return filteredSql; + }; + + Func getSortedSql = sql => + { + //copy to var so that the original isn't changed + var sortedSql = new Sql(sql.SQL, sql.Arguments); + // Apply order according to parameters + if (string.IsNullOrEmpty(orderBy) == false) + { + var orderByParams = new[] { GetDatabaseFieldNameForOrderBy(orderBy) }; + if (orderDirection == Direction.Ascending) + { + sortedSql.OrderBy(orderByParams); + } + else + { + sortedSql.OrderByDescending(orderByParams); + } + return sortedSql; + } + return sortedSql; + }; + + // Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery, + // but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id. + // So we'll modify the SQL. + var sqlNodeIds = new Sql(sqlQuery.SQL.Replace("SELECT *", nodeIdSelect), sqlQuery.Arguments); + + var sqlNodeIdsWithSort = getSortedSql( + getFilteredSql(sqlNodeIds, null)); + + // Get page of results and total count + IEnumerable result; + var pagedResult = Database.Page(pageIndex + 1, pageSize, sqlNodeIdsWithSort); + totalRecords = Convert.ToInt32(pagedResult.TotalItems); + if (totalRecords > 0) + { + var fullQuery = getSortedSql( + getFilteredSql(sqlQuery, string.Format("umbracoNode.id IN ({0})", sqlNodeIds.SQL))); + + var content = processQuery(fullQuery) + //.Cast() + .AsQueryable(); + + // Now we need to ensure this result is also ordered by the same order by clause + var orderByProperty = GetEntityPropertyNameForOrderBy(orderBy); + if (orderDirection == Direction.Ascending) + { + result = content.OrderBy(orderByProperty); + } + else + { + result = content.OrderByDescending(orderByProperty); + } + } + else + { + result = Enumerable.Empty(); + } + + return result; + } + protected IDictionary GetPropertyCollection( Sql docSql, - params DocumentDefinition[] documentDefs) + IEnumerable documentDefs) { - if (documentDefs.Length <= 0) return new Dictionary(); + if (documentDefs.Any() == false) return new Dictionary(); //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use // the statement to go get the property data for all of the items by using an inner join @@ -231,7 +330,7 @@ namespace Umbraco.Core.Persistence.Repositories //now remove everything from an Orderby clause and beyond if (parsedOriginalSql.InvariantContains("ORDER BY ")) { - parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.IndexOf("ORDER BY ", System.StringComparison.Ordinal)); + parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.IndexOf("ORDER BY ", StringComparison.Ordinal)); } var propSql = new Sql(@"SELECT cmsPropertyData.* @@ -264,49 +363,62 @@ ON cmsPropertyType.contentTypeId = docData.contentType", docSql.Arguments); var result = new Dictionary(); - foreach (var def in documentDefs) + var propertiesWithTagSupport = new Dictionary(); + + using (DisposableTimer.DebugDuration>( + () => "Building properties for each document", () => "Finished building properties")) { - var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct(); + foreach (var def in documentDefs) + { + var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct(); - var propertyFactory = new PropertyFactory(def.Composition, def.Version, def.Id, def.CreateDate, def.VersionDate); - var properties = propertyFactory.BuildEntity(propertyDataDtos).ToArray(); + var propertyFactory = new PropertyFactory(def.Composition, def.Version, def.Id, def.CreateDate, def.VersionDate); + var properties = propertyFactory.BuildEntity(propertyDataDtos).ToArray(); - var newProperties = properties.Where(x => x.HasIdentity == false && x.PropertyType.HasIdentity); - foreach (var property in newProperties) - { - var propertyDataDto = new PropertyDataDto { NodeId = def.Id, PropertyTypeId = property.PropertyTypeId, VersionId = def.Version }; - int primaryKey = Convert.ToInt32(Database.Insert(propertyDataDto)); - - property.Version = def.Version; - property.Id = primaryKey; - } - - foreach (var property in properties) - { - //NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck - var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias); - var tagSupport = TagExtractor.GetAttribute(editor); - - if (tagSupport != null) + var newProperties = properties.Where(x => x.HasIdentity == false && x.PropertyType.HasIdentity); + foreach (var property in newProperties) { - //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up - var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId) - .Distinct() - .ToArray(); - - var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder)); + var propertyDataDto = new PropertyDataDto { NodeId = def.Id, PropertyTypeId = property.PropertyTypeId, VersionId = def.Version }; + int primaryKey = Convert.ToInt32(Database.Insert(propertyDataDto)); - var preVals = new PreValueCollection(asDictionary); - - var contentPropData = new ContentPropertyData(property.Value, - preVals, - new Dictionary()); - - TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport); + property.Version = def.Version; + property.Id = primaryKey; } - } - result.Add(def.Id, new PropertyCollection(properties)); + foreach (var property in properties) + { + //NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck + var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias); + + var tagSupport = propertiesWithTagSupport.ContainsKey(property.PropertyType.PropertyEditorAlias) + ? propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] + : TagExtractor.GetAttribute(editor); + + if (tagSupport != null) + { + //add to local cache so we don't need to reflect next time for this property editor alias + propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport; + + //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up + var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId) + .Distinct() + .ToArray(); + + var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder)); + + var preVals = new PreValueCollection(asDictionary); + + var contentPropData = new ContentPropertyData(property.Value, + preVals, + new Dictionary()); + + TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport); + } + } + + + result.Add(def.Id, new PropertyCollection(properties)); + } } return result;