diff --git a/src/Umbraco.Core/Models/UmbracoEntityExtensions.cs b/src/Umbraco.Core/Models/UmbracoEntityExtensions.cs index 42e1a0d415..a27657cbe0 100644 --- a/src/Umbraco.Core/Models/UmbracoEntityExtensions.cs +++ b/src/Umbraco.Core/Models/UmbracoEntityExtensions.cs @@ -113,6 +113,11 @@ namespace Umbraco.Core.Models } } + /// + /// When resolved from EntityService this checks if the entity has the HasChildren flag + /// + /// + /// public static bool HasChildren(this IUmbracoEntity entity) { if (entity.AdditionalData.ContainsKey("HasChildren")) @@ -133,6 +138,11 @@ namespace Umbraco.Core.Models return entity.AdditionalData.GetValueIgnoreCase(key, defaultVal); } + /// + /// When resolved from EntityService this checks if the entity has the IsContainer flag + /// + /// + /// public static bool IsContainer(this IUmbracoEntity entity) { if (entity.AdditionalData.ContainsKeyIgnoreCase("IsContainer") == false) return false; diff --git a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs index 18073c088e..4eb3fe0659 100644 --- a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs +++ b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs @@ -50,7 +50,7 @@ namespace Umbraco.Core.Persistence.Factories entity.ContentTypeThumbnail = asDictionary.ContainsKey("thumbnail") ? (d.thumbnail ?? string.Empty) : string.Empty; var publishedVersion = default(Guid); - //some content items don't have a published version + //some content items don't have a published/newest version if (asDictionary.ContainsKey("publishedVersion") && asDictionary["publishedVersion"] != null) { Guid.TryParse(d.publishedVersion.ToString(), out publishedVersion); diff --git a/src/Umbraco.Core/Persistence/PetaPoco.cs b/src/Umbraco.Core/Persistence/PetaPoco.cs index 3cde02c287..50df3183e1 100644 --- a/src/Umbraco.Core/Persistence/PetaPoco.cs +++ b/src/Umbraco.Core/Persistence/PetaPoco.cs @@ -703,7 +703,8 @@ namespace Umbraco.Core.Persistence static Regex rxColumns = new Regex(@"\A\s*SELECT\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?(?:\s*,\s*(?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|[\w\(\)\.])+(?:\s+(?:ASC|DESC))?)*", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); - static Regex rxDistinct = new Regex(@"\ADISTINCT\s", RegexOptions.IgnoreCase | RegexOptions.Multiline | RegexOptions.Singleline | RegexOptions.Compiled); + static Regex rxGroupBy = new Regex(@"\bGROUP\s+BY\s+((?:\((?>\((?)|\)(?<-depth>)|.?)*(?(depth)(?!))\)|.)*?)(?" clause - m = rxOrderBy.Match(sqlCount); + + // Look for an "ORDER BY " clause + m = rxOrderBy.Match(sqlCount); if (!m.Success) { sqlOrderBy = null; @@ -738,7 +738,15 @@ namespace Umbraco.Core.Persistence sqlCount = sqlCount.Substring(0, g.Index) + sqlCount.Substring(g.Index + g.Length); } - return true; + // Look for an "GROUP BY " (end) + m = rxGroupBy.Match(sqlCount); + if (m.Success != false) + { + g = m.Groups[0]; + sqlCount = sqlCount.Substring(0, g.Index) + sqlCount.Substring(g.Index + g.Length); + } + + return true; } /// diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs index 81c3944e31..ad421f835c 100644 --- a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Models; using Umbraco.Core; using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Models.Rdbms; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Factories; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.UnitOfWork; @@ -49,6 +50,96 @@ namespace Umbraco.Core.Persistence.Repositories #region Query Methods + public IEnumerable GetPagedResultsByQuery(IQuery query, Guid objectTypeId, long pageIndex, int pageSize, out long totalRecords, + string orderBy, Direction orderDirection, IQuery filter = null) + { + bool isContent = objectTypeId == new Guid(Constants.ObjectTypes.Document); + bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media); + + var sqlClause = GetBaseWhere(GetBase, isContent, isMedia, null, objectTypeId); + + var translator = new SqlTranslator(sqlClause, query); + var entitySql = translator.Translate(); + + var factory = new UmbracoEntityFactory(); + + //use dynamic so that we can get ALL properties from the SQL so we can chuck that data into our AdditionalData + var pagedSql = entitySql.Append(GetGroupBy(isContent, isMedia, false)).OrderBy("umbracoNode.id"); + + if (isMedia) + { + //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! + var pagedResult = _work.Database.Page(pageIndex + 1, pageSize, pagedSql); + totalRecords = pagedResult.TotalItems; + + var ids = pagedResult.Items.Select(x => (int) x.id).InGroupsOf(2000); + var entities = pagedResult.Items.Select(factory.BuildEntityFromDynamic).Cast().ToList(); + + //Now we need to merge in the property data since we need paging and we can't do this the way that the big media query was working before + foreach (var idGroup in ids) + { + var propSql = GetPropertySql(Constants.ObjectTypes.Media) + .Where("contentNodeId IN (@ids)", new {ids = idGroup}) + .OrderBy("contentNodeId"); + + //This does NOT fetch all data into memory in a list, this will read + // over the records as a data reader, this is much better for performance and memory, + // but it means that during the reading of this data set, nothing else can be read + // from SQL server otherwise we'll get an exception. + var allPropertyData = _work.Database.Query(propSql); + + //keep track of the current property data item being enumerated + var propertyDataSetEnumerator = allPropertyData.GetEnumerator(); + var hasCurrent = false; // initially there is no enumerator.Current + + try + { + //This must be sorted by node id (which is done by SQL) because this is how we are sorting the query to lookup property types above, + // which allows us to more efficiently iterate over the large data set of property values. + foreach (var entity in entities) + { + // assemble the dtos for this def + // use the available enumerator.Current if any else move to next + while (hasCurrent || propertyDataSetEnumerator.MoveNext()) + { + if (propertyDataSetEnumerator.Current.contentNodeId == entity.Id) + { + hasCurrent = false; // enumerator.Current is not available + + //the property data goes into the additional data + entity.AdditionalData[propertyDataSetEnumerator.Current.propertyTypeAlias] = new UmbracoEntity.EntityProperty + { + PropertyEditorAlias = propertyDataSetEnumerator.Current.propertyEditorAlias, + Value = StringExtensions.IsNullOrWhiteSpace(propertyDataSetEnumerator.Current.dataNtext) + ? propertyDataSetEnumerator.Current.dataNvarchar + : StringExtensions.ConvertToJsonIfPossible(propertyDataSetEnumerator.Current.dataNtext) + }; + } + else + { + hasCurrent = true; // enumerator.Current is available for another def + break; // no more propertyDataDto for this def + } + } + } + } + finally + { + propertyDataSetEnumerator.Dispose(); + } + } + + totalRecords = pagedResult.TotalItems; + return entities; + } + else + { + var pagedResult = _work.Database.Page(pageIndex + 1, pageSize, pagedSql); + totalRecords = pagedResult.TotalItems; + return pagedResult.Items.Select(factory.BuildEntityFromDynamic).Cast().ToList(); + } + } + public IUmbracoEntity GetByKey(Guid key) { var sql = GetBaseWhere(GetBase, false, false, key); @@ -71,8 +162,7 @@ namespace Umbraco.Core.Persistence.Repositories if (isMedia) { - //for now treat media differently - //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields + //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, sql); @@ -115,8 +205,7 @@ namespace Umbraco.Core.Persistence.Repositories if (isMedia) { - //for now treat media differently - //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields + //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, sql); @@ -171,8 +260,7 @@ namespace Umbraco.Core.Persistence.Repositories if (isMedia) { - //for now treat media differently - //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields + //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, sql); foreach (var entity in entities) @@ -231,8 +319,7 @@ namespace Umbraco.Core.Persistence.Repositories } }); - //treat media differently for now - //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields + //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, mediaSql); return entities; @@ -278,11 +365,9 @@ namespace Umbraco.Core.Persistence.Repositories return GetFullSqlForMedia(entitySql.Append(GetGroupBy(isContent, true, false)), filter); } - private Sql GetFullSqlForMedia(Sql entitySql, Action filter = null) + private Sql GetPropertySql(string nodeObjectType) { - //this will add any dataNvarchar property to the output which can be added to the additional properties - - var joinSql = new Sql() + var sql = new Sql() .Select("contentNodeId, versionId, dataNvarchar, dataNtext, propertyEditorAlias, alias as propertyTypeAlias") .From() .InnerJoin() @@ -291,7 +376,16 @@ namespace Umbraco.Core.Persistence.Repositories .On(dto => dto.Id, dto => dto.PropertyTypeId) .InnerJoin() .On(dto => dto.DataTypeId, dto => dto.DataTypeId) - .Where("umbracoNode.nodeObjectType = @nodeObjectType", new {nodeObjectType = Constants.ObjectTypes.Media}); + .Where("umbracoNode.nodeObjectType = @nodeObjectType", new { nodeObjectType = nodeObjectType }); + + return sql; + } + + private Sql GetFullSqlForMedia(Sql entitySql, Action filter = null) + { + //this will add any dataNvarchar property to the output which can be added to the additional properties + + var joinSql = GetPropertySql(Constants.ObjectTypes.Media); if (filter != null) { @@ -315,25 +409,30 @@ namespace Umbraco.Core.Persistence.Repositories protected virtual Sql GetBase(bool isContent, bool isMedia, Action customFilter) { var columns = new List - { - "umbracoNode.id", - "umbracoNode.trashed", - "umbracoNode.parentID", - "umbracoNode.nodeUser", - "umbracoNode.level", - "umbracoNode.path", - "umbracoNode.sortOrder", - "umbracoNode.uniqueID", - "umbracoNode.text", - "umbracoNode.nodeObjectType", - "umbracoNode.createDate", - "COUNT(parent.parentID) as children" - }; + { + "umbracoNode.id", + "umbracoNode.trashed", + "umbracoNode.parentID", + "umbracoNode.nodeUser", + "umbracoNode.level", + "umbracoNode.path", + "umbracoNode.sortOrder", + "umbracoNode.uniqueID", + "umbracoNode.text", + "umbracoNode.nodeObjectType", + "umbracoNode.createDate", + "COUNT(parent.parentID) as children" + }; if (isContent || isMedia) { - columns.Add("published.versionId as publishedVersion"); - columns.Add("latest.versionId as newestVersion"); + if (isContent) + { + //only content has this info + columns.Add("published.versionId as publishedVersion"); + columns.Add("document.versionId as newestVersion"); + } + columns.Add("contenttype.alias"); columns.Add("contenttype.icon"); columns.Add("contenttype.thumbnail"); @@ -348,14 +447,18 @@ namespace Umbraco.Core.Persistence.Repositories if (isContent || isMedia) { - entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id") - .LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType") - .LeftJoin( - "(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1 GROUP BY nodeId, versionId) as published") - .On("umbracoNode.id = published.nodeId") - .LeftJoin( - "(SELECT nodeId, versionId FROM cmsDocument WHERE newest = 1 GROUP BY nodeId, versionId) as latest") - .On("umbracoNode.id = latest.nodeId"); + entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id"); + + if (isContent) + { + //only content has this info + entitySql + .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id") + .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published") + .On("umbracoNode.id = published.nodeId"); + } + + entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType"); } entitySql.LeftJoin("umbracoNode parent").On("parent.parentID = umbracoNode.id"); @@ -372,22 +475,42 @@ namespace Umbraco.Core.Persistence.Repositories { var sql = baseQuery(isContent, isMedia, filter) .Where("umbracoNode.nodeObjectType = @NodeObjectType", new { NodeObjectType = nodeObjectType }); + + if (isContent) + { + sql.Where("document.newest = 1"); + } + return sql; } protected virtual Sql GetBaseWhere(Func, Sql> baseQuery, bool isContent, bool isMedia, int id) { var sql = baseQuery(isContent, isMedia, null) - .Where("umbracoNode.id = @Id", new { Id = id }) - .Append(GetGroupBy(isContent, isMedia)); + .Where("umbracoNode.id = @Id", new { Id = id }); + + if (isContent) + { + sql.Where("document.newest = 1"); + } + + sql.Append(GetGroupBy(isContent, isMedia)); + return sql; } protected virtual Sql GetBaseWhere(Func, Sql> baseQuery, bool isContent, bool isMedia, Guid key) { var sql = baseQuery(isContent, isMedia, null) - .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key }) - .Append(GetGroupBy(isContent, isMedia)); + .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key }); + + if (isContent) + { + sql.Where("document.newest = 1"); + } + + sql.Append(GetGroupBy(isContent, isMedia)); + return sql; } @@ -396,6 +519,12 @@ namespace Umbraco.Core.Persistence.Repositories var sql = baseQuery(isContent, isMedia, null) .Where("umbracoNode.id = @Id AND umbracoNode.nodeObjectType = @NodeObjectType", new {Id = id, NodeObjectType = nodeObjectType}); + + if (isContent) + { + sql.Where("document.newest = 1"); + } + return sql; } @@ -404,30 +533,39 @@ namespace Umbraco.Core.Persistence.Repositories var sql = baseQuery(isContent, isMedia, null) .Where("umbracoNode.uniqueID = @UniqueID AND umbracoNode.nodeObjectType = @NodeObjectType", new { UniqueID = key, NodeObjectType = nodeObjectType }); + + if (isContent) + { + sql.Where("document.newest = 1"); + } + return sql; } protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true) { var columns = new List - { - "umbracoNode.id", - "umbracoNode.trashed", - "umbracoNode.parentID", - "umbracoNode.nodeUser", - "umbracoNode.level", - "umbracoNode.path", - "umbracoNode.sortOrder", - "umbracoNode.uniqueID", - "umbracoNode.text", - "umbracoNode.nodeObjectType", - "umbracoNode.createDate" - }; + { + "umbracoNode.id", + "umbracoNode.trashed", + "umbracoNode.parentID", + "umbracoNode.nodeUser", + "umbracoNode.level", + "umbracoNode.path", + "umbracoNode.sortOrder", + "umbracoNode.uniqueID", + "umbracoNode.text", + "umbracoNode.nodeObjectType", + "umbracoNode.createDate" + }; if (isContent || isMedia) { - columns.Add("published.versionId"); - columns.Add("latest.versionId"); + if (isContent) + { + columns.Add("published.versionId"); + columns.Add("document.versionId"); + } columns.Add("contenttype.alias"); columns.Add("contenttype.icon"); columns.Add("contenttype.thumbnail"); diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IEntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IEntityRepository.cs index bd2c863fef..54e86e64d4 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IEntityRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IEntityRepository.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using Umbraco.Core.Models.EntityBase; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; namespace Umbraco.Core.Persistence.Repositories @@ -16,6 +17,21 @@ namespace Umbraco.Core.Persistence.Repositories IEnumerable GetByQuery(IQuery query); IEnumerable GetByQuery(IQuery query, Guid objectTypeId); + /// + /// Gets paged results + /// + /// Query to excute + /// + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// + /// An Enumerable list of objects + IEnumerable GetPagedResultsByQuery(IQuery query, Guid objectTypeId, long pageIndex, int pageSize, out long totalRecords, + string orderBy, Direction orderDirection, IQuery filter = null); + /// /// Returns true if the entity exists /// diff --git a/src/Umbraco.Core/Services/EntityService.cs b/src/Umbraco.Core/Services/EntityService.cs index 3ff28aee0a..c18e62cd28 100644 --- a/src/Umbraco.Core/Services/EntityService.cs +++ b/src/Umbraco.Core/Services/EntityService.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Models; using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.UnitOfWork; @@ -374,6 +375,27 @@ namespace Umbraco.Core.Services } } + public IEnumerable GetPagedChildren(int parentId, UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords, + string orderBy = "SortOrder", Direction orderDirection = Direction.Ascending, string filter = "") + { + var objectTypeId = umbracoObjectType.GetGuid(); + using (var uow = UowProvider.GetUnitOfWork()) + { + var repository = RepositoryFactory.CreateEntityRepository(uow); + var query = Query.Builder.Where(x => x.ParentId == parentId); + + IQuery filterQuery = null; + if (filter.IsNullOrWhiteSpace() == false) + { + filterQuery = Query.Builder.Where(x => x.Name.Contains(filter)); + } + + var contents = repository.GetPagedResultsByQuery(query, objectTypeId, pageIndex, pageSize, out totalRecords, orderBy, orderDirection, filterQuery); + uow.Commit(); + return contents; + } + } + /// /// Gets a collection of descendents by the parents Id /// diff --git a/src/Umbraco.Core/Services/IEntityService.cs b/src/Umbraco.Core/Services/IEntityService.cs index 66e91a9786..d9052f4bb8 100644 --- a/src/Umbraco.Core/Services/IEntityService.cs +++ b/src/Umbraco.Core/Services/IEntityService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using Umbraco.Core.Models; using Umbraco.Core.Models.EntityBase; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; namespace Umbraco.Core.Services { @@ -137,6 +138,9 @@ namespace Umbraco.Core.Services /// An enumerable list of objects IEnumerable GetChildren(int parentId, UmbracoObjectTypes umbracoObjectType); + IEnumerable GetPagedChildren(int parentId, UmbracoObjectTypes umbracoObjectType, long pageIndex, int pageSize, out long totalRecords, + string orderBy = "SortOrder", Direction orderDirection = Direction.Ascending, string filter = ""); + /// /// Gets a collection of descendents by the parents Id /// diff --git a/src/Umbraco.Tests/Services/EntityServiceTests.cs b/src/Umbraco.Tests/Services/EntityServiceTests.cs index 361bd4e8a2..eb2eccb94d 100644 --- a/src/Umbraco.Tests/Services/EntityServiceTests.cs +++ b/src/Umbraco.Tests/Services/EntityServiceTests.cs @@ -26,7 +26,57 @@ namespace Umbraco.Tests.Services { base.TearDown(); } - + + [Test] + public void EntityService_Can_Get_Paged_Content_Children() + { + + var contentType = ServiceContext.ContentTypeService.GetContentType("umbTextpage"); + + var root = MockedContent.CreateSimpleContent(contentType); + ServiceContext.ContentService.Save(root); + for (int i = 0; i < 10; i++) + { + var c1 = MockedContent.CreateSimpleContent(contentType, Guid.NewGuid().ToString(), root); + ServiceContext.ContentService.Save(c1); + } + + var service = ServiceContext.EntityService; + + long total; + var entities = service.GetPagedChildren(root.Id, UmbracoObjectTypes.Document, 0, 6, out total).ToArray(); + Assert.That(entities.Length, Is.EqualTo(6)); + Assert.That(total, Is.EqualTo(10)); + entities = service.GetPagedChildren(root.Id, UmbracoObjectTypes.Document, 1, 6, out total).ToArray(); + Assert.That(entities.Length, Is.EqualTo(4)); + Assert.That(total, Is.EqualTo(10)); + } + + [Test] + public void EntityService_Can_Get_Paged_Media_Children() + { + var folderType = ServiceContext.ContentTypeService.GetMediaType(1031); + var imageMediaType = ServiceContext.ContentTypeService.GetMediaType(1032); + + var root = MockedMedia.CreateMediaFolder(folderType, -1); + ServiceContext.MediaService.Save(root); + for (int i = 0; i < 10; i++) + { + var c1 = MockedMedia.CreateMediaImage(imageMediaType, root.Id); + ServiceContext.MediaService.Save(c1); + } + + var service = ServiceContext.EntityService; + + long total; + var entities = service.GetPagedChildren(root.Id, UmbracoObjectTypes.Media, 0, 6, out total).ToArray(); + Assert.That(entities.Length, Is.EqualTo(6)); + Assert.That(total, Is.EqualTo(10)); + entities = service.GetPagedChildren(root.Id, UmbracoObjectTypes.Media, 1, 6, out total).ToArray(); + Assert.That(entities.Length, Is.EqualTo(4)); + Assert.That(total, Is.EqualTo(10)); + } + [Test] public void EntityService_Can_Find_All_Content_By_UmbracoObjectTypes() {