yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V14/feature/management tree count by take zero (#15308)

Browse Source 
* Allow Tree endpoints that query entities to return count without entity data

* Apply count by take 0 in FileSystem Endpoints

* Apply count by take 0 in Dictionary Endpoints

* Apply count by take 0 in RootRelationType Endpoints

* Revert PaginationService takeZero flag as it only guards against things that already blow up

* Mark PagedResult as Obsolete as we want to step away from classic pagination system to skip/take

* Pushed management api RelationType pagination and async preperation down to the service layer

* Scope fix and allocation optimizations

* Pushed management api dictionary pagination and down to the service layer

Also did some nice allocation optimizations

* PR feedback + related strange count behaviour

* Moved count by pagesize logic from EntryController to service

* A tiny bit of formatting and comments

* Fix bad count filter logic

* Added integration tests for creating datatypes in a folder

* Added tests for count testing on TreeControllers

- ChildrenDataType
- RootDataType
- ChildrenDictionary
- RootDictionary
- ChildrenDocument
- RootDocument
- RootBluePrint
- RootDocumentType
- ChildrenDocumentType

* Revert "Added tests for count testing on TreeControllers", should be on services

This reverts commit ee2501fe620a584bba13ecd4fdce8142133fd82b.
This reverts commit 808d5b276fad267a645e474ead3278d4bb79d0c4.

---------

Co-authored-by: Sven Geusens <sge@umbraco.dk>
Co-authored-by: kjac <kja@umbraco.dk>
Co-authored-by: Andreas Zerbst <andr317c@live.dk>
This commit is contained in:
Sven Geusens
2024-01-02 13:53:24 +01:00
committed by GitHub
parent f3cb8fe117
commit c937d0f2ed
22 changed files with 184 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/Umbraco.Cms.Api.Management/Controllers/Dictionary/Tree/ChildrenDictionaryTreeController.cs
View File

@@ -27,15 +27,8 @@ public class ChildrenDictionaryTreeController : DictionaryTreeControllerBase
return BadRequest(error);
}
IDictionaryItem[] dictionaryItems = PaginatedDictionaryItems(
pageNumber,
pageSize,
await DictionaryItemService.GetChildrenAsync(parentId),
out var totalItems);
PagedModel<IDictionaryItem> paginatedItems = await DictionaryItemService.GetPagedAsync(parentId, skip, take);
EntityTreeItemResponseModel[] viewModels = await MapTreeItemViewModels(parentId, dictionaryItems);
PagedViewModel<EntityTreeItemResponseModel> result = PagedViewModel(viewModels, totalItems);
return await Task.FromResult(Ok(result));
return Ok(PagedViewModel(await MapTreeItemViewModels(parentId, paginatedItems.Items), paginatedItems.Total));
}
}

25
src/Umbraco.Cms.Api.Management/Controllers/Dictionary/Tree/DictionaryTreeControllerBase.cs
View File

@@ -27,11 +27,11 @@ public class DictionaryTreeControllerBase : EntityTreeControllerBase<EntityTreeI
protected IDictionaryItemService DictionaryItemService { get; }
protected async Task<EntityTreeItemResponseModel[]> MapTreeItemViewModels(Guid? parentKey, IDictionaryItem[] dictionaryItems)
protected async Task<IEnumerable<EntityTreeItemResponseModel>> MapTreeItemViewModels(Guid? parentKey, IEnumerable<IDictionaryItem> dictionaryItems)
{
async Task<EntityTreeItemResponseModel> CreateEntityTreeItemViewModelAsync(IDictionaryItem dictionaryItem)
{
var hasChildren = (await DictionaryItemService.GetChildrenAsync(dictionaryItem.Key)).Any();
var hasChildren = await DictionaryItemService.CountChildrenAsync(dictionaryItem.Key) > 0;
return new EntityTreeItemResponseModel
{
Name = dictionaryItem.ItemKey,
@@ -43,25 +43,6 @@ public class DictionaryTreeControllerBase : EntityTreeControllerBase<EntityTreeI
};
}
var items = new List<EntityTreeItemResponseModel>(dictionaryItems.Length);
foreach (IDictionaryItem dictionaryItem in dictionaryItems)
{
items.Add(await CreateEntityTreeItemViewModelAsync(dictionaryItem));
}
return items.ToArray();
}
// language service does not (yet) allow pagination of dictionary items, we have to do it in memory for now
protected IDictionaryItem[] PaginatedDictionaryItems(long pageNumber, int pageSize, IEnumerable<IDictionaryItem> allDictionaryItems, out long totalItems)
{
IDictionaryItem[] allDictionaryItemsAsArray = allDictionaryItems.ToArray();
totalItems = allDictionaryItemsAsArray.Length;
return allDictionaryItemsAsArray
.OrderBy(item => item.ItemKey)
.Skip((int)pageNumber * pageSize)
.Take(pageSize)
.ToArray();
return await Task.WhenAll(dictionaryItems.Select(CreateEntityTreeItemViewModelAsync));
}
}

11
src/Umbraco.Cms.Api.Management/Controllers/Dictionary/Tree/RootDictionaryTreeController.cs
View File

@@ -27,15 +27,8 @@ public class RootDictionaryTreeController : DictionaryTreeControllerBase
return BadRequest(error);
}
IDictionaryItem[] dictionaryItems = PaginatedDictionaryItems(
pageNumber,
pageSize,
await DictionaryItemService.GetAtRootAsync(),
out var totalItems);
PagedModel<IDictionaryItem> paginatedItems = await DictionaryItemService.GetPagedAsync(null, skip, take);
EntityTreeItemResponseModel[] viewModels = await MapTreeItemViewModels(null, dictionaryItems);
PagedViewModel<EntityTreeItemResponseModel> result = PagedViewModel(viewModels, totalItems);
return await Task.FromResult(Ok(result));
return Ok(PagedViewModel(await MapTreeItemViewModels(null, paginatedItems.Items), paginatedItems.Total));
}
}

4
src/Umbraco.Cms.Api.Management/Controllers/RelationType/Tree/RelationTypeTreeControllerBase.cs
View File

@@ -25,7 +25,7 @@ public class RelationTypeTreeControllerBase : EntityTreeControllerBase<EntityTre
protected override UmbracoObjectTypes ItemObjectType => UmbracoObjectTypes.RelationType;
protected EntityTreeItemResponseModel[] MapTreeItemViewModels(Guid? parentKey, IRelationType[] relationTypes)
protected IEnumerable<EntityTreeItemResponseModel> MapTreeItemViewModels(Guid? parentKey, IEnumerable<IRelationType> relationTypes)
=> relationTypes.Select(relationType => new EntityTreeItemResponseModel
{
Name = relationType.Name!,
@@ -34,5 +34,5 @@ public class RelationTypeTreeControllerBase : EntityTreeControllerBase<EntityTre
HasChildren = false,
IsContainer = false,
ParentId = parentKey
}).ToArray();
});
}

22
src/Umbraco.Cms.Api.Management/Controllers/RelationType/Tree/RootRelationTypeTreeController.cs
View File

@@ -23,24 +23,12 @@ public class RootRelationTypeTreeController : RelationTypeTreeControllerBase
[ProducesResponseType(typeof(PagedViewModel<EntityTreeItemResponseModel>), StatusCodes.Status200OK)]
public async Task<ActionResult<PagedViewModel<EntityTreeItemResponseModel>>> Root(int skip = 0, int take = 100)
{
if (PaginationService.ConvertSkipTakeToPaging(skip, take, out var pageNumber, out var pageSize, out ProblemDetails? error) == false)
{
return BadRequest(error);
}
PagedModel<IRelationType> pagedRelationTypes = await _relationService.GetPagedRelationTypesAsync(skip, take);
// pagination is not supported (yet) by relation service, so we do it in memory for now
// - chances are we won't have many relation types, so it won't be an actual issue
IRelationType[] allRelationTypes = _relationService.GetAllRelationTypes().ToArray();
PagedViewModel<EntityTreeItemResponseModel> pagedResult = PagedViewModel(
MapTreeItemViewModels(null, pagedRelationTypes.Items),
pagedRelationTypes.Total);
EntityTreeItemResponseModel[] viewModels = MapTreeItemViewModels(
null,
allRelationTypes
.OrderBy(relationType => relationType.Name)
.Skip((int)(pageNumber * pageSize))
.Take(pageSize)
.ToArray());
PagedViewModel<EntityTreeItemResponseModel> result = PagedViewModel(viewModels, allRelationTypes.Length);
return await Task.FromResult(Ok(result));
return Ok(pagedResult);
}
}

15
src/Umbraco.Cms.Api.Management/Controllers/Tree/FolderTreeControllerBase.cs
View File

@@ -66,6 +66,15 @@ public abstract class FolderTreeControllerBase<TItem> : EntityTreeControllerBase
{
totalItems = 0;
if (pageSize == 0)
{
totalItems = _foldersOnly
? EntityService.CountChildren(parentId, FolderObjectType)
: EntityService.CountChildren(parentId, FolderObjectType)
+ EntityService.CountChildren(parentId, ItemObjectType);
return Array.Empty<IEntitySlim>();
}
// EntityService is not able to paginate children of multiple item types, so we will only paginate the
// item type entities and always return all folders as part of the the first result page
IEntitySlim[] folderEntities = pageNumber == 0
@@ -82,6 +91,12 @@ public abstract class FolderTreeControllerBase<TItem> : EntityTreeControllerBase
ordering: ItemOrdering)
.ToArray();
// the GetChildren for folders does not return an amount and does not get executed when beyond the first page
// but the items still count towards the total, so add these to either 0 when only folders, or the out param from paged
totalItems += pageNumber == 0
? folderEntities.Length
: EntityService.CountChildren(parentId, FolderObjectType);
return folderEntities.Union(itemEntities).ToArray();
}
}

8
src/Umbraco.Cms.Api.Management/Services/Paging/PaginationService.cs
View File

@@ -12,12 +12,12 @@ internal static class PaginationService
{
internal static bool ConvertSkipTakeToPaging(int skip, int take, out long pageNumber, out int pageSize, out ProblemDetails? error)
{
if (take <= 0)
if (take < 0)
{
throw new ArgumentException("Must be greater than zero", nameof(take));
throw new ArgumentException("Must be equal to or greater than zero", nameof(take));
}
if (skip % take != 0)
if (take != 0 && skip % take != 0)
{
pageSize = 0;
pageNumber = 0;
@@ -32,7 +32,7 @@ internal static class PaginationService
}
pageSize = take;
pageNumber = skip / take;
pageNumber = take == 0 ? 0 : skip / take;
error = null;
return true;
}

1
src/Umbraco.Core/Models/PagedResult.cs
View File

@@ -5,6 +5,7 @@ namespace Umbraco.Cms.Core.Models;
/// <summary>
/// Represents a paged result for a model collection
/// </summary>
[Obsolete ("Superseded by PagedModel for service layer and below OR PagedViewModel in apis. Expected to be removed when skip/take pattern has been fully implemented v14+")]
[DataContract(Name = "pagedCollection", Namespace = "")]
public abstract class PagedResult
{

1
src/Umbraco.Core/Models/PagedResultOfT.cs
View File

@@ -7,6 +7,7 @@ namespace Umbraco.Cms.Core.Models;
/// </summary>
/// <typeparam name="T"></typeparam>
[DataContract(Name = "pagedCollection", Namespace = "")]
[Obsolete ("Superseded by PagedModel for service layer and below OR PagedViewModel in apis. Expected to be removed when skip/take pattern has been fully implemented v14+")]
public class PagedResult<T> : PagedResult
{
public PagedResult(long totalItems, long pageNumber, long pageSize)

2
src/Umbraco.Core/Persistence/IQueryRepository.cs
View File

@@ -15,5 +15,5 @@ public interface IQueryRepository<TEntity> : IRepository
/// <summary>
/// Counts entities.
/// </summary>
int Count(IQuery<TEntity> query);
int Count(IQuery<TEntity>? query);
}

2
src/Umbraco.Core/Persistence/Repositories/IEntityRepository.cs
View File

@@ -83,4 +83,6 @@ public interface IEntityRepository : IRepository
out long totalRecords,
IQuery<IUmbracoEntity>? filter,
Ordering? ordering);
int CountByQuery(IQuery<IUmbracoEntity> query, Guid objectType, IQuery<IUmbracoEntity>? filter);
}

41
src/Umbraco.Core/Services/DictionaryItemService.cs
View File

@@ -72,10 +72,39 @@ internal sealed class DictionaryItemService : RepositoryService, IDictionaryItem
}
}
/// <summary>
/// Gets the dictionary items in a paged manner.
/// Currently implements the paging in memory on the itenkey property because the underlying repository does not support paging yet
/// </summary>
public async Task<PagedModel<IDictionaryItem>> GetPagedAsync(Guid? parentId, int skip, int take)
{
using ICoreScope coreScope = ScopeProvider.CreateCoreScope(autoComplete: true);
if (take == 0)
{
return parentId is null
? new PagedModel<IDictionaryItem>(await CountRootAsync(), Enumerable.Empty<IDictionaryItem>())
: new PagedModel<IDictionaryItem>(await CountChildrenAsync(parentId.Value), Enumerable.Empty<IDictionaryItem>());
}
IDictionaryItem[] items = (parentId is null
? await GetAtRootAsync()
: await GetChildrenAsync(parentId.Value)).ToArray();
return new PagedModel<IDictionaryItem>(
items.Length,
items.OrderBy(i => i.ItemKey)
.Skip(skip)
.Take(take));
}
/// <inheritdoc />
public async Task<IEnumerable<IDictionaryItem>> GetChildrenAsync(Guid parentId)
=> await GetByQueryAsync(Query<IDictionaryItem>().Where(x => x.ParentId == parentId));
public async Task<int> CountChildrenAsync(Guid parentId)
=> await CountByQueryAsync(Query<IDictionaryItem>().Where(x => x.ParentId == parentId));
/// <inheritdoc />
public async Task<IEnumerable<IDictionaryItem>> GetDescendantsAsync(Guid? parentId)
{
@@ -90,6 +119,9 @@ internal sealed class DictionaryItemService : RepositoryService, IDictionaryItem
public async Task<IEnumerable<IDictionaryItem>> GetAtRootAsync()
=> await GetByQueryAsync(Query<IDictionaryItem>().Where(x => x.ParentId == null));
public async Task<int> CountRootAsync()
=> await CountByQueryAsync(Query<IDictionaryItem>().Where(x => x.ParentId == null));
/// <inheritdoc/>
public async Task<bool> ExistsAsync(string key)
{
@@ -237,6 +269,15 @@ internal sealed class DictionaryItemService : RepositoryService, IDictionaryItem
}
}
private async Task<int> CountByQueryAsync(IQuery<IDictionaryItem> query)
{
using (ScopeProvider.CreateCoreScope(autoComplete: true))
{
var items = _dictionaryRepository.Count(query);
return await Task.FromResult(items);
}
}
private async Task<Attempt<IDictionaryItem, DictionaryItemOperationStatus>> SaveAsync(
IDictionaryItem dictionaryItem,
Func<DictionaryItemOperationStatus> operationValidation,

19
src/Umbraco.Core/Services/EntityService.cs
View File

@@ -537,6 +537,19 @@ public class EntityService : RepositoryService, IEntityService
}
}
public int CountChildren(
int id,
UmbracoObjectTypes objectType,
IQuery<IUmbracoEntity>? filter = null)
{
using (ScopeProvider.CreateCoreScope(autoComplete: true))
{
IQuery<IUmbracoEntity> query = Query<IUmbracoEntity>().Where(x => x.ParentId == id && x.Trashed == false);
return _entityRepository.CountByQuery(query, objectType.GetGuid(), filter);
}
}
// gets the object type, throws if not supported
private UmbracoObjectTypes GetObjectType(Type? type)
{
@@ -562,6 +575,12 @@ public class EntityService : RepositoryService, IEntityService
{
IQuery<IUmbracoEntity> query = Query<IUmbracoEntity>().Where(x => x.ParentId == id && x.Trashed == trashed);
if (pageSize == 0)
{
totalRecords = _entityRepository.CountByQuery(query, objectType.GetGuid(), filter);
return Enumerable.Empty<IEntitySlim>();
}
return _entityRepository.GetPagedResultsByQuery(query, objectType.GetGuid(), pageIndex, pageSize, out totalRecords, filter, ordering);
}
}

10
src/Umbraco.Core/Services/IDictionaryItemService.cs
View File

@@ -1,4 +1,5 @@
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Persistence.Querying;
using Umbraco.Cms.Core.Services.OperationStatus;
namespace Umbraco.Cms.Core.Services;
@@ -98,4 +99,13 @@ public interface IDictionaryItemService
/// <param name="parentId">Id of the new <see cref="IDictionaryItem" /> parent, null if the item should be moved to the root</param>
/// <param name="userKey">Key of the user moving the dictionary item</param>
Task<Attempt<IDictionaryItem, DictionaryItemOperationStatus>> MoveAsync(IDictionaryItem dictionaryItem, Guid? parentId, Guid userKey);
Task<int> CountChildrenAsync(Guid parentId);
Task<int> CountRootAsync();
/// <summary>
/// Gets the dictionary items in a paged manner.
/// Currently implements the paging in memory on the itenkey property because the underlying repository does not support paging yet
/// </summary>
Task<PagedModel<IDictionaryItem>> GetPagedAsync(Guid? parentId, int skip, int take);
}

5
src/Umbraco.Core/Services/IEntityService.cs
View File

@@ -308,4 +308,9 @@ public interface IEntityService
/// <returns>The identifier.</returns>
/// <remarks>When a new content or a media is saved with the key, it will have the reserved identifier.</remarks>
int ReserveId(Guid key);
/// <summary>
/// Counts the children of an entity
/// </summary>
int CountChildren(int id, UmbracoObjectTypes objectType, IQuery<IUmbracoEntity>? filter = null);
}

11
src/Umbraco.Core/Services/IRelationService.cs
View File

@@ -171,7 +171,7 @@ public interface IRelationService : IService
/// <param name="totalRecords"></param>
/// <param name="ordering"></param>
/// <returns></returns>
IEnumerable<IRelation> GetPagedByRelationTypeId(int relationTypeId, long pageIndex, int pageSize, out long totalRecords, Ordering? ordering = null);/// <summary>
IEnumerable<IRelation> GetPagedByRelationTypeId(int relationTypeId, long pageIndex, int pageSize, out long totalRecords, Ordering? ordering = null);
/// <summary>
/// Gets a paged result of <see cref="IRelation" />
@@ -397,4 +397,13 @@ public interface IRelationService : IService
IEnumerable<UmbracoObjectTypes> GetAllowedObjectTypes();
Task<PagedModel<IRelation>> GetPagedByChildKeyAsync(Guid childKey, int skip, int take, string? relationTypeAlias);
int CountRelationTypes();
/// <summary>
/// Gets the Relation types in a paged manner.
/// Currently implements the paging in memory on the name attribute because the underlying repository does not support paging yet
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
Task<PagedModel<IRelationType>> GetPagedRelationTypesAsync(int skip, int take, params int[] ids);
}

27
src/Umbraco.Core/Services/RelationService.cs
View File

@@ -122,6 +122,33 @@ public class RelationService : RepositoryService, IRelationService
}
}
/// <summary>
/// Gets the Relation types in a paged manner.
/// Currently implements the paging in memory on the name property because the underlying repository does not support paging yet
/// </summary>
public async Task<PagedModel<IRelationType>> GetPagedRelationTypesAsync(int skip, int take, params int[] ids)
{
using ICoreScope scope = ScopeProvider.CreateCoreScope(autoComplete: true);
if (take == 0)
{
return new PagedModel<IRelationType>(CountRelationTypes(), Enumerable.Empty<IRelationType>());
}
IRelationType[] items = await Task.FromResult(_relationTypeRepository.GetMany(ids).ToArray());
return new PagedModel<IRelationType>(
items.Length,
items.OrderBy(relationType => relationType.Name)
.Skip(skip)
.Take(take));
}
public int CountRelationTypes()
{
return _relationTypeRepository.Count(null);
}
/// <inheritdoc />
public IEnumerable<IRelation> GetByParentId(int id) => GetByParentId(id, null);

24
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepository.cs
View File

@@ -30,6 +30,30 @@ internal class EntityRepository : RepositoryBase, IEntityRepositoryExtended
#region Repository
public int CountByQuery(IQuery<IUmbracoEntity> query, Guid objectType, IQuery<IUmbracoEntity>? filter)
{
Sql<ISqlContext> sql = Sql();
sql.SelectCount();
sql
.From<NodeDto>();
sql.WhereIn<NodeDto>(x => x.NodeObjectType, new[] { objectType } );
foreach (Tuple<string, object[]> queryClause in query.GetWhereClauses())
{
sql.Where(queryClause.Item1, queryClause.Item2);
}
if (filter is not null)
{
foreach (Tuple<string, object[]> filterClause in filter.GetWhereClauses())
{
sql.Where(filterClause.Item1, filterClause.Item2);
}
}
return Database.ExecuteScalar<int>(sql);
}
public IEnumerable<IEntitySlim> GetPagedResultsByQuery(IQuery<IUmbracoEntity> query, Guid objectType,
long pageIndex, int pageSize, out long totalRecords,
IQuery<IUmbracoEntity>? filter, Ordering? ordering) =>

9
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepositoryBase.cs
View File

@@ -181,7 +181,7 @@ public abstract class EntityRepositoryBase<TId, TEntity> : RepositoryBase, IRead
/// <summary>
/// Returns an integer with the count of entities found with the passed in query
/// </summary>
public int Count(IQuery<TEntity> query)
public int Count(IQuery<TEntity>? query)
=> PerformCount(query);
/// <summary>
@@ -221,9 +221,14 @@ public abstract class EntityRepositoryBase<TId, TEntity> : RepositoryBase, IRead
return count == 1;
}
protected virtual int PerformCount(IQuery<TEntity> query)
protected virtual int PerformCount(IQuery<TEntity>? query)
{
Sql<ISqlContext> sqlClause = GetBaseQuery(true);
if (query is null)
{
return Database.ExecuteScalar<int>(sqlClause);
}
var translator = new SqlTranslator<TEntity>(sqlClause, query);
Sql<ISqlContext> sql = translator.Translate();

2
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/ExternalLoginRepository.cs
View File

@@ -46,7 +46,7 @@ internal class ExternalLoginRepository : EntityRepositoryBase<int, IIdentityUser
/// </summary>
/// <param name="query"></param>
/// <returns></returns>
public int Count(IQuery<IIdentityUserToken> query)
public int Count(IQuery<IIdentityUserToken>? query)
{
Sql<ISqlContext> sql = Sql().SelectCount().From<ExternalLoginDto>();
return Database.ExecuteScalar<int>(sql);

2
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/RedirectUrlRepository.cs
View File

@@ -158,7 +158,7 @@ internal class RedirectUrlRepository : EntityRepositoryBase<Guid, IRedirectUrl>,
return rules;
}
protected override int PerformCount(IQuery<IRedirectUrl> query) =>
protected override int PerformCount(IQuery<IRedirectUrl>? query) =>
throw new NotSupportedException("This repository does not support this method.");
protected override bool PerformExists(Guid id) => PerformGet(id) != null;

2
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/ServerRegistrationRepository.cs
View File

@@ -45,7 +45,7 @@ internal class ServerRegistrationRepository : EntityRepositoryBase<int, IServerR
// (cleanup in v8)
new FullDataSetRepositoryCachePolicy<IServerRegistration, int>(AppCaches.RuntimeCache, ScopeAccessor, GetEntityId, /*expires:*/ false);
protected override int PerformCount(IQuery<IServerRegistration> query) =>
protected override int PerformCount(IQuery<IServerRegistration>? query) =>
throw new NotSupportedException("This repository does not support this method.");
protected override bool PerformExists(int id) =>