diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs index f3afe99b28..b3a7c31e54 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs @@ -99,7 +99,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement private string VariantNameSqlExpression => SqlContext.VisitDto((ccv, node) => ccv.Name ?? node.Text, "ccv").Sql; - protected virtual Sql GetBaseQuery(QueryType queryType, bool current) + protected Sql GetBaseQuery(QueryType queryType, bool current) { var sql = SqlContext.Sql(); diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index 7371686c7c..0cfa0bb601 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -79,9 +79,15 @@ namespace Umbraco.Core.Services IEnumerable GetByIds(IEnumerable ids); /// - /// Gets documents of a given document type. + /// Gets paged documents of a content content /// - IEnumerable GetByType(int documentTypeId); + /// The page number. + /// The page number. + /// The page size. + /// Total number of documents. + /// Search text filter. + /// Ordering infos. + IEnumerable GetPagedOfType(int contentType, long pageIndex, int pageSize, out long totalRecords, IQuery filter = null, Ordering ordering = null); /// /// Gets documents at a given level. diff --git a/src/Umbraco.Core/Services/Implement/ContentService.cs b/src/Umbraco.Core/Services/Implement/ContentService.cs index 3953e7a640..64c6a23aff 100644 --- a/src/Umbraco.Core/Services/Implement/ContentService.cs +++ b/src/Umbraco.Core/Services/Implement/ContentService.cs @@ -405,28 +405,20 @@ namespace Umbraco.Core.Services.Implement } } - /// - /// Gets a collection of objects by the Id of the - /// - /// Id of the - /// An Enumerable list of objects - public IEnumerable GetByType(int id) + public IEnumerable GetPagedOfType(int contentTypeId, long pageIndex, int pageSize, out long totalRecords, IQuery filter, Ordering ordering = null) { - using (var scope = ScopeProvider.CreateScope(autoComplete: true)) - { - scope.ReadLock(Constants.Locks.ContentTree); - var query = Query().Where(x => x.ContentTypeId == id); - return _documentRepository.Get(query); - } - } + if(pageIndex < 0) throw new ArgumentOutOfRangeException(nameof(pageIndex)); + if (pageSize <= 0) throw new ArgumentOutOfRangeException(nameof(pageSize)); + + if (ordering == null) + ordering = Ordering.By("sortOrder"); - internal IEnumerable GetPublishedContentOfContentType(int id) - { using (var scope = ScopeProvider.CreateScope(autoComplete: true)) { scope.ReadLock(Constants.Locks.ContentTree); - var query = Query().Where(x => x.ContentTypeId == id); - return _documentRepository.Get(query); + return _documentRepository.GetPage( + Query().Where(x => x.ContentTypeId == contentTypeId), + pageIndex, pageSize, out totalRecords, filter, ordering); } } diff --git a/src/Umbraco.Tests/Services/ContentServiceTests.cs b/src/Umbraco.Tests/Services/ContentServiceTests.cs index 5aad7c4d90..5b06037f90 100644 --- a/src/Umbraco.Tests/Services/ContentServiceTests.cs +++ b/src/Umbraco.Tests/Services/ContentServiceTests.cs @@ -1738,7 +1738,7 @@ namespace Umbraco.Tests.Services // Act contentService.DeleteOfType(contentType.Id); var rootContent = contentService.GetRootContent(); - var contents = contentService.GetByType(contentType.Id); + var contents = contentService.GetPagedOfType(contentType.Id, 0, int.MaxValue, out var _); // Assert Assert.That(rootContent.Any(), Is.False); diff --git a/src/Umbraco.Tests/Services/PerformanceTests.cs b/src/Umbraco.Tests/Services/PerformanceTests.cs index 9b0117c266..900a466a1d 100644 --- a/src/Umbraco.Tests/Services/PerformanceTests.cs +++ b/src/Umbraco.Tests/Services/PerformanceTests.cs @@ -108,40 +108,6 @@ namespace Umbraco.Tests.Services } } - [Test] - public void Get_All_Published_Content_Of_Type() - { - var result = PrimeDbWithLotsOfContent(); - var contentSvc = (ContentService)ServiceContext.ContentService; - - var countOfPublished = result.Count(x => x.Published); - var contentTypeId = result.First().ContentTypeId; - - var proflog = GetTestProfilingLogger(); - using (proflog.DebugDuration("Getting published content of type normally")) - { - //do this 10x! - for (var i = 0; i < 10; i++) - { - - //get all content items that are published of this type - var published = contentSvc.GetByType(contentTypeId).Where(content => content.Published); - Assert.AreEqual(countOfPublished, published.Count(x => x.ContentTypeId == contentTypeId)); - } - } - - using (proflog.DebugDuration("Getting published content of type optimized")) - { - - //do this 10x! - for (var i = 0; i < 10; i++) - { - //get all content items that are published of this type - var published = contentSvc.GetPublishedContentOfContentType(contentTypeId); - Assert.AreEqual(countOfPublished, published.Count(x => x.ContentTypeId == contentTypeId)); - } - } - } [Test] public void Truncate_Insert_Vs_Update_Insert() diff --git a/src/Umbraco.Web/Search/ExamineComponent.cs b/src/Umbraco.Web/Search/ExamineComponent.cs index ff7d0c8dc4..530e8ba449 100644 --- a/src/Umbraco.Web/Search/ExamineComponent.cs +++ b/src/Umbraco.Web/Search/ExamineComponent.cs @@ -102,13 +102,10 @@ namespace Umbraco.Web.Search // bind to distributed cache events - this ensures that this logic occurs on ALL servers // that are taking part in a load balanced environment. ContentCacheRefresher.CacheUpdated += ContentCacheRefresherUpdated; + ContentTypeCacheRefresher.CacheUpdated += ContentTypeCacheRefresherUpdated; ; MediaCacheRefresher.CacheUpdated += MediaCacheRefresherUpdated; MemberCacheRefresher.CacheUpdated += MemberCacheRefresherUpdated; - // fixme - content type? - // events handling removed in ef013f9d3b945d0a48a306ff1afbd49c10c3fff8 - // because, could not make sense of it? - EnsureUnlocked(profilingLogger.Logger, examineManager); RebuildIndexesOnStartup(profilingLogger.Logger); @@ -317,6 +314,88 @@ namespace Umbraco.Web.Search } } + private void ContentTypeCacheRefresherUpdated(ContentTypeCacheRefresher sender, CacheRefresherEventArgs args) + { + + //before content type changes just caused full blown re-indexing: + // https://github.com/umbraco/Umbraco-CMS/commit/ef013f9d3b945d0a48a306ff1afbd49c10c3fff8 + + + if (Suspendable.ExamineEvents.CanIndex == false) + return; + + if (args.MessageType != MessageType.RefreshByPayload) + throw new NotSupportedException(); + + var contentService = _services.ContentService; + + var removedIds = new List(); + var refreshedIds = new List(); + + //TODO: What do we do about these? + var otherIds = new List(); + + foreach (var payload in (ContentTypeCacheRefresher.JsonPayload[])args.MessageObject) + { + if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.Remove)) + removedIds.Add(payload.Id); + else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshMain)) + refreshedIds.Add(payload.Id); + else if (payload.ChangeTypes.HasType(ContentTypeChangeTypes.RefreshOther)) + otherIds.Add(payload.Id); + } + + const int pageSize = 500; + + //Re-index all content of these types + foreach(var id in refreshedIds) + { + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + var contentToRefresh = _services.ContentService.GetPagedOfType(id, page++, pageSize, out total); + foreach(var c in contentToRefresh) + { + //TODO: We might have to order by Path ascending or something since we're going to need to check + // contentService.IsPathPublished(content) but we don't want to make that check for every content item + IContent published = null; + if (c.Published && contentService.IsPathPublished(c)) + published = c; + + ReIndexForContent(c, published); + } + } + } + + //Delete all content of this content type that is in any content indexer by looking up matched examine docs + foreach(var id in removedIds) + { + foreach(var index in _examineManager.IndexProviders.Values.OfType()) + { + var searcher = index.GetSearcher(); + + var page = 0; + var total = long.MaxValue; + while (page * pageSize < total) + { + //paging with examine, see https://shazwazza.com/post/paging-with-examine/ + var results = searcher.Search( + searcher.CreateCriteria().Field("nodeType", id).Compile(), + maxResults: pageSize * (page + 1)); + total = results.TotalItemCount; + var paged = results.Skip(page * pageSize); + + foreach(var item in paged) + if (int.TryParse(item.Id, out var contentId)) + DeleteIndexForEntity(contentId, false); + } + } + + + } + } + private void ContentCacheRefresherUpdated(ContentCacheRefresher sender, CacheRefresherEventArgs args) { if (Suspendable.ExamineEvents.CanIndex == false) @@ -340,6 +419,8 @@ namespace Umbraco.Web.Search // ExamineEvents does not support RefreshAll // just ignore that payload // so what?! + + //fixme: Rebuild the index at this point? } else // RefreshNode or RefreshBranch (maybe trashed) { @@ -355,7 +436,7 @@ namespace Umbraco.Web.Search } IContent published = null; - if (content.Published && ((ContentService)contentService).IsPathPublished(content)) + if (content.Published && contentService.IsPathPublished(content)) published = content; // just that content