From 4c0f95a93a6a8911fa892aba2c8773e4f2c23ed9 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 30 Sep 2014 15:13:10 +1000 Subject: [PATCH] Adds GetPagedDescendants to media/content services. Fixes PDF reader disposal. Updates the umb examine indexers to use the services layer, now reindexing is far more efficient. Fixes the examine mgmt dashboard with $timeout. Updates GetPagedResultsByQuery to be more readable. Fixes GetPagedResultsByQuery to use SQL parameters. --- .../Repositories/ContentRepository.cs | 25 ++- .../Interfaces/IContentRepository.cs | 2 +- .../Repositories/MediaRepository.cs | 15 +- .../Repositories/MemberRepository.cs | 17 +- .../Repositories/VersionableRepositoryBase.cs | 5 +- src/Umbraco.Core/Services/ContentService.cs | 39 +++- src/Umbraco.Core/Services/IContentService.cs | 16 +- src/Umbraco.Core/Services/IMediaService.cs | 16 +- src/Umbraco.Core/Services/MediaService.cs | 37 +++- .../Persistence/Querying/ExpressionTests.cs | 14 ++ .../UmbracoExamine/IndexInitializer.cs | 36 ++- .../developer/examinemgmt.controller.js | 8 +- src/UmbracoExamine.PDF/PDFIndexer.cs | 20 +- src/UmbracoExamine/BaseUmbracoIndexer.cs | 9 +- .../DataServices/IContentService.cs | 2 + .../DataServices/IMediaService.cs | 2 + src/UmbracoExamine/UmbracoContentIndexer.cs | 209 +++++++++++++++--- src/UmbracoExamine/UmbracoMemberIndexer.cs | 74 +++++-- 18 files changed, 466 insertions(+), 80 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index a2ffd29bcf..07a64592c1 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -4,6 +4,7 @@ using System.Data; using System.Globalization; using System.Linq; using System.Linq.Expressions; +using System.Text; using System.Xml.Linq; using Umbraco.Core.Configuration; using Umbraco.Core.Dynamics; @@ -617,16 +618,26 @@ namespace Umbraco.Core.Persistence.Repositories public IEnumerable GetPagedResultsByQuery(IQuery query, int pageIndex, int pageSize, out int totalRecords, string orderBy, Direction orderDirection, string filter = "") { + + //NOTE: This uses the GetBaseQuery method but that does not take into account the required 'newest' field which is + // what we always require for a paged result, so we'll ensure it's included in the filter + + var args = new List(); + var sbWhere = new StringBuilder("AND (cmsDocument.newest = 1)"); + + if (filter.IsNullOrWhiteSpace() == false) + { + sbWhere.Append(" AND (cmsDocument." + SqlSyntaxContext.SqlSyntaxProvider.GetQuotedColumnName("text") + " LIKE @" + args.Count + ")"); + args.Add("%" + filter + "%"); + } + + Func> filterCallback = () => new Tuple(sbWhere.ToString(), args.ToArray()); + + return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, "SELECT cmsDocument.nodeId", ProcessQuery, orderBy, orderDirection, - filter.IsNullOrWhiteSpace() - - //NOTE: This uses the GetBaseQuery method but that does not take into account the required 'newest' field which is - // what we always require for a paged result, so we'll ensure it's included in the filter - - ? new Func(() => "AND (cmsDocument.newest = 1)") - : new Func(() => "AND (cmsDocument.newest = 1) AND (cmsDocument." + SqlSyntaxContext.SqlSyntaxProvider.GetQuotedColumnName("text") + " LIKE '%" + filter + "%')")); + filterCallback); } diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs index 10ab5e7f91..e52789c3ac 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs @@ -79,7 +79,7 @@ namespace Umbraco.Core.Persistence.Repositories /// Direction to order by /// Search text filter /// An Enumerable list of objects - IEnumerable GetPagedResultsByQuery(IQuery query, int pageIndex, int pageSize, out int totalRecords, + IEnumerable GetPagedResultsByQuery(IQuery query, int pageIndex, int pageSize, out int totalRecords, string orderBy, Direction orderDirection, string filter = ""); } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index c5966a5bc9..6032ac926b 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Text; using System.Xml.Linq; using Umbraco.Core.Configuration; using Umbraco.Core.Dynamics; @@ -376,12 +377,20 @@ namespace Umbraco.Core.Persistence.Repositories public IEnumerable GetPagedResultsByQuery(IQuery query, int pageIndex, int pageSize, out int totalRecords, string orderBy, Direction orderDirection, string filter = "") { + var args = new List(); + var sbWhere = new StringBuilder(); + Func> filterCallback = null; + if (filter.IsNullOrWhiteSpace() == false) + { + sbWhere.Append("AND (umbracoNode." + SqlSyntaxContext.SqlSyntaxProvider.GetQuotedColumnName("text") + " LIKE @" + args.Count + ")"); + args.Add("%" + filter + "%"); + filterCallback = () => new Tuple(sbWhere.ToString().Trim(), args.ToArray()); + } + return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, "SELECT cmsContentVersion.contentId", ProcessQuery, orderBy, orderDirection, - filter.IsNullOrWhiteSpace() - ? (Func)null - : () => "AND (umbracoNode." + SqlSyntaxContext.SqlSyntaxProvider.GetQuotedColumnName("text") + " LIKE '%" + filter + "%')"); + filterCallback); } diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index 0edd6eff2b..c4fb8d0fd0 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Linq.Expressions; +using System.Text; using System.Xml.Linq; using Umbraco.Core.Configuration; using Umbraco.Core.IO; @@ -571,13 +572,21 @@ namespace Umbraco.Core.Persistence.Repositories public IEnumerable GetPagedResultsByQuery(IQuery query, int pageIndex, int pageSize, out int totalRecords, string orderBy, Direction orderDirection, string filter = "") { + var args = new List(); + var sbWhere = new StringBuilder(); + Func> filterCallback = null; + if (filter.IsNullOrWhiteSpace() == false) + { + sbWhere.Append("AND ((umbracoNode. " + SqlSyntaxContext.SqlSyntaxProvider.GetQuotedColumnName("text") + " LIKE @" + args.Count + ") " + + "OR (cmsMember.LoginName LIKE @0" + args.Count + "))"); + args.Add("%" + filter + "%"); + filterCallback = () => new Tuple(sbWhere.ToString().Trim(), args.ToArray()); + } + return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, "SELECT cmsMember.nodeId", ProcessQuery, orderBy, orderDirection, - filter.IsNullOrWhiteSpace() - ? (Func) null - : () => "AND ((umbracoNode. " + SqlSyntaxContext.SqlSyntaxProvider.GetQuotedColumnName("text") + " LIKE '%" + filter + "%') " + - "OR (cmsMember.LoginName LIKE '%" + filter + "%'))"); + filterCallback); } public void AddOrUpdateContentXml(IMember content, Func xml) diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index 9f453109ab..1cff1cad88 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -246,7 +246,7 @@ namespace Umbraco.Core.Persistence.Repositories Func> processQuery, string orderBy, Direction orderDirection, - Func defaultFilter = null) + Func> defaultFilter = null) where TContentBase : class, IAggregateRoot, TEntity { if (orderBy == null) throw new ArgumentNullException("orderBy"); @@ -265,7 +265,8 @@ namespace Umbraco.Core.Persistence.Repositories // Apply filter if (defaultFilter != null) { - filteredSql.Append(defaultFilter()); + var filterResult = defaultFilter(); + filteredSql.Append(filterResult.Item1, filterResult.Item2); } if (string.IsNullOrEmpty(additionalFilter) == false) { diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index dee50f6080..55c2d33546 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -414,6 +414,7 @@ namespace Umbraco.Core.Services return repository.GetByVersion(versionId); } } + /// /// Gets a collection of an objects versions by Id @@ -491,7 +492,43 @@ namespace Umbraco.Core.Services Mandate.ParameterCondition(pageSize > 0, "pageSize"); using (var repository = _repositoryFactory.CreateContentRepository(_uowProvider.GetUnitOfWork())) { - var query = Query.Builder.Where(x => x.ParentId == id); + + var query = Query.Builder; + //if the id is -1, then just get all + if (id > 0) + { + query.Where(x => x.ParentId == id); + } + var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, filter); + + return contents; + } + } + + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Descendants from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Search text filter + /// An Enumerable list of objects + public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "") + { + Mandate.ParameterCondition(pageIndex >= 0, "pageSize"); + Mandate.ParameterCondition(pageSize > 0, "pageSize"); + using (var repository = _repositoryFactory.CreateContentRepository(_uowProvider.GetUnitOfWork())) + { + + var query = Query.Builder; + //if the id is -1, then just get all + if (id > 0) + { + query.Where(x => x.Path.SqlContains(string.Format(",{0},", id), TextColumnType.NVarchar)); + } var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, filter); return contents; diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index 8408a61296..c85647ccf9 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -121,7 +121,21 @@ namespace Umbraco.Core.Services /// Search text filter /// An Enumerable list of objects IEnumerable GetPagedChildren(int id, int pageIndex, int pageSize, out int totalRecords, - string orderBy, Direction orderDirection, string filter = ""); + string orderBy = "SortOrder", Direction orderDirection = Direction.Ascending, string filter = ""); + + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Descendants from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Search text filter + /// An Enumerable list of objects + IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, + string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of an objects versions by its Id diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs index bfef76b1ec..62b94cc34e 100644 --- a/src/Umbraco.Core/Services/IMediaService.cs +++ b/src/Umbraco.Core/Services/IMediaService.cs @@ -74,7 +74,21 @@ namespace Umbraco.Core.Services /// Search text filter /// An Enumerable list of objects IEnumerable GetPagedChildren(int id, int pageIndex, int pageSize, out int totalRecords, - string orderBy, Direction orderDirection, string filter = ""); + string orderBy = "SortOrder", Direction orderDirection = Direction.Ascending, string filter = ""); + + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Descendants from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Search text filter + /// An Enumerable list of objects + IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, + string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets descendants of a object by its Id diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index 7b6bca4f9c..eb2aeedd21 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -408,13 +408,48 @@ namespace Umbraco.Core.Services Mandate.ParameterCondition(pageSize > 0, "pageSize"); using (var repository = _repositoryFactory.CreateMediaRepository(_uowProvider.GetUnitOfWork())) { - var query = Query.Builder.Where(x => x.ParentId == id); + var query = Query.Builder; + //if the id is -1, then just get all + if (id > 0) + { + query.Where(x => x.ParentId == id); + } var medias = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, filter); return medias; } } + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Descendants from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Search text filter + /// An Enumerable list of objects + public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "") + { + Mandate.ParameterCondition(pageIndex >= 0, "pageSize"); + Mandate.ParameterCondition(pageSize > 0, "pageSize"); + using (var repository = _repositoryFactory.CreateMediaRepository(_uowProvider.GetUnitOfWork())) + { + + var query = Query.Builder; + //if the id is -1, then just get all + if (id > 0) + { + query.Where(x => x.Path.SqlContains(string.Format(",{0},", id), TextColumnType.NVarchar)); + } + var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, filter); + + return contents; + } + } + /// /// Gets descendants of a object by its Id /// diff --git a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs index 29c73232d3..4a2e3f74dc 100644 --- a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs +++ b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs @@ -13,6 +13,20 @@ namespace Umbraco.Tests.Persistence.Querying [TestFixture] public class ExpressionTests : BaseUsingSqlCeSyntax { + // [Test] + // public void Can_Query_With_Content_Type_Alias() + // { + // //Arrange + // Expression> predicate = content => content.ContentType.Alias == "Test"; + // var modelToSqlExpressionHelper = new ModelToSqlExpressionHelper(); + // var result = modelToSqlExpressionHelper.Visit(predicate); + + // Console.WriteLine("Model to Sql ExpressionHelper: \n" + result); + + // Assert.AreEqual("[cmsContentType].[alias] = @0", result); + // Assert.AreEqual("Test", modelToSqlExpressionHelper.GetSqlParameters()[0]); + // } + [Test] public void Can_Verify_Path_StartsWith_Predicate_In_Same_Result() { diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index ba30200f66..e4995ff4e5 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -1,14 +1,19 @@ using System; using System.Linq; +using System.Net.Mime; using Examine; using Examine.LuceneEngine.Config; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; +using Moq; +using Umbraco.Core.Services; using UmbracoExamine; using UmbracoExamine.Config; using UmbracoExamine.DataServices; using UmbracoExamine.PDF; +using IContentService = UmbracoExamine.DataServices.IContentService; +using IMediaService = UmbracoExamine.DataServices.IMediaService; namespace Umbraco.Tests.UmbracoExamine { @@ -20,12 +25,37 @@ namespace Umbraco.Tests.UmbracoExamine public static UmbracoContentIndexer GetUmbracoIndexer( Lucene.Net.Store.Directory luceneDir, Analyzer analyzer = null, - IDataService dataService = null) + IDataService dataService = null, + Umbraco.Core.Services.IContentService contentService = null, + Umbraco.Core.Services.IMediaService mediaService = null, + IDataTypeService dataTypeService = null, + IContentTypeService contentTypeService = null, + IMemberService memberService = null) { if (dataService == null) { dataService = new TestDataService(); } + if (contentService == null) + { + contentService = Mock.Of(); + } + if (mediaService == null) + { + mediaService = Mock.Of(); + } + if (dataTypeService == null) + { + dataTypeService = Mock.Of(); + } + if (contentTypeService == null) + { + contentTypeService = Mock.Of(); + } + if (memberService == null) + { + memberService = Mock.Of(); + } if (analyzer == null) { @@ -38,6 +68,10 @@ namespace Umbraco.Tests.UmbracoExamine var i = new UmbracoContentIndexer(indexCriteria, luceneDir, //custom lucene directory dataService, + contentService, + mediaService, + dataTypeService, + contentTypeService, analyzer, false); diff --git a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js index 84a6d0fe3a..125f6bc7f4 100644 --- a/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/dashboard/developer/examinemgmt.controller.js @@ -1,4 +1,4 @@ -function examineMgmtController($scope, umbRequestHelper, $log, $http, $q) { +function examineMgmtController($scope, umbRequestHelper, $log, $http, $q, $timeout) { $scope.indexerDetails = []; $scope.searcherDetails = []; @@ -19,7 +19,7 @@ function examineMgmtController($scope, umbRequestHelper, $log, $http, $q) { indexer.isProcessing = false; } else { - setTimeout(function() { + $timeout(function () { //don't continue if we've tried 100 times if (indexer.processingAttempts < 100) { checkProcessing(indexer, checkActionName); @@ -76,7 +76,7 @@ function examineMgmtController($scope, umbRequestHelper, $log, $http, $q) { //rebuilding has started, nothing is returned accept a 200 status code. //lets poll to see if it is done. - setTimeout(function () { + $timeout(function () { checkProcessing(indexer, "PostCheckRebuildIndex"); }, 1000); @@ -97,7 +97,7 @@ function examineMgmtController($scope, umbRequestHelper, $log, $http, $q) { //optimizing has started, nothing is returned accept a 200 status code. //lets poll to see if it is done. - setTimeout(function () { + $timeout(function () { checkProcessing(indexer, "PostCheckOptimizeIndex"); }, 1000); diff --git a/src/UmbracoExamine.PDF/PDFIndexer.cs b/src/UmbracoExamine.PDF/PDFIndexer.cs index 475a75c3ce..ef07b3929f 100644 --- a/src/UmbracoExamine.PDF/PDFIndexer.cs +++ b/src/UmbracoExamine.PDF/PDFIndexer.cs @@ -232,17 +232,19 @@ namespace UmbracoExamine.PDF try { - var reader = new PdfReader(pdfPath); - - for (int i = 1; i <= reader.NumberOfPages; i++) + using (var reader = new PdfReader(pdfPath)) { - var result = - ExceptChars( - PdfTextExtractor.GetTextFromPage(reader, i, new SimpleTextExtractionStrategy()), - UnsupportedRange, - ReplaceWithSpace); - output.Write(result); + for (int i = 1; i <= reader.NumberOfPages; i++) + { + var result = + ExceptChars( + PdfTextExtractor.GetTextFromPage(reader, i, new SimpleTextExtractionStrategy()), + UnsupportedRange, + ReplaceWithSpace); + output.Write(result); + } } + } catch (Exception ex) { diff --git a/src/UmbracoExamine/BaseUmbracoIndexer.cs b/src/UmbracoExamine/BaseUmbracoIndexer.cs index 1ab0eae498..44ecb0cfcb 100644 --- a/src/UmbracoExamine/BaseUmbracoIndexer.cs +++ b/src/UmbracoExamine/BaseUmbracoIndexer.cs @@ -264,8 +264,8 @@ namespace UmbracoExamine /// protected override void PerformIndexAll(string type) { - //TODO: Fix all of this up, the whole xpath thing is horrible and was made sooooooooo long ago to only work with published content - // but not it's being used for all content types and is really bad for performance. + //NOTE: the logic below is ONLY used for published content, for media and members and non-published content, this method is overridden + // and we query directly against the umbraco service layer. if (!SupportedTypes.Contains(type)) return; @@ -275,7 +275,7 @@ namespace UmbracoExamine var sb = new StringBuilder(); //create the xpath statement to match node type aliases if specified - if (IndexerData.IncludeNodeTypes.Count() > 0) + if (IndexerData.IncludeNodeTypes.Any()) { sb.Append("("); foreach (var field in IndexerData.IncludeNodeTypes) @@ -328,6 +328,9 @@ namespace UmbracoExamine /// Either the Content or Media xml. If the type is not of those specified null is returned protected virtual XDocument GetXDocument(string xPath, string type) { + //TODO: We need to get rid of this! it will now only ever be called for published content - but we're keeping the other + // logic here for backwards compatibility in case inheritors are calling this for some reason. + if (type == IndexTypes.Content) { if (this.SupportUnpublishedContent) diff --git a/src/UmbracoExamine/DataServices/IContentService.cs b/src/UmbracoExamine/DataServices/IContentService.cs index 502938fbd2..f0224bc771 100644 --- a/src/UmbracoExamine/DataServices/IContentService.cs +++ b/src/UmbracoExamine/DataServices/IContentService.cs @@ -7,7 +7,9 @@ namespace UmbracoExamine.DataServices { public interface IContentService { + [Obsolete("This should no longer be used, latest content will be indexed by using the IContentService directly")] XDocument GetLatestContentByXPath(string xpath); + XDocument GetPublishedContentByXPath(string xpath); /// diff --git a/src/UmbracoExamine/DataServices/IMediaService.cs b/src/UmbracoExamine/DataServices/IMediaService.cs index 88c07a281c..13afe84bd5 100644 --- a/src/UmbracoExamine/DataServices/IMediaService.cs +++ b/src/UmbracoExamine/DataServices/IMediaService.cs @@ -2,8 +2,10 @@ using System.Xml.Linq; namespace UmbracoExamine.DataServices { + [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] public interface IMediaService { + [Obsolete("This should no longer be used, latest content will be indexed by using the IMediaService directly")] XDocument GetLatestMediaByXpath(string xpath); } } diff --git a/src/UmbracoExamine/UmbracoContentIndexer.cs b/src/UmbracoExamine/UmbracoContentIndexer.cs index 50afe7e9cd..ab54116e6e 100644 --- a/src/UmbracoExamine/UmbracoContentIndexer.cs +++ b/src/UmbracoExamine/UmbracoContentIndexer.cs @@ -14,6 +14,9 @@ using Examine.Providers; using Lucene.Net.Documents; using Umbraco.Core; using umbraco.cms.businesslogic; +using Umbraco.Core.Models; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; +using Umbraco.Core.Services; using UmbracoExamine.DataServices; using Examine.LuceneEngine; using Examine.LuceneEngine.Config; @@ -21,6 +24,8 @@ using UmbracoExamine.Config; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; using umbraco.BasePages; +using IContentService = Umbraco.Core.Services.IContentService; +using IMediaService = Umbraco.Core.Services.IMediaService; namespace UmbracoExamine @@ -30,13 +35,24 @@ namespace UmbracoExamine /// public class UmbracoContentIndexer : BaseUmbracoIndexer { + private readonly IContentTypeService _contentTypeService; + private readonly IContentService _contentService; + private readonly IMediaService _mediaService; + private readonly IDataTypeService _dataTypeService; + #region Constructors /// /// Default constructor /// public UmbracoContentIndexer() - : base() { } + : base() + { + _contentService = ApplicationContext.Current.Services.ContentService; + _mediaService = ApplicationContext.Current.Services.MediaService; + _dataTypeService = ApplicationContext.Current.Services.DataTypeService; + _contentTypeService = ApplicationContext.Current.Services.ContentTypeService; + } /// /// Constructor to allow for creating an indexer at runtime @@ -45,21 +61,60 @@ namespace UmbracoExamine /// /// /// - - public UmbracoContentIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) - : base(indexerData, indexPath, dataService, analyzer, async) { } + /// + [Obsolete("Use the overload that specifies the Umbraco services")] + public UmbracoContentIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, indexPath, dataService, analyzer, async) + { + _contentService = ApplicationContext.Current.Services.ContentService; + _mediaService = ApplicationContext.Current.Services.MediaService; + _dataTypeService = ApplicationContext.Current.Services.DataTypeService; + _contentTypeService = ApplicationContext.Current.Services.ContentTypeService; + } - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - /// - /// - /// - - public UmbracoContentIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, Analyzer analyzer, bool async) - : base(indexerData, luceneDirectory, dataService, analyzer, async) { } + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + /// + [Obsolete("Use the overload that specifies the Umbraco services")] + public UmbracoContentIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, luceneDirectory, dataService, analyzer, async) + { + _contentService = ApplicationContext.Current.Services.ContentService; + _mediaService = ApplicationContext.Current.Services.MediaService; + _dataTypeService = ApplicationContext.Current.Services.DataTypeService; + _contentTypeService = ApplicationContext.Current.Services.ContentTypeService; + } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public UmbracoContentIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, + IContentService contentService, + IMediaService mediaService, + IDataTypeService dataTypeService, + IContentTypeService contentTypeService, + Analyzer analyzer, bool async) + : base(indexerData, luceneDirectory, dataService, analyzer, async) + { + _contentService = contentService; + _mediaService = mediaService; + _dataTypeService = dataTypeService; + _contentTypeService = contentTypeService; + } #endregion @@ -124,10 +179,10 @@ namespace UmbracoExamine /// /// An attempt is made to call on a provider after the provider has already been initialized. /// - + public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { - + //check if there's a flag specifying to support unpublished content, //if not, set to false; bool supportUnpublished; @@ -181,19 +236,19 @@ namespace UmbracoExamine /// This ensures that the special __Raw_ fields are indexed /// /// - + protected override void OnDocumentWriting(DocumentWritingEventArgs docArgs) { var d = docArgs.Document; foreach (var f in docArgs.Fields.Where(x => x.Key.StartsWith(RawFieldPrefix))) - { + { d.Add(new Field( f.Key, f.Value, Field.Store.YES, Field.Index.NO, //don't index this field, we never want to search by it - Field.TermVector.NO)); - } + Field.TermVector.NO)); + } base.OnDocumentWriting(docArgs); } @@ -220,7 +275,7 @@ namespace UmbracoExamine #region Public methods - + /// /// Overridden for logging /// @@ -233,14 +288,14 @@ namespace UmbracoExamine if (node.Attribute("id") != null) { - DataService.LogService.AddVerboseLog((int) node.Attribute("id"), string.Format("ReIndexNode with type: {0}", type)); + DataService.LogService.AddVerboseLog((int)node.Attribute("id"), string.Format("ReIndexNode with type: {0}", type)); base.ReIndexNode(node, type); } else { DataService.LogService.AddErrorLog(-1, string.Format("ReIndexNode cannot proceed, the format of the XElement is invalid, the xml has no 'id' attribute. {0}", node)); } - + } /// @@ -279,7 +334,107 @@ namespace UmbracoExamine #region Protected + protected override void PerformIndexAll(string type) + { + + const int pageSize = 5000; + var pageIndex = 0; + switch (type) + { + case IndexTypes.Content: + if (this.SupportUnpublishedContent == false) + { + //use the base implementation which will use the published XML cache to perform the lookups + base.PerformIndexAll(type); + } + else + { + var contentParentId = -1; + if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) + { + contentParentId = IndexerData.ParentNodeId.Value; + } + IContent[] content; + + do + { + int total; + var descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total); + + //if specific types are declared we need to post filter them + //TODO: Update the service layer to join the cmsContentType table so we can query by content type too + if (IndexerData.IncludeNodeTypes.Any()) + { + content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); + } + else + { + content = descendants.ToArray(); + } + + AddNodesToIndex(GetSerializedContent(content), type); + pageIndex++; + } while (content.Length == pageSize); + + } + break; + case IndexTypes.Media: + + var mediaParentId = -1; + if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) + { + mediaParentId = IndexerData.ParentNodeId.Value; + } + IMedia[] media; + + do + { + int total; + var descendants = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out total); + + //if specific types are declared we need to post filter them + //TODO: Update the service layer to join the cmsContentType table so we can query by content type too + if (IndexerData.IncludeNodeTypes.Any()) + { + media = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); + } + else + { + media = descendants.ToArray(); + } + + AddNodesToIndex(GetSerializedMedia(media), type); + pageIndex++; + } while (media.Length == pageSize); + + break; + } + } + + private IEnumerable GetSerializedMedia(IEnumerable media) + { + var serializer = new EntityXmlSerializer(); + foreach (var m in media) + { + yield return serializer.Serialize( + _mediaService, + _dataTypeService, + m); + } + } + + private IEnumerable GetSerializedContent(IEnumerable content) + { + var serializer = new EntityXmlSerializer(); + foreach (var c in content) + { + yield return serializer.Serialize( + _contentService, + _dataTypeService, + c); + } + } /// /// Overridden for logging. @@ -316,7 +471,7 @@ namespace UmbracoExamine /// ensure our special Path field is added to the collection /// /// - + protected override void OnGatheringNodeData(IndexingNodeDataEventArgs e) { //strip html of all users fields if we detect it has HTML in it. @@ -333,7 +488,7 @@ namespace UmbracoExamine //First save the raw value to a raw field, we will change the policy of this field by detecting the prefix later e.Fields[RawFieldPrefix + field.Name] = e.Fields[field.Name]; //now replace the original value with the stripped html - e.Fields[field.Name] = DataService.ContentService.StripHtml(e.Fields[field.Name]); + e.Fields[field.Name] = DataService.ContentService.StripHtml(e.Fields[field.Name]); } } } @@ -401,7 +556,7 @@ namespace UmbracoExamine { return base.GetIndexerData(indexSet); } - + } /// diff --git a/src/UmbracoExamine/UmbracoMemberIndexer.cs b/src/UmbracoExamine/UmbracoMemberIndexer.cs index c55a73c7ff..83652ecd13 100644 --- a/src/UmbracoExamine/UmbracoMemberIndexer.cs +++ b/src/UmbracoExamine/UmbracoMemberIndexer.cs @@ -7,6 +7,7 @@ using System.Xml.XPath; using Examine.LuceneEngine.Config; using Umbraco.Core; using Umbraco.Core.Models; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Services; using UmbracoExamine.Config; using Examine.LuceneEngine; @@ -25,22 +26,49 @@ namespace UmbracoExamine public class UmbracoMemberIndexer : UmbracoContentIndexer { + private readonly IMemberService _memberService; + private readonly IDataTypeService _dataTypeService; + /// /// Default constructor /// public UmbracoMemberIndexer() : base() { } + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + [Obsolete("Use the overload that specifies the Umbraco services")] + public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) + : base(indexerData, indexPath, dataService, analyzer, async) + { + _dataTypeService = ApplicationContext.Current.Services.DataTypeService; + _memberService = ApplicationContext.Current.Services.MemberService; + } + /// /// Constructor to allow for creating an indexer at runtime /// /// /// /// + /// + /// /// - - public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) - : base(indexerData, indexPath, dataService, analyzer, async) { } + /// + public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, + IDataTypeService dataTypeService, + IMemberService memberService, + Analyzer analyzer, bool async) + : base(indexerData, indexPath, dataService, analyzer, async) + { + _dataTypeService = dataTypeService; + _memberService = memberService; + } /// /// Ensures that the'_searchEmail' is added to the user fields so that it is indexed - without having to modify the config @@ -83,23 +111,39 @@ namespace UmbracoExamine protected override void PerformIndexAll(string type) { //This only supports members - if (!SupportedTypes.Contains(type)) + if (SupportedTypes.Contains(type) == false) return; //Re-index all members in batches of 5000 - IEnumerable members; + IMember[] members; const int pageSize = 5000; var pageIndex = 0; - do - { - int total; - members = ApplicationContext.Current.Services.MemberService.GetAll(pageIndex, pageSize, out total); - AddNodesToIndex(GetSerializedMembers(members), type); - - pageIndex++; - - } while (members.Count() == pageSize); + if (IndexerData.IncludeNodeTypes.Any()) + { + //if there are specific node types then just index those + foreach (var nodeType in IndexerData.IncludeNodeTypes) + { + do + { + int total; + members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, nodeType).ToArray(); + AddNodesToIndex(GetSerializedMembers(members), type); + pageIndex++; + } while (members.Length == pageSize); + } + } + else + { + //no node types specified, do all members + do + { + int total; + members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray(); + AddNodesToIndex(GetSerializedMembers(members), type); + pageIndex++; + } while (members.Length == pageSize); + } } private IEnumerable GetSerializedMembers(IEnumerable members) @@ -107,7 +151,7 @@ namespace UmbracoExamine var serializer = new EntityXmlSerializer(); foreach (var member in members) { - yield return serializer.Serialize(ApplicationContext.Current.Services.DataTypeService, member); + yield return serializer.Serialize(_dataTypeService, member); } }