From 221354a7d8c2bcb41646c56fe3602e8a57b90152 Mon Sep 17 00:00:00 2001 From: Claus Date: Thu, 27 Oct 2016 10:47:32 +0200 Subject: [PATCH] U4-9104 Update the UmbracoExamine logic for Media to read the data directly from the umbracoXml table. updated some tests to use the testbehavior attribute so they dont rely on being run in a specific order. --- .../Interfaces/IMediaRepository.cs | 10 ++ .../Repositories/MediaRepository.cs | 24 ++++ src/Umbraco.Core/Services/IMediaService.cs | 14 ++ src/Umbraco.Core/Services/MediaService.cs | 21 +++ .../LegacyExamineBackedMediaTests.cs | 1 + .../UmbracoExamine/EventsTest.cs | 4 +- .../UmbracoExamine/IndexInitializer.cs | 60 ++++++-- src/Umbraco.Tests/UmbracoExamine/IndexTest.cs | 13 +- .../UmbracoExamine/SearchTests.cs | 2 + src/UmbracoExamine/UmbracoContentIndexer.cs | 130 +++++++++--------- 10 files changed, 193 insertions(+), 86 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs index 907f9b62c5..64989f9269 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs @@ -38,5 +38,15 @@ namespace Umbraco.Core.Persistence.Repositories /// An Enumerable list of objects IEnumerable GetPagedResultsByQuery(IQuery query, long pageIndex, int pageSize, out long totalRecords, string orderBy, Direction orderDirection, bool orderBySystemField, string filter = ""); + + /// + /// Gets paged media descendants as XML by path + /// + /// Path starts with + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of media items + IEnumerable GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords); } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index 598c9e912d..7a6fa4c34e 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -465,6 +465,30 @@ namespace Umbraco.Core.Persistence.Repositories } + /// + /// Gets paged media descendants as XML by path + /// + /// Path starts with + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of media items + public IEnumerable GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords) + { + Sql query; + if (path == "-1") + { + query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE nodeObjectType = @0)", Guid.Parse(Constants.ObjectTypes.Media)).OrderBy("nodeId"); + } + else + { + query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE path LIKE @0)", path.EnsureEndsWith(",%")).OrderBy("nodeId"); + } + var pagedResult = Database.Page(pageIndex+1, pageSize, query); + totalRecords = pagedResult.TotalItems; + return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml)); + } + private IEnumerable ProcessQuery(Sql sql) { //NOTE: This doesn't allow properties to be part of the query diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs index 6ff8f75402..d25ddf7f58 100644 --- a/src/Umbraco.Core/Services/IMediaService.cs +++ b/src/Umbraco.Core/Services/IMediaService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Xml.Linq; using Umbraco.Core.Configuration; using Umbraco.Core.Models; using Umbraco.Core.Persistence.DatabaseModelDefinitions; @@ -57,6 +58,19 @@ namespace Umbraco.Core.Services /// public interface IMediaService : IService { + /// + /// Gets all XML entries found in the cmsContentXml table based on the given path + /// + /// Path starts with + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of media items + /// + /// If -1 is passed, then this will return all media xml entries, otherwise will return all descendents from the path + /// + IEnumerable GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords); + /// /// Rebuilds all xml content in the cmsContentXml table for all media /// diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index 29235cc7ab..a1182ce80a 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -1199,6 +1199,27 @@ namespace Umbraco.Core.Services return true; } + /// + /// Gets paged media descendants as XML by path + /// + /// Path starts with + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of media items + public IEnumerable GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords) + { + Mandate.ParameterCondition(pageIndex >= 0, "pageIndex"); + Mandate.ParameterCondition(pageSize > 0, "pageSize"); + + var uow = UowProvider.GetUnitOfWork(); + using (var repository = RepositoryFactory.CreateMediaRepository(uow)) + { + var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, out totalRecords); + return contents; + } + } + /// /// Rebuilds all xml content in the cmsContentXml table for all media /// diff --git a/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs index 8a83bea75a..5bf6a4edc5 100644 --- a/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs @@ -13,6 +13,7 @@ using Umbraco.Core.Persistence.Mappers; namespace Umbraco.Tests.PublishedContent { + [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] public class LegacyExamineBackedMediaTests : ExamineBaseTest { public override void Initialize() diff --git a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs index 6bd01c7f1c..3e7377f3b6 100644 --- a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs @@ -3,11 +3,13 @@ using System.Linq; using Examine; using Lucene.Net.Store; using NUnit.Framework; +using Umbraco.Tests.TestHelpers; using UmbracoExamine; namespace Umbraco.Tests.UmbracoExamine { - [TestFixture] + [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] + [TestFixture] public class EventsTest : ExamineBaseTest { [Test] diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index b303eed997..89a9df8052 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.Linq; +using System.Xml.Linq; using Examine; using Examine.LuceneEngine.Config; using Examine.LuceneEngine.Providers; @@ -7,10 +9,14 @@ using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; using Lucene.Net.Store; using Moq; +using Umbraco.Core; +using Umbraco.Core.Logging; using Umbraco.Core.Models; using Umbraco.Core.Models.Membership; +using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Querying; +using Umbraco.Core.Persistence.SqlSyntax; using Umbraco.Core.Services; using UmbracoExamine; using UmbracoExamine.Config; @@ -34,7 +40,8 @@ namespace Umbraco.Tests.UmbracoExamine IMediaService mediaService = null, IDataTypeService dataTypeService = null, IMemberService memberService = null, - IUserService userService = null) + IUserService userService = null, + IContentTypeService contentTypeService = null) { if (dataService == null) { @@ -94,7 +101,8 @@ namespace Umbraco.Tests.UmbracoExamine long longTotalRecs; int intTotalRecs; - var allRecs = dataService.MediaService.GetLatestMediaByXpath("//node") + var mediaXml = dataService.MediaService.GetLatestMediaByXpath("//node"); + var allRecs = mediaXml .Root .Elements() .Select(x => Mock.Of( @@ -114,20 +122,29 @@ namespace Umbraco.Tests.UmbracoExamine mt.Id == (int)x.Attribute("nodeType")))) .ToArray(); + // MOCK! + var mediaServiceMock = new Mock(); + + mediaServiceMock + .Setup(x => x.GetPagedDescendants( + It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()) + ).Returns(() => allRecs); + + mediaServiceMock + .Setup(x => x.GetPagedDescendants( + It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs, It.IsAny(), It.IsAny(), It.IsAny()) + ).Returns(() => allRecs); + + mediaServiceMock + .Setup(x => x.GetPagedDescendants( + It.IsAny(), It.IsAny(), It.IsAny(), out intTotalRecs, It.IsAny(), It.IsAny(), It.IsAny()) + ).Returns(() => allRecs); + + mediaServiceMock.Setup(service => service.GetPagedXmlEntries(It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs)) + .Returns(() => allRecs.Select(x => x.ToXml())); + + mediaService = mediaServiceMock.Object; - mediaService = Mock.Of( - x => x.GetPagedDescendants( - It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()) - == - allRecs - && x.GetPagedDescendants( - It.IsAny(), It.IsAny(), It.IsAny(), out longTotalRecs, It.IsAny(), It.IsAny(), It.IsAny()) - == - allRecs - && x.GetPagedDescendants( - It.IsAny(), It.IsAny(), It.IsAny(), out intTotalRecs, It.IsAny(), It.IsAny(), It.IsAny()) - == - allRecs); } if (dataTypeService == null) { @@ -139,6 +156,18 @@ namespace Umbraco.Tests.UmbracoExamine memberService = Mock.Of(); } + if (contentTypeService == null) + { + var contentTypeServiceMock = new Mock(); + contentTypeServiceMock.Setup(x => x.GetAllContentTypes()) + .Returns(new List() + { + new ContentType(-1) {Alias = "Folder", Name = "Folder", Id = 1031, Icon = "icon-folder"}, + new ContentType(-1) {Alias = "Image", Name = "Image", Id = 1032, Icon = "icon-picture"} + }); + contentTypeService = contentTypeServiceMock.Object; + } + if (analyzer == null) { analyzer = new StandardAnalyzer(Version.LUCENE_29); @@ -154,6 +183,7 @@ namespace Umbraco.Tests.UmbracoExamine mediaService, dataTypeService, userService, + contentTypeService, analyzer, false); diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs index 7c36dd2953..3bc635bb23 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs @@ -10,15 +10,17 @@ using Lucene.Net.Index; using Lucene.Net.Search; using Lucene.Net.Store; using NUnit.Framework; +using Umbraco.Tests.TestHelpers; using UmbracoExamine; namespace Umbraco.Tests.UmbracoExamine { - /// - /// Tests the standard indexing capabilities - /// - [TestFixture, RequiresSTA] + /// + /// Tests the standard indexing capabilities + /// + //[DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] + [TestFixture, RequiresSTA] public class IndexTest : ExamineBaseTest { @@ -85,12 +87,11 @@ namespace Umbraco.Tests.UmbracoExamine //RESET the parent id existingCriteria = ((IndexCriteria)_indexer.IndexerData); _indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes, - null); + null); //now ensure it's deleted var newResults = _searcher.Search(_searcher.CreateSearchCriteria().Id(2112).Compile()); Assert.AreEqual(1, newResults.Count()); - } [Test] diff --git a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs index bcd9922e21..9b8b3d50d7 100644 --- a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs +++ b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs @@ -8,9 +8,11 @@ using Examine.LuceneEngine.Providers; using Lucene.Net.Store; using NUnit.Framework; using Examine.LuceneEngine.SearchCriteria; +using Umbraco.Tests.TestHelpers; namespace Umbraco.Tests.UmbracoExamine { + [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] [TestFixture] public class SearchTests : ExamineBaseTest { diff --git a/src/UmbracoExamine/UmbracoContentIndexer.cs b/src/UmbracoExamine/UmbracoContentIndexer.cs index 4fcf51deae..ce3583a6be 100644 --- a/src/UmbracoExamine/UmbracoContentIndexer.cs +++ b/src/UmbracoExamine/UmbracoContentIndexer.cs @@ -1,20 +1,12 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; using System.IO; using System.Linq; -using System.Security; -using System.Text; -using System.Web; using System.Xml.Linq; using Examine; -using Examine.Config; -using Examine.Providers; using Lucene.Net.Documents; -using Lucene.Net.Index; using Umbraco.Core; -using umbraco.cms.businesslogic; using Umbraco.Core.Models; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Services; @@ -22,12 +14,9 @@ using UmbracoExamine.DataServices; using Examine.LuceneEngine; using Examine.LuceneEngine.Config; using UmbracoExamine.Config; -using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; -using umbraco.BasePages; using Umbraco.Core.Persistence.Querying; using IContentService = Umbraco.Core.Services.IContentService; -using UmbracoExamine.LocalStorage; using IMediaService = Umbraco.Core.Services.IMediaService; @@ -42,6 +31,7 @@ namespace UmbracoExamine private readonly IMediaService _mediaService; private readonly IDataTypeService _dataTypeService; private readonly IUserService _userService; + private readonly IContentTypeService _contentTypeService; #region Constructors @@ -55,6 +45,7 @@ namespace UmbracoExamine _mediaService = ApplicationContext.Current.Services.MediaService; _dataTypeService = ApplicationContext.Current.Services.DataTypeService; _userService = ApplicationContext.Current.Services.UserService; + _contentTypeService = ApplicationContext.Current.Services.ContentTypeService; } /// @@ -73,6 +64,7 @@ namespace UmbracoExamine _mediaService = ApplicationContext.Current.Services.MediaService; _dataTypeService = ApplicationContext.Current.Services.DataTypeService; _userService = ApplicationContext.Current.Services.UserService; + _contentTypeService = ApplicationContext.Current.Services.ContentTypeService; } /// @@ -91,6 +83,7 @@ namespace UmbracoExamine _mediaService = ApplicationContext.Current.Services.MediaService; _dataTypeService = ApplicationContext.Current.Services.DataTypeService; _userService = ApplicationContext.Current.Services.UserService; + _contentTypeService = ApplicationContext.Current.Services.ContentTypeService; } /// @@ -105,6 +98,7 @@ namespace UmbracoExamine /// /// /// + [Obsolete("Use the overload that specifies the Umbraco services")] public UmbracoContentIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, IContentService contentService, IMediaService mediaService, @@ -117,13 +111,43 @@ namespace UmbracoExamine _mediaService = mediaService; _dataTypeService = dataTypeService; _userService = userService; + _contentTypeService = ApplicationContext.Current.Services.ContentTypeService; + } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public UmbracoContentIndexer(IIndexCriteria indexerData, Lucene.Net.Store.Directory luceneDirectory, IDataService dataService, + IContentService contentService, + IMediaService mediaService, + IDataTypeService dataTypeService, + IUserService userService, + IContentTypeService contentTypeService, + Analyzer analyzer, bool async) + : base(indexerData, luceneDirectory, dataService, analyzer, async) + { + _contentService = contentService; + _mediaService = mediaService; + _dataTypeService = dataTypeService; + _userService = userService; + _contentTypeService = contentTypeService; } #endregion #region Constants & Fields - + /// /// Used to store the path of a content object @@ -206,13 +230,8 @@ namespace UmbracoExamine SupportProtectedContent = supportProtected; else SupportProtectedContent = false; - - + base.Initialize(name, config); - - - - } #endregion @@ -285,10 +304,7 @@ namespace UmbracoExamine #endregion #region Public methods - - - /// /// Overridden for logging /// @@ -308,7 +324,6 @@ namespace UmbracoExamine { DataService.LogService.AddErrorLog(-1, string.Format("ReIndexNode cannot proceed, the format of the XElement is invalid, the xml has no 'id' attribute. {0}", node)); } - } /// @@ -355,8 +370,6 @@ namespace UmbracoExamine switch (type) { case IndexTypes.Content: - - var contentParentId = -1; if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) { @@ -391,69 +404,63 @@ namespace UmbracoExamine { content = descendants.ToArray(); } - AddNodesToIndex(GetSerializedContent(content), type); pageIndex++; - - } while (content.Length == pageSize); break; case IndexTypes.Media: - var mediaParentId = -1; + if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) { mediaParentId = IndexerData.ParentNodeId.Value; } - IMedia[] media; + XElement[] mediaXElements; + + var nodeTypes = _contentTypeService.GetAllContentTypes().ToArray(); + var icons = nodeTypes.ToDictionary(x => x.Id, y => y.Icon); + do { long total; - var descendants = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out total); + if (mediaParentId == -1) + { + mediaXElements = _mediaService.GetPagedXmlEntries("-1", pageIndex, pageSize, out total).ToArray(); + } + else + { + //Get the parent + var parent = _mediaService.GetById(mediaParentId); + if (parent == null) + mediaXElements = new XElement[0]; + else + mediaXElements = _mediaService.GetPagedXmlEntries(parent.Path, pageIndex, pageSize, out total).ToArray(); + } //if specific types are declared we need to post filter them //TODO: Update the service layer to join the cmsContentType table so we can query by content type too if (IndexerData.IncludeNodeTypes.Any()) { - media = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); + var includeNodeTypeIds = nodeTypes.Where(x => IndexerData.IncludeNodeTypes.Contains(x.Alias)).Select(x => x.Id); + mediaXElements = mediaXElements.Where(elm => includeNodeTypeIds.Contains(elm.AttributeValue("nodeType"))).ToArray(); } - else + + // ReSharper disable once ForCanBeConvertedToForeach + for (var i = 0; i < mediaXElements.Length; i++) { - media = descendants.ToArray(); + mediaXElements[i].Add(new XAttribute("icon", icons[mediaXElements[i].AttributeValue("nodeType")])); } - - AddNodesToIndex(GetSerializedMedia(media), type); + + AddNodesToIndex(mediaXElements, type); pageIndex++; - } while (media.Length == pageSize); + } while (mediaXElements.Length == pageSize); break; } } - private IEnumerable GetSerializedMedia(IEnumerable media) - { - var serializer = new EntityXmlSerializer(); - foreach (var m in media) - { - var xml = serializer.Serialize( - _mediaService, - _dataTypeService, - _userService, - m); - - //add a custom 'icon' attribute - if (m.ContentType.Icon.IsNullOrWhiteSpace() == false) - { - xml.Add(new XAttribute("icon", m.ContentType.Icon)); - } - - - yield return xml; - } - } - private IEnumerable GetSerializedContent(IEnumerable content) { var serializer = new EntityXmlSerializer(); @@ -510,7 +517,6 @@ namespace UmbracoExamine protected override void OnGatheringNodeData(IndexingNodeDataEventArgs e) { - //strip html of all users fields if we detect it has HTML in it. //if that is the case, we'll create a duplicate 'raw' copy of it so that we can return //the value of the field 'as-is'. @@ -546,7 +552,6 @@ namespace UmbracoExamine var icon = (string)e.Node.Attribute("icon"); if (!e.Fields.ContainsKey(IconFieldName)) e.Fields.Add(IconFieldName, icon); - } /// @@ -584,7 +589,6 @@ namespace UmbracoExamine } return fields; - } /// @@ -605,7 +609,6 @@ namespace UmbracoExamine { return base.GetIndexerData(indexSet); } - } /// @@ -635,10 +638,9 @@ namespace UmbracoExamine { return false; } - return base.ValidateDocument(node); } #endregion } -} +} \ No newline at end of file