diff --git a/build/NuSpecs/UmbracoCms.Core.nuspec b/build/NuSpecs/UmbracoCms.Core.nuspec index 2536316906..0f31f77d04 100644 --- a/build/NuSpecs/UmbracoCms.Core.nuspec +++ b/build/NuSpecs/UmbracoCms.Core.nuspec @@ -32,9 +32,9 @@ - + - + @@ -104,4 +104,4 @@ - \ No newline at end of file + diff --git a/src/Umbraco.Core/Models/IMediaType.cs b/src/Umbraco.Core/Models/IMediaType.cs index 29e4b665ba..e8f7c16190 100644 --- a/src/Umbraco.Core/Models/IMediaType.cs +++ b/src/Umbraco.Core/Models/IMediaType.cs @@ -4,7 +4,7 @@ namespace Umbraco.Core.Models { /// /// Defines a ContentType, which Media is based on - /// public interface IMediaType : IContentTypeComposition { diff --git a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs index b265a5b587..7f5e479af6 100644 --- a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs +++ b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs @@ -20,6 +20,7 @@ namespace Umbraco.Core.Persistence.Querying _mapper = mapper; } + [Obsolete("Use the overload the specifies a SqlSyntaxProvider")] public ModelToSqlExpressionVisitor() : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T))) { } diff --git a/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionVisitor.cs b/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionVisitor.cs index e0a4f07e48..4569b95853 100644 --- a/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionVisitor.cs +++ b/src/Umbraco.Core/Persistence/Querying/PocoToSqlExpressionVisitor.cs @@ -18,14 +18,13 @@ namespace Umbraco.Core.Persistence.Querying public PocoToSqlExpressionVisitor(ISqlSyntaxProvider syntaxProvider) : base(syntaxProvider) { - + _pd = new Database.PocoData(typeof(T)); } [Obsolete("Use the overload the specifies a SqlSyntaxProvider")] public PocoToSqlExpressionVisitor() - : base(SqlSyntaxContext.SqlSyntaxProvider) - { - _pd = new Database.PocoData(typeof(T)); + : this(SqlSyntaxContext.SqlSyntaxProvider) + { } protected override string VisitMemberAccess(MemberExpression m) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 7f3e657dc2..ba95e1c574 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -625,13 +625,13 @@ namespace Umbraco.Core.Persistence.Repositories var sqlClause = GetBaseQuery(false); var translator = new SqlTranslator(sqlClause, query); var sql = translator.Translate() - .Where(x => x.Published) + .Where(x => x.Published, SqlSyntax) .OrderBy(x => x.Level, SqlSyntax) .OrderBy(x => x.SortOrder, SqlSyntax); return ProcessQuery(sql, true); } - + /// /// This builds the Xml document used for the XML cache /// @@ -663,10 +663,14 @@ namespace Umbraco.Core.Persistence.Repositories parent.Attributes.Append(pIdAtt); xmlDoc.AppendChild(parent); - const string sql = @"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.xml, umbracoNode.level from umbracoNode + //Ensure that only nodes that have published versions are selected + var sql = string.Format(@"select umbracoNode.id, umbracoNode.parentID, umbracoNode.sortOrder, cmsContentXml.{0}, umbracoNode.{1} from umbracoNode inner join cmsContentXml on cmsContentXml.nodeId = umbracoNode.id and umbracoNode.nodeObjectType = @type where umbracoNode.id in (select cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1) -order by umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder"; +order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", + SqlSyntax.GetQuotedColumnName("xml"), + SqlSyntax.GetQuotedColumnName("level"), + SqlSyntax.GetQuotedColumnName("level")); XmlElement last = null; diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityContainerRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityContainerRepository.cs index cc13275798..b76abf4db6 100644 --- a/src/Umbraco.Core/Persistence/Repositories/EntityContainerRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/EntityContainerRepository.cs @@ -62,17 +62,27 @@ namespace Umbraco.Core.Persistence.Repositories protected override IEnumerable PerformGetAll(params int[] ids) { - //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit - return ids.InGroupsOf(2000).SelectMany(@group => + if (ids.Any()) + { + //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit + return ids.InGroupsOf(2000).SelectMany(@group => + { + var sql = GetBaseQuery(false) + .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId}) + .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new {ids = @group}); + + sql.OrderBy(x => x.Level, SqlSyntax); + + return Database.Fetch(sql).Select(CreateEntity); + }); + } + else { var sql = GetBaseQuery(false) - .Where("nodeObjectType=@umbracoObjectTypeId", new { umbracoObjectTypeId = NodeObjectTypeId }) - .Where(string.Format("{0} IN (@ids)", SqlSyntax.GetQuotedColumnName("id")), new { ids = @group }); - + .Where("nodeObjectType=@umbracoObjectTypeId", new {umbracoObjectTypeId = NodeObjectTypeId}); sql.OrderBy(x => x.Level, SqlSyntax); - return Database.Fetch(sql).Select(CreateEntity); - }); + } } protected override IEnumerable PerformGetByQuery(IQuery query) diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs index e2555995d2..28e4fbf199 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs @@ -96,5 +96,6 @@ namespace Umbraco.Core.Persistence.Repositories /// An Enumerable list of objects IEnumerable GetPagedResultsByQuery(IQuery query, long pageIndex, int pageSize, out long totalRecords, string orderBy, Direction orderDirection, bool orderBySystemField, IQuery filter = null); + } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs index 64989f9269..9e8890a37b 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs @@ -38,15 +38,6 @@ namespace Umbraco.Core.Persistence.Repositories /// An Enumerable list of objects IEnumerable GetPagedResultsByQuery(IQuery query, long pageIndex, int pageSize, out long totalRecords, string orderBy, Direction orderDirection, bool orderBySystemField, string filter = ""); - - /// - /// Gets paged media descendants as XML by path - /// - /// Path starts with - /// Page number - /// Page size - /// Total records the query would return without paging - /// A paged enumerable of XML entries of media items - IEnumerable GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords); + } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMemberRepository.cs index a24116f0e2..9528fd3d76 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMemberRepository.cs @@ -73,6 +73,6 @@ namespace Umbraco.Core.Persistence.Repositories /// /// void AddOrUpdatePreviewXml(IMember content, Func xml); - + } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs index 3e05d1feaf..b318223ca7 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs @@ -67,5 +67,16 @@ namespace Umbraco.Core.Persistence.Repositories /// Id of the object to delete versions from /// Latest version date void DeleteVersions(int id, DateTime versionDate); + + /// + /// Gets paged content descendants as XML by path + /// + /// Path starts with + /// Page number + /// Page size + /// + /// Total records the query would return without paging + /// A paged enumerable of XML entries of content items + IEnumerable GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords); } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index 08c2f7e0be..2687848fa5 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -502,30 +502,6 @@ namespace Umbraco.Core.Persistence.Repositories } - /// - /// Gets paged media descendants as XML by path - /// - /// Path starts with - /// Page number - /// Page size - /// Total records the query would return without paging - /// A paged enumerable of XML entries of media items - public IEnumerable GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, out long totalRecords) - { - Sql query; - if (path == "-1") - { - query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE nodeObjectType = @0)", Guid.Parse(Constants.ObjectTypes.Media)).OrderBy("nodeId"); - } - else - { - query = new Sql().Select("nodeId, xml").From("cmsContentXml").Where("nodeId IN (SELECT id FROM umbracoNode WHERE path LIKE @0)", path.EnsureEndsWith(",%")).OrderBy("nodeId"); - } - var pagedResult = Database.Page(pageIndex+1, pageSize, query); - totalRecords = pagedResult.TotalItems; - return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml)); - } - /// /// Private method to create a media object from a ContentDto /// diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index dcab898685..a92794775f 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -381,7 +381,7 @@ namespace Umbraco.Core.Persistence.Repositories .Where(GetBaseWhereClause(), new { Id = id }) .OrderByDescending(x => x.VersionDate, SqlSyntax); return ProcessQuery(sql, true); - } + } public void RebuildXmlStructures(Func serializer, int groupSize = 200, IEnumerable contentTypeIds = null) { diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index 163e0810ad..a98f032e1d 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using System.Xml.Linq; using Umbraco.Core.Configuration; using Umbraco.Core.Configuration.UmbracoSettings; using Umbraco.Core.Logging; @@ -138,6 +139,40 @@ namespace Umbraco.Core.Persistence.Repositories #endregion + /// + /// Gets paged document descendants as XML by path + /// + /// Path starts with + /// Page number + /// Page size + /// + /// Total records the query would return without paging + /// A paged enumerable of XML entries of content items + public virtual IEnumerable GetPagedXmlEntriesByPath(string path, long pageIndex, int pageSize, string[] orderBy, out long totalRecords) + { + var query = new Sql().Select(string.Format("umbracoNode.id, cmsContentXml.{0}", SqlSyntax.GetQuotedColumnName("xml"))) + .From("umbracoNode") + .InnerJoin("cmsContentXml").On("cmsContentXml.nodeId = umbracoNode.id"); + + if (path == "-1") + { + query.Where("umbracoNode.nodeObjectType = @type", new { type = NodeObjectTypeId }); + } + else + { + query.Where(string.Format("umbracoNode.{0} LIKE (@0)", SqlSyntax.GetQuotedColumnName("path")), path.EnsureEndsWith(",%")); + } + + //each order by param needs to be in a bracket! see: https://github.com/toptensoftware/PetaPoco/issues/177 + query.OrderBy(orderBy == null + ? "(umbracoNode.id)" + : string.Join(",", orderBy.Select(x => string.Format("({0})", SqlSyntax.GetQuotedColumnName(x))))); + + var pagedResult = Database.Page(pageIndex + 1, pageSize, query); + totalRecords = pagedResult.TotalItems; + return pagedResult.Items.Select(dto => XElement.Parse(dto.Xml)); + } + public int CountDescendants(int parentId, string contentTypeAlias = null) { var pathMatch = parentId == -1 diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index 0838a0eb85..6f2007912f 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -1764,6 +1764,31 @@ namespace Umbraco.Core.Services return true; } + /// + /// Gets paged content descendants as XML by path + /// + /// Path starts with + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of content items + public IEnumerable GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords) + { + Mandate.ParameterCondition(pageIndex >= 0, "pageIndex"); + Mandate.ParameterCondition(pageSize > 0, "pageSize"); + + var uow = UowProvider.GetUnitOfWork(); + using (var repository = RepositoryFactory.CreateContentRepository(uow)) + { + var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, + //This order by is VERY important! This allows us to figure out what is implicitly not published, see ContentRepository.BuildXmlCache and + // UmbracoContentIndexer.PerformIndexAll which uses the logic based on this sort order + new[] {"level", "parentID", "sortOrder"}, + out totalRecords); + return contents; + } + } + /// /// This builds the Xml document used for the XML cache /// diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index ec5db8c4fb..7722bf9c65 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; using System.Xml; +using System.Xml.Linq; using Umbraco.Core.Models; using Umbraco.Core.Models.Membership; using Umbraco.Core.Persistence.DatabaseModelDefinitions; @@ -95,6 +96,19 @@ namespace Umbraco.Core.Services /// public interface IContentService : IService { + /// + /// Gets all XML entries found in the cmsContentXml table based on the given path + /// + /// Path starts with + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of content items + /// + /// If -1 is passed, then this will return all content xml entries, otherwise will return all descendents from the path + /// + IEnumerable GetPagedXmlEntries(string path, long pageIndex, int pageSize, out long totalRecords); + /// /// This builds the Xml document used for the XML cache /// diff --git a/src/Umbraco.Core/Services/IMemberService.cs b/src/Umbraco.Core/Services/IMemberService.cs index a893d89feb..38f53f2d68 100644 --- a/src/Umbraco.Core/Services/IMemberService.cs +++ b/src/Umbraco.Core/Services/IMemberService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Xml.Linq; using Umbraco.Core.Models; using Umbraco.Core.Models.Membership; using Umbraco.Core.Persistence.DatabaseModelDefinitions; @@ -13,6 +14,15 @@ namespace Umbraco.Core.Services /// public interface IMemberService : IMembershipMemberService { + /// + /// Gets all XML entries found in the cmsContentXml table + /// + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of content items + IEnumerable GetPagedXmlEntries(long pageIndex, int pageSize, out long totalRecords); + /// /// Rebuilds all xml content in the cmsContentXml table for all documents /// diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index 7c20d64739..03a155c74e 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -1228,7 +1228,7 @@ namespace Umbraco.Core.Services var uow = UowProvider.GetUnitOfWork(); using (var repository = RepositoryFactory.CreateMediaRepository(uow)) { - var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, out totalRecords); + var contents = repository.GetPagedXmlEntriesByPath(path, pageIndex, pageSize, null, out totalRecords); return contents; } } diff --git a/src/Umbraco.Core/Services/MemberService.cs b/src/Umbraco.Core/Services/MemberService.cs index 2ea4f1f023..70d45926fc 100644 --- a/src/Umbraco.Core/Services/MemberService.cs +++ b/src/Umbraco.Core/Services/MemberService.cs @@ -612,6 +612,26 @@ namespace Umbraco.Core.Services Audit(AuditType.Publish, "MemberService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); } + /// + /// Gets paged member descendants as XML by path + /// + /// Page number + /// Page size + /// Total records the query would return without paging + /// A paged enumerable of XML entries of member items + public IEnumerable GetPagedXmlEntries(long pageIndex, int pageSize, out long totalRecords) + { + Mandate.ParameterCondition(pageIndex >= 0, "pageIndex"); + Mandate.ParameterCondition(pageSize > 0, "pageSize"); + + var uow = UowProvider.GetUnitOfWork(); + using (var repository = RepositoryFactory.CreateMemberRepository(uow)) + { + var contents = repository.GetPagedXmlEntriesByPath("-1", pageIndex, pageSize, null, out totalRecords); + return contents; + } + } + #endregion #region IMembershipMemberService Implementation diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentTypeRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentTypeRepositoryTest.cs index 84cdef73e1..699e563407 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/ContentTypeRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/ContentTypeRepositoryTest.cs @@ -172,6 +172,38 @@ namespace Umbraco.Tests.Persistence.Repositories } } + [Test] + public void Can_Get_All_Containers() + { + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + EntityContainer container1, container2, container3; + using (var containerRepository = CreateContainerRepository(unitOfWork, Constants.ObjectTypes.DocumentTypeContainerGuid)) + { + container1 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container1" }; + containerRepository.AddOrUpdate(container1); + container2 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container2" }; + containerRepository.AddOrUpdate(container2); + container3 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid) { Name = "container3" }; + containerRepository.AddOrUpdate(container3); + unitOfWork.Commit(); + Assert.That(container1.Id, Is.GreaterThan(0)); + Assert.That(container2.Id, Is.GreaterThan(0)); + Assert.That(container3.Id, Is.GreaterThan(0)); + } + using (var containerRepository = CreateContainerRepository(unitOfWork, Constants.ObjectTypes.DocumentTypeContainerGuid)) + { + var found1 = containerRepository.Get(container1.Id); + Assert.IsNotNull(found1); + var found2 = containerRepository.Get(container2.Id); + Assert.IsNotNull(found2); + var found3 = containerRepository.Get(container3.Id); + Assert.IsNotNull(found3); + var allContainers = containerRepository.GetAll(); + Assert.AreEqual(3, allContainers.Count()); + } + } + [Test] public void Can_Delete_Container() { diff --git a/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs index 5bf6a4edc5..f48887498e 100644 --- a/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs @@ -1,6 +1,8 @@ using System; using System.Linq; +using Lucene.Net.Analysis.Standard; using Lucene.Net.Documents; +using Lucene.Net.Index; using Lucene.Net.Store; using Moq; using NUnit.Framework; @@ -31,12 +33,15 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ensure_Children_Are_Sorted() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile()); Assert.IsNotNull(result); Assert.AreEqual(1, result.TotalItemCount); @@ -60,12 +65,15 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ensure_Result_Has_All_Values() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile()); Assert.IsNotNull(result); Assert.AreEqual(1, result.TotalItemCount); diff --git a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs index 6ec347cc0f..eadc08d834 100644 --- a/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs +++ b/src/Umbraco.Tests/PublishedContent/PublishedMediaTests.cs @@ -27,6 +27,7 @@ using UmbracoExamine; using UmbracoExamine.DataServices; using umbraco.BusinessLogic; using System.Linq; +using Lucene.Net.Index; namespace Umbraco.Tests.PublishedContent { @@ -106,11 +107,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ensure_Children_Sorted_With_Examine() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var ctx = GetUmbracoContext("/test", 1234); var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx); @@ -135,11 +139,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Do_Not_Find_In_Recycle_Bin() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var ctx = GetUmbracoContext("/test", 1234); var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx); @@ -175,11 +182,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Children_With_Examine() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var ctx = GetUmbracoContext("/test", 1234); var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx); @@ -197,11 +207,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Descendants_With_Examine() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var ctx = GetUmbracoContext("/test", 1234); var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx); @@ -219,11 +232,14 @@ namespace Umbraco.Tests.PublishedContent [Test] public void DescendantsOrSelf_With_Examine() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var ctx = GetUmbracoContext("/test", 1234); var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx); @@ -241,12 +257,15 @@ namespace Umbraco.Tests.PublishedContent [Test] public void Ancestors_With_Examine() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); var ctx = GetUmbracoContext("/test", 1234); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx); //we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace @@ -260,12 +279,15 @@ namespace Umbraco.Tests.PublishedContent [Test] public void AncestorsOrSelf_With_Examine() { - using (var luceneDir = new RAMDirectory()) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); + //var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir); indexer.RebuildIndex(); var ctx = GetUmbracoContext("/test", 1234); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + //var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx); //we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceTests.cs index 3fd0e1f6cd..78f4a5dad9 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceTests.cs @@ -304,6 +304,42 @@ namespace Umbraco.Tests.Services Assert.IsNull(deletedContentType); } + [Test] + public void Can_Create_Container() + { + // Arrange + var cts = ServiceContext.ContentTypeService; + + // Act + var container = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid); + container.Name = "container1"; + cts.SaveContentTypeContainer(container); + + // Assert + var createdContainer = cts.GetContentTypeContainer(container.Id); + Assert.IsNotNull(createdContainer); + } + + [Test] + public void Can_Get_All_Containers() + { + // Arrange + var cts = ServiceContext.ContentTypeService; + + // Act + var container1 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid); + container1.Name = "container1"; + cts.SaveContentTypeContainer(container1); + + var container2 = new EntityContainer(Constants.ObjectTypes.DocumentTypeGuid); + container2.Name = "container2"; + cts.SaveContentTypeContainer(container2); + + // Assert + var containers = cts.GetContentTypeContainers(new int[0]); + Assert.AreEqual(2, containers.Count()); + } + [Test] public void Deleting_ContentType_Sends_Correct_Number_Of_DeletedEntities_In_Events() { diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 6c1c7797d8..87fca57446 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -58,8 +58,9 @@ ..\packages\AutoMapper.3.3.1\lib\net40\AutoMapper.Net4.dll - - ..\packages\Examine.0.1.70.0\lib\Examine.dll + + ..\packages\Examine.0.1.80\lib\net45\Examine.dll + True ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll @@ -170,6 +171,7 @@ + diff --git a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs index 3e7377f3b6..34c7a1f6ac 100644 --- a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs @@ -1,6 +1,8 @@ using System; using System.Linq; using Examine; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Index; using Lucene.Net.Store; using NUnit.Framework; using Umbraco.Tests.TestHelpers; @@ -15,53 +17,39 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Events_Ignoring_Node() { - //change the parent id so that they are all ignored - var existingCriteria = _indexer.IndexerData; - _indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes, - 999); //change to 999 + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) + { + //change the parent id so that they are all ignored + var existingCriteria = indexer.IndexerData; + indexer.IndexerData = new IndexCriteria(existingCriteria.StandardFields, existingCriteria.UserFields, existingCriteria.IncludeNodeTypes, existingCriteria.ExcludeNodeTypes, + 999); //change to 999 - var isIgnored = false; + var isIgnored = false; - EventHandler ignoringNode = (s, e) => - { - isIgnored = true; - }; + EventHandler ignoringNode = (s, e) => + { + isIgnored = true; + }; - _indexer.IgnoringNode += ignoringNode; + indexer.IgnoringNode += ignoringNode; - //get a node from the data repo - var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") - .Root - .Elements() - .First(); + //get a node from the data repo + var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") + .Root + .Elements() + .First(); - _indexer.ReIndexNode(node, IndexTypes.Content); + indexer.ReIndexNode(node, IndexTypes.Content); - Assert.IsTrue(isIgnored); - + Assert.IsTrue(isIgnored); + } } private readonly TestContentService _contentService = new TestContentService(); - private static UmbracoExamineSearcher _searcher; - private static UmbracoContentIndexer _indexer; - private Lucene.Net.Store.Directory _luceneDir; - - public override void Initialize() - { - base.Initialize(); - - _luceneDir = new RAMDirectory(); - _indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir); - _indexer.RebuildIndex(); - _searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir); - } - - public override void TearDown() - { - base.TearDown(); - _luceneDir.Dispose(); - } } } \ No newline at end of file diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs index f57b1af213..c7922b9d58 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs @@ -7,6 +7,7 @@ using Examine.LuceneEngine.Config; using Examine.LuceneEngine.Providers; using Lucene.Net.Analysis; using Lucene.Net.Analysis.Standard; +using Lucene.Net.Index; using Lucene.Net.Store; using Moq; using Umbraco.Core; @@ -33,7 +34,7 @@ namespace Umbraco.Tests.UmbracoExamine internal static class IndexInitializer { public static UmbracoContentIndexer GetUmbracoIndexer( - Directory luceneDir, + IndexWriter writer, Analyzer analyzer = null, IDataService dataService = null, IContentService contentService = null, @@ -178,15 +179,14 @@ namespace Umbraco.Tests.UmbracoExamine var indexCriteria = indexSet.ToIndexCriteria(dataService, UmbracoContentIndexer.IndexFieldPolicies); var i = new UmbracoContentIndexer(indexCriteria, - luceneDir, //custom lucene directory - dataService, - contentService, - mediaService, - dataTypeService, - userService, - contentTypeService, - analyzer, - false) + writer, + dataService, + contentService, + mediaService, + dataTypeService, + userService, + contentTypeService, + false) { SupportUnpublishedContent = supportUnpublishedContent }; @@ -197,13 +197,14 @@ namespace Umbraco.Tests.UmbracoExamine return i; } - public static UmbracoExamineSearcher GetUmbracoSearcher(Directory luceneDir, Analyzer analyzer = null) + + public static UmbracoExamineSearcher GetUmbracoSearcher(IndexWriter writer, Analyzer analyzer = null) { if (analyzer == null) { analyzer = new StandardAnalyzer(Version.LUCENE_29); } - return new UmbracoExamineSearcher(luceneDir, analyzer); + return new UmbracoExamineSearcher(writer, analyzer); } public static LuceneSearcher GetLuceneSearcher(Directory luceneDir) diff --git a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs index 5f3f5525c1..3f324bb2ed 100644 --- a/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/IndexTest.cs @@ -1,11 +1,11 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using Examine; using Examine.LuceneEngine; using Examine.LuceneEngine.Providers; using Examine.LuceneEngine.SearchCriteria; using Examine.SearchCriteria; +using Lucene.Net.Analysis.Standard; using Lucene.Net.Index; using Lucene.Net.Search; using Lucene.Net.Store; @@ -15,7 +15,6 @@ using UmbracoExamine; namespace Umbraco.Tests.UmbracoExamine { - /// /// Tests the standard indexing capabilities /// @@ -30,9 +29,10 @@ namespace Umbraco.Tests.UmbracoExamine public void Index_Protected_Content_Not_Indexed() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir)) - using (var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir)) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { indexer.RebuildIndex(); @@ -59,9 +59,10 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Move_Media_From_Non_Indexable_To_Indexable_ParentID() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir)) - using (var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir)) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { indexer.RebuildIndex(); @@ -77,7 +78,7 @@ namespace Umbraco.Tests.UmbracoExamine //ensure that node 2112 doesn't exist var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); - Assert.AreEqual(0, results.Count()); + Assert.AreEqual(0, results.TotalItemCount); //get a node from the data repo (this one exists underneath 2222) var node = mediaService.GetLatestMediaByXpath("//*[string-length(@id)>0 and number(@id)>0]") @@ -103,19 +104,19 @@ namespace Umbraco.Tests.UmbracoExamine //now ensure it's deleted var newResults = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); - Assert.AreEqual(1, newResults.Count()); + Assert.AreEqual(1, newResults.TotalItemCount); } } [Test] - [Ignore] public void Index_Move_Media_To_Non_Indexable_ParentID() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir)) - using (var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir)) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { indexer.RebuildIndex(); @@ -152,7 +153,7 @@ namespace Umbraco.Tests.UmbracoExamine //now ensure it's deleted var results = searcher.Search(searcher.CreateSearchCriteria().Id(2112).Compile()); - Assert.AreEqual(0, results.Count()); + Assert.AreEqual(0, results.TotalItemCount); } } @@ -164,9 +165,10 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Index_Reindex_Content() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir, supportUnpublishedContent:true)) - using (var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir)) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer, supportUnpublishedContent: true)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { indexer.RebuildIndex(); @@ -175,10 +177,9 @@ namespace Umbraco.Tests.UmbracoExamine //first delete all 'Content' (not media). This is done by directly manipulating the index with the Lucene API, not examine! var contentTerm = new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content); - var writer = indexer.GetIndexWriter(); writer.DeleteDocuments(contentTerm); writer.Commit(); - + //make sure the content is gone. This is done with lucene APIs, not examine! var collector = new AllHitsCollector(false, true); var query = new TermQuery(contentTerm); @@ -205,13 +206,13 @@ namespace Umbraco.Tests.UmbracoExamine /// This will delete an item from the index and ensure that all children of the node are deleted too! /// [Test] - [Ignore] public void Index_Delete_Index_Item_Ensure_Heirarchy_Removed() { - using (var luceneDir = new RAMDirectory()) - using (var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir)) - using (var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir)) + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) { indexer.RebuildIndex(); @@ -221,10 +222,10 @@ namespace Umbraco.Tests.UmbracoExamine //this node had children: 1141 & 1142, let's ensure they are also removed var results = searcher.Search(searcher.CreateSearchCriteria().Id(1141).Compile()); - Assert.AreEqual(0, results.Count()); + Assert.AreEqual(0, results.TotalItemCount); results = searcher.Search(searcher.CreateSearchCriteria().Id(1142).Compile()); - Assert.AreEqual(0, results.Count()); + Assert.AreEqual(0, results.TotalItemCount); } } diff --git a/src/Umbraco.Tests/UmbracoExamine/RandomIdRAMDirectory.cs b/src/Umbraco.Tests/UmbracoExamine/RandomIdRAMDirectory.cs new file mode 100644 index 0000000000..34b69274c8 --- /dev/null +++ b/src/Umbraco.Tests/UmbracoExamine/RandomIdRAMDirectory.cs @@ -0,0 +1,14 @@ +using System; +using Lucene.Net.Store; + +namespace Umbraco.Tests.UmbracoExamine +{ + public class RandomIdRAMDirectory : RAMDirectory + { + private readonly string _lockId = Guid.NewGuid().ToString(); + public override string GetLockID() + { + return _lockId; + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs index 7eb92ad49d..f739286f98 100644 --- a/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs +++ b/src/Umbraco.Tests/UmbracoExamine/SearchTests.cs @@ -8,6 +8,8 @@ using Examine.LuceneEngine.Providers; using Lucene.Net.Store; using NUnit.Framework; using Examine.LuceneEngine.SearchCriteria; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Index; using Umbraco.Tests.TestHelpers; namespace Umbraco.Tests.UmbracoExamine @@ -20,16 +22,17 @@ namespace Umbraco.Tests.UmbracoExamine [Test] public void Test_Sort_Order_Sorting() { - using (var luceneDir = new RAMDirectory()) - { - var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir, null, + using (var luceneDir = new RandomIdRAMDirectory()) + using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED)) + using (var indexer = IndexInitializer.GetUmbracoIndexer(writer, null, new TestDataService() - { - ContentService = new TestContentService(TestFiles.umbraco_sort) - }, - supportUnpublishedContent:true); - indexer.RebuildIndex(); - var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir); + { + ContentService = new TestContentService(TestFiles.umbraco_sort) + }, + supportUnpublishedContent: true)) + using (var searcher = IndexInitializer.GetUmbracoSearcher(writer)) + { + indexer.RebuildIndex(); var s = (LuceneSearcher)searcher; var luceneSearcher = s.GetSearcher(); diff --git a/src/Umbraco.Tests/packages.config b/src/Umbraco.Tests/packages.config index 6428eebadf..b71ba7a170 100644 --- a/src/Umbraco.Tests/packages.config +++ b/src/Umbraco.Tests/packages.config @@ -2,7 +2,7 @@ - + diff --git a/src/Umbraco.Web.UI.Client/src/init.js b/src/Umbraco.Web.UI.Client/src/init.js index ded5ba08e4..3233974cee 100644 --- a/src/Umbraco.Web.UI.Client/src/init.js +++ b/src/Umbraco.Web.UI.Client/src/init.js @@ -1,6 +1,6 @@ /** Executed when the application starts, binds to events and set global state */ -app.run(['userService', '$log', '$rootScope', '$location', 'navigationService', 'appState', 'editorState', 'fileManager', 'assetsService', 'eventsService', '$cookies', '$templateCache', - function (userService, $log, $rootScope, $location, navigationService, appState, editorState, fileManager, assetsService, eventsService, $cookies, $templateCache) { +app.run(['userService', '$log', '$rootScope', '$location', 'navigationService', 'appState', 'editorState', 'fileManager', 'assetsService', 'eventsService', '$cookies', '$templateCache', 'localStorageService', + function (userService, $log, $rootScope, $location, navigationService, appState, editorState, fileManager, assetsService, eventsService, $cookies, $templateCache, localStorageService) { //This sets the default jquery ajax headers to include our csrf token, we // need to user the beforeSend method because our token changes per user/login so @@ -13,6 +13,11 @@ app.run(['userService', '$log', '$rootScope', '$location', 'navigationService', /** Listens for authentication and checks if our required assets are loaded, if/once they are we'll broadcast a ready event */ eventsService.on("app.authenticated", function(evt, data) { + + //Removes all stored LocalStorage browser items - that may contain sensitive data + //So if a machine or computer is shared and a new user logs in, we clear out the previous persons localStorage items + localStorageService.clearAll(); + assetsService._loadInitAssets().then(function() { appState.setGlobalState("isReady", true); diff --git a/src/Umbraco.Web.UI.Client/src/views/common/overlays/mediaPicker/mediapicker.controller.js b/src/Umbraco.Web.UI.Client/src/views/common/overlays/mediaPicker/mediapicker.controller.js index 43c65d99ee..3b6c39e84a 100644 --- a/src/Umbraco.Web.UI.Client/src/views/common/overlays/mediaPicker/mediapicker.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/common/overlays/mediaPicker/mediapicker.controller.js @@ -1,7 +1,7 @@ //used for the media picker dialog angular.module("umbraco") .controller("Umbraco.Overlays.MediaPickerController", - function($scope, mediaResource, umbRequestHelper, entityResource, $log, mediaHelper, mediaTypeHelper, eventsService, treeService, $element, $timeout, $cookies, $cookieStore, localizationService) { + function ($scope, mediaResource, umbRequestHelper, entityResource, $log, mediaHelper, mediaTypeHelper, eventsService, treeService, $element, $timeout, $cookies, localStorageService, localizationService) { if (!$scope.model.title) { $scope.model.title = localizationService.localize("defaultdialogs_selectMedia"); @@ -15,7 +15,7 @@ angular.module("umbraco") $scope.multiPicker = (dialogOptions.multiPicker && dialogOptions.multiPicker !== "0") ? true : false; $scope.startNodeId = dialogOptions.startNodeId ? dialogOptions.startNodeId : -1; $scope.cropSize = dialogOptions.cropSize; - $scope.lastOpenedNode = $cookieStore.get("umbLastOpenedMediaNodeId"); + $scope.lastOpenedNode = localStorageService.get("umbLastOpenedMediaNodeId"); if ($scope.onlyImages) { $scope.acceptedFileTypes = mediaHelper .formatFileTypes(Umbraco.Sys.ServerVariables.umbracoSettings.imageFileTypes); @@ -133,8 +133,7 @@ angular.module("umbraco") }); $scope.currentFolder = folder; - // for some reason i cannot set cookies with cookieStore - document.cookie = "umbLastOpenedMediaNodeId=" + folder.id; + localStorageService.set("umbLastOpenedMediaNodeId", folder.id); }; diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/googlemaps/googlemaps.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/googlemaps/googlemaps.controller.js index f8e89d7200..1d6261c10a 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/googlemaps/googlemaps.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/googlemaps/googlemaps.controller.js @@ -2,7 +2,7 @@ angular.module("umbraco") .controller("Umbraco.PropertyEditors.GoogleMapsController", function ($element, $rootScope, $scope, notificationsService, dialogService, assetsService, $log, $timeout) { - assetsService.loadJs('http://www.google.com/jsapi') + assetsService.loadJs('https://www.google.com/jsapi') .then(function () { google.load("maps", "3", { diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 27e437d8bf..81df39e51b 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -127,8 +127,8 @@ False ..\packages\dotless.1.4.1.0\lib\dotless.Core.dll - - ..\packages\Examine.0.1.70.0\lib\Examine.dll + + ..\packages\Examine.0.1.80\lib\net45\Examine.dll True @@ -138,8 +138,9 @@ ..\packages\ImageProcessor.2.5.1\lib\net45\ImageProcessor.dll - - ..\packages\ImageProcessor.Web.4.7.2\lib\net45\ImageProcessor.Web.dll + + ..\packages\ImageProcessor.Web.4.8.0\lib\net45\ImageProcessor.Web.dll + True False diff --git a/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx b/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx index 59a96e9a37..7728d281c3 100644 --- a/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx +++ b/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx @@ -13,6 +13,7 @@ + diff --git a/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx.designer.cs b/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx.designer.cs index 19ac019a8f..628e82a767 100644 --- a/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx.designer.cs +++ b/src/Umbraco.Web.UI/Umbraco/dashboard/UserControlProxy.aspx.designer.cs @@ -48,6 +48,15 @@ namespace Umbraco.Web.UI.Umbraco.Dashboard { /// protected global::ClientDependency.Core.Controls.JsInclude JsInclude3; + /// + /// JsInclude4 control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::ClientDependency.Core.Controls.JsInclude JsInclude4; + /// /// JsInclude6 control. /// diff --git a/src/Umbraco.Web.UI/packages.config b/src/Umbraco.Web.UI/packages.config index 6919e10f64..629438762f 100644 --- a/src/Umbraco.Web.UI/packages.config +++ b/src/Umbraco.Web.UI/packages.config @@ -4,9 +4,9 @@ - + - + diff --git a/src/Umbraco.Web/Editors/BackOfficeController.cs b/src/Umbraco.Web/Editors/BackOfficeController.cs index c092811a08..4f19f73c58 100644 --- a/src/Umbraco.Web/Editors/BackOfficeController.cs +++ b/src/Umbraco.Web/Editors/BackOfficeController.cs @@ -262,7 +262,7 @@ namespace Umbraco.Web.Editors }, { "mediaTypeApiBaseUrl", Url.GetUmbracoApiServiceBaseUrl( - controller => controller.GetAllowedChildren(0)) + controller => controller.GetAllowedChildren("0")) }, { "macroApiBaseUrl", Url.GetUmbracoApiServiceBaseUrl( diff --git a/src/Umbraco.Web/Editors/EntityController.cs b/src/Umbraco.Web/Editors/EntityController.cs index 9f8b15449c..104c451273 100644 --- a/src/Umbraco.Web/Editors/EntityController.cs +++ b/src/Umbraco.Web/Editors/EntityController.cs @@ -656,7 +656,7 @@ namespace Umbraco.Web.Editors .Select(Mapper.Map); // entities are in "some" order, put them back in order - var xref = entities.ToDictionary(x => x.Id); + var xref = entities.ToDictionary(x => x.Key); var result = keysArray.Select(x => xref.ContainsKey(x) ? xref[x] : null).Where(x => x != null); return result; diff --git a/src/Umbraco.Web/Editors/MediaController.cs b/src/Umbraco.Web/Editors/MediaController.cs index fb17d9462e..2ed01c0192 100644 --- a/src/Umbraco.Web/Editors/MediaController.cs +++ b/src/Umbraco.Web/Editors/MediaController.cs @@ -36,6 +36,7 @@ using Umbraco.Core.Configuration; using Umbraco.Core.Persistence.FaultHandling; using Umbraco.Web.UI; using Notification = Umbraco.Web.Models.ContentEditing.Notification; +using Umbraco.Core.Persistence; namespace Umbraco.Web.Editors { @@ -175,7 +176,37 @@ namespace Umbraco.Web.Editors /// Returns the child media objects /// [FilterAllowedOutgoingMedia(typeof(IEnumerable>), "Items")] - public PagedResult> GetChildren(int id, + public PagedResult> GetChildren(string id, + int pageNumber = 0, + int pageSize = 0, + string orderBy = "SortOrder", + Direction orderDirection = Direction.Ascending, + bool orderBySystemField = true, + string filter = "") + { + int idInt; Guid idGuid; + + if (Guid.TryParse(id, out idGuid)) + { + var entity = Services.EntityService.GetByKey(idGuid); + if (entity != null) + { + return GetChildren(entity.Id, pageNumber, pageSize, orderBy, orderDirection, orderBySystemField, filter); + } + else + { + throw new HttpResponseException(HttpStatusCode.NotFound); + } + } + else if (int.TryParse(id, out idInt)) + { + return GetChildren(idInt, pageNumber, pageSize, orderBy, orderDirection, orderBySystemField, filter); + } + + throw new HttpResponseException(HttpStatusCode.NotFound); + } + + private PagedResult> GetChildren(int id, int pageNumber = 0, int pageSize = 0, string orderBy = "SortOrder", @@ -448,12 +479,37 @@ namespace Umbraco.Web.Editors } //get the string json from the request - int parentId; - if (int.TryParse(result.FormData["currentFolder"], out parentId) == false) + int parentId; bool entityFound; + string currentFolderId = result.FormData["currentFolder"]; + if (int.TryParse(currentFolderId, out parentId) == false) { - return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer"); + // if a guid then try to look up the entity + Guid idGuid; + if (Guid.TryParse(currentFolderId, out idGuid)) + { + var entity = Services.EntityService.GetByKey(idGuid); + if (entity != null) + { + entityFound = true; + parentId = entity.Id; + } + else + { + throw new EntityNotFoundException(currentFolderId, "The passed id doesn't exist"); + } + } + else + { + return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer or Guid"); + } + + if (entityFound == false) + { + return Request.CreateValidationErrorResponse("The request was not formatted correctly, the currentFolder is not an integer or Guid"); + } } + //ensure the user has access to this folder by parent id! if (CheckPermissions( new Dictionary(), diff --git a/src/Umbraco.Web/Editors/MediaTypeController.cs b/src/Umbraco.Web/Editors/MediaTypeController.cs index 0f83ad6d03..67b8ae6d1f 100644 --- a/src/Umbraco.Web/Editors/MediaTypeController.cs +++ b/src/Umbraco.Web/Editors/MediaTypeController.cs @@ -11,6 +11,8 @@ using System.Net; using System.Net.Http; using Umbraco.Web.WebApi; using Umbraco.Core.Services; +using Umbraco.Core.Models.EntityBase; +using System; namespace Umbraco.Web.Editors { @@ -174,7 +176,22 @@ namespace Umbraco.Web.Editors /// /// [UmbracoTreeAuthorize(Constants.Trees.MediaTypes, Constants.Trees.Media)] - public IEnumerable GetAllowedChildren(int contentId) + public IEnumerable GetAllowedChildren(string contentId) + { + Guid idGuid = Guid.Empty; + int idInt; + if (Guid.TryParse(contentId, out idGuid)) { + var entity = ApplicationContext.Services.EntityService.GetByKey(idGuid); + return GetAllowedChildrenInternal(entity.Id); + } else if (int.TryParse(contentId, out idInt)) + { + return GetAllowedChildrenInternal(idInt); + } + + throw new HttpResponseException(HttpStatusCode.NotFound); + } + + private IEnumerable GetAllowedChildrenInternal(int contentId) { if (contentId == Constants.System.RecycleBinContent) return Enumerable.Empty(); diff --git a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs index b40cf3bbf3..db9792572f 100644 --- a/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs +++ b/src/Umbraco.Web/PropertyEditors/GridPropertyEditor.cs @@ -92,7 +92,12 @@ namespace Umbraco.Web.PropertyEditors //swallow...on purpose, there's a chance that this isn't json and we don't want that to affect // the website. } + catch (ArgumentException) + { + //swallow on purpose to prevent this error: + // Can not add Newtonsoft.Json.Linq.JValue to Newtonsoft.Json.Linq.JObject. + } } } } diff --git a/src/Umbraco.Web/Trees/ContentTreeController.cs b/src/Umbraco.Web/Trees/ContentTreeController.cs index 9f33a44ea9..9a5f90ff89 100644 --- a/src/Umbraco.Web/Trees/ContentTreeController.cs +++ b/src/Umbraco.Web/Trees/ContentTreeController.cs @@ -202,11 +202,13 @@ namespace Umbraco.Web.Trees /// protected override bool HasPathAccess(string id, FormDataCollection queryStrings) { - var content = Services.ContentService.GetById(int.Parse(id)); - if (content == null) + var entity = GetEntityFromId(id); + if (entity == null) { return false; } + + IContent content = Services.ContentService.GetById(entity.Id); return Security.CurrentUser.HasPathAccess(content); } diff --git a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs index 3708c6fd4f..027e4e488c 100644 --- a/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs +++ b/src/Umbraco.Web/Trees/ContentTreeControllerBase.cs @@ -96,7 +96,7 @@ namespace Umbraco.Web.Trees LogHelper.Warn("The user " + Security.CurrentUser.Username + " does not have access to the tree node " + id); return new TreeNodeCollection(); } - + // So there's an alt id specified, it's not the root node and the user has access to it, great! But there's one thing we // need to consider: // If the tree is being rendered in a dialog view we want to render only the children of the specified id, but @@ -110,7 +110,7 @@ namespace Umbraco.Web.Trees id = Constants.System.Root.ToString(CultureInfo.InvariantCulture); } } - + var entities = GetChildEntities(id); nodes.AddRange(entities.Select(entity => GetSingleTreeNode(entity, id, queryStrings)).Where(node => node != null)); return nodes; @@ -122,11 +122,24 @@ namespace Umbraco.Web.Trees protected IEnumerable GetChildEntities(string id) { + // use helper method to ensure we support both integer and guid lookups int iid; - if (int.TryParse(id, out iid) == false) + + // if it's the root node, we won't use the look up + if (id != "-1") { - throw new InvalidCastException("The id for the media tree must be an integer"); + var idEntity = GetEntityFromId(id); + if (idEntity == null) + { + throw new HttpResponseException(HttpStatusCode.NotFound); + } + iid = idEntity.Id; } + else + { + iid = int.Parse(id); + } + //if a request is made for the root node data but the user's start node is not the default, then // we need to return their start node data @@ -134,11 +147,12 @@ namespace Umbraco.Web.Trees { //just return their single start node, it will show up under the 'Content' label var startNode = Services.EntityService.Get(UserStartNode, UmbracoObjectType); - if (startNode == null) + if (startNode != null) + return new[] { startNode }; + else { throw new EntityNotFoundException(UserStartNode, "User's start content node could not be found"); } - return new[] { startNode }; } return Services.EntityService.GetChildren(iid, UmbracoObjectType).ToArray(); @@ -176,9 +190,9 @@ namespace Umbraco.Web.Trees { id = altStartId; } - + var nodes = GetTreeNodesInternal(id, queryStrings); - + //only render the recycle bin if we are not in dialog and the start id id still the root if (IsDialog(queryStrings) == false && id == Constants.System.Root.ToInvariantString()) { @@ -210,8 +224,9 @@ namespace Umbraco.Web.Trees /// private TreeNodeCollection GetTreeNodesInternal(string id, FormDataCollection queryStrings) { + IUmbracoEntity current = GetEntityFromId(id); + //before we get the children we need to see if this is a container node - var current = Services.EntityService.Get(int.Parse(id), UmbracoObjectType); //test if the parent is a listview / container if (current != null && current.IsContainer()) @@ -286,5 +301,32 @@ namespace Umbraco.Web.Trees { return allowedUserOptions.Select(x => x.Action).OfType().Any(); } + + /// + /// Get an entity via an id that can be either an integer or a Guid + /// + /// + /// + internal IUmbracoEntity GetEntityFromId(string id) + { + IUmbracoEntity entity; + Guid idGuid = Guid.Empty; + int idInt; + if (Guid.TryParse(id, out idGuid)) + { + entity = Services.EntityService.GetByKey(idGuid, UmbracoObjectType); + + } + else if (int.TryParse(id, out idInt)) + { + entity = Services.EntityService.Get(idInt, UmbracoObjectType); + } + else + { + return null; + } + + return entity; + } } } \ No newline at end of file diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj index 0a8d253ee5..cb6478be00 100644 --- a/src/Umbraco.Web/Umbraco.Web.csproj +++ b/src/Umbraco.Web/Umbraco.Web.csproj @@ -112,8 +112,9 @@ ..\packages\dotless.1.4.1.0\lib\dotless.Core.dll - - ..\packages\Examine.0.1.70.0\lib\Examine.dll + + ..\packages\Examine.0.1.80\lib\net45\Examine.dll + True ..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll diff --git a/src/Umbraco.Web/packages.config b/src/Umbraco.Web/packages.config index 520a6e4f76..1382e9ceea 100644 --- a/src/Umbraco.Web/packages.config +++ b/src/Umbraco.Web/packages.config @@ -3,7 +3,7 @@ - + diff --git a/src/UmbracoExamine/BaseUmbracoIndexer.cs b/src/UmbracoExamine/BaseUmbracoIndexer.cs index 74b3ff676d..1d87b0dfff 100644 --- a/src/UmbracoExamine/BaseUmbracoIndexer.cs +++ b/src/UmbracoExamine/BaseUmbracoIndexer.cs @@ -55,6 +55,7 @@ namespace UmbracoExamine /// /// /// + /// protected BaseUmbracoIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, Analyzer analyzer, bool async) : base(indexerData, indexPath, analyzer, async) { @@ -67,6 +68,19 @@ namespace UmbracoExamine DataService = dataService; } + /// + /// Creates an NRT indexer + /// + /// + /// + /// + /// + protected BaseUmbracoIndexer(IIndexCriteria indexerData, IndexWriter writer, IDataService dataService, bool async) + : base(indexerData, writer, async) + { + DataService = dataService; + } + #endregion /// diff --git a/src/UmbracoExamine/UmbracoContentIndexer.cs b/src/UmbracoExamine/UmbracoContentIndexer.cs index 447fd62f4c..f7e4c45f50 100644 --- a/src/UmbracoExamine/UmbracoContentIndexer.cs +++ b/src/UmbracoExamine/UmbracoContentIndexer.cs @@ -4,6 +4,7 @@ using System.Collections.Specialized; using System.Diagnostics; using System.IO; using System.Linq; +using System.Xml; using System.Xml.Linq; using Examine; using Lucene.Net.Documents; @@ -16,6 +17,7 @@ using Examine.LuceneEngine; using Examine.LuceneEngine.Config; using UmbracoExamine.Config; using Lucene.Net.Analysis; +using Lucene.Net.Index; using Umbraco.Core.Persistence.Querying; using IContentService = Umbraco.Core.Services.IContentService; using IMediaService = Umbraco.Core.Services.IMediaService; @@ -145,6 +147,34 @@ namespace UmbracoExamine _contentTypeService = contentTypeService; } + /// + /// Creates an NRT indexer + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public UmbracoContentIndexer(IIndexCriteria indexerData, IndexWriter writer, IDataService dataService, + IContentService contentService, + IMediaService mediaService, + IDataTypeService dataTypeService, + IUserService userService, + IContentTypeService contentTypeService, + bool async) + : base(indexerData, writer, dataService, async) + { + _contentService = contentService; + _mediaService = mediaService; + _dataTypeService = dataTypeService; + _userService = userService; + _contentTypeService = contentTypeService; + } + #endregion #region Constants & Fields @@ -230,7 +260,13 @@ namespace UmbracoExamine SupportProtectedContent = supportProtected; else SupportProtectedContent = false; - + + bool disableXmlDocLookup; + if (config["disableXmlDocLookup"] != null && bool.TryParse(config["disableXmlDocLookup"], out disableXmlDocLookup)) + DisableXmlDocumentLookup = disableXmlDocLookup; + else + DisableXmlDocumentLookup = false; + base.Initialize(name, config); } @@ -238,6 +274,11 @@ namespace UmbracoExamine #region Properties + /// + /// Whether to use the cmsContentXml data to re-index when possible (i.e. for published content, media and members) + /// + public bool DisableXmlDocumentLookup { get; private set; } + /// /// By default this is false, if set to true then the indexer will include indexing content that is flagged as publicly protected. /// This property is ignored if SupportUnpublishedContent is set to true. @@ -364,6 +405,9 @@ namespace UmbracoExamine protected override void PerformIndexAll(string type) { + if (SupportedTypes.Contains(type) == false) + return; + const int pageSize = 10000; var pageIndex = 0; @@ -371,111 +415,220 @@ namespace UmbracoExamine var stopwatch = new Stopwatch(); stopwatch.Start(); - switch (type) + try { - case IndexTypes.Content: - var contentParentId = -1; - if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) - { - contentParentId = IndexerData.ParentNodeId.Value; - } - IContent[] content; - - //used to track non-published entities so we can determine what items are implicitly not published - var notPublished = new HashSet(); - - do - { - long total; - - IEnumerable descendants; - if (SupportUnpublishedContent) + switch (type) + { + case IndexTypes.Content: + var contentParentId = -1; + if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) { - descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total); - } - else - { - //get all paged records but order by level ascending, we need to do this because we need to track which nodes are not published so that we can determine - // which descendent nodes are implicitly not published - descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "level", Direction.Ascending, true, (string)null); - } - - //if specific types are declared we need to post filter them - //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (IndexerData.IncludeNodeTypes.Any()) - { - content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray(); - } - else - { - content = descendants.ToArray(); - } - - AddNodesToIndex(GetSerializedContent( - SupportUnpublishedContent, - c => _serializer.Serialize(_contentService, _dataTypeService, _userService, c), - content, notPublished).WhereNotNull(), type); - - pageIndex++; - } while (content.Length == pageSize); - - notPublished.Clear(); - - break; - case IndexTypes.Media: - var mediaParentId = -1; - - if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) - { - mediaParentId = IndexerData.ParentNodeId.Value; - } - - XElement[] mediaXElements; - - var mediaTypes = _contentTypeService.GetAllMediaTypes().ToArray(); - var icons = mediaTypes.ToDictionary(x => x.Id, y => y.Icon); - - do - { - long total; - if (mediaParentId == -1) - { - mediaXElements = _mediaService.GetPagedXmlEntries("-1", pageIndex, pageSize, out total).ToArray(); - } - else - { - //Get the parent - var parent = _mediaService.GetById(mediaParentId); - if (parent == null) - mediaXElements = new XElement[0]; - else - mediaXElements = _mediaService.GetPagedXmlEntries(parent.Path, pageIndex, pageSize, out total).ToArray(); - } - - //if specific types are declared we need to post filter them - //TODO: Update the service layer to join the cmsContentType table so we can query by content type too - if (IndexerData.IncludeNodeTypes.Any()) - { - var includeNodeTypeIds = mediaTypes.Where(x => IndexerData.IncludeNodeTypes.Contains(x.Alias)).Select(x => x.Id); - mediaXElements = mediaXElements.Where(elm => includeNodeTypeIds.Contains(elm.AttributeValue("nodeType"))).ToArray(); + contentParentId = IndexerData.ParentNodeId.Value; } - foreach (var element in mediaXElements) + if (SupportUnpublishedContent == false && DisableXmlDocumentLookup == false) { - element.Add(new XAttribute("icon", icons[element.AttributeValue("nodeType")])); + //get all node Ids that have a published version - this is a fail safe check, in theory + // only document nodes that have a published version would exist in the cmsContentXml table + var allNodesWithPublishedVersions = ApplicationContext.Current.DatabaseContext.Database.Fetch( + "select DISTINCT cmsDocument.nodeId from cmsDocument where cmsDocument.published = 1"); + + XElement last = null; + var trackedIds = new HashSet(); + + ReindexWithXmlEntries(type, contentParentId, + () => _contentTypeService.GetAllContentTypes().ToArray(), + (path, pIndex, pSize) => + { + long totalContent; + + //sorted by: umbracoNode.level, umbracoNode.parentID, umbracoNode.sortOrder + var result = _contentService.GetPagedXmlEntries(path, pIndex, pSize, out totalContent).ToArray(); + + //then like we do in the ContentRepository.BuildXmlCache we need to track what Parents have been processed + // already so that we can then exclude implicitly unpublished content items + var filtered = new List(); + + foreach (var xml in result) + { + var id = xml.AttributeValue("id"); + + //don't include this if it doesn't have a published version + if (allNodesWithPublishedVersions.Contains(id) == false) + continue; + + var parentId = xml.AttributeValue("parentID"); + + if (parentId == null) continue; //this shouldn't happen + + //if the parentid is changing + if (last != null && last.AttributeValue("parentID") != parentId) + { + var found = trackedIds.Contains(parentId); + if (found == false) + { + //Need to short circuit here, if the parent is not there it means that the parent is unpublished + // and therefore the child is not published either so cannot be included in the xml cache + continue; + } + } + + last = xml; + trackedIds.Add(xml.AttributeValue("id")); + + filtered.Add(xml); + } + + return new Tuple(totalContent, filtered.ToArray()); + }, + i => _contentService.GetById(i)); + } + else + { + //used to track non-published entities so we can determine what items are implicitly not published + //currently this is not in use apart form in tests + var notPublished = new HashSet(); + + int currentPageSize; + do + { + long total; + + IContent[] descendants; + if (SupportUnpublishedContent) + { + descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total).ToArray(); + } + else + { + //get all paged records but order by level ascending, we need to do this because we need to track which nodes are not published so that we can determine + // which descendent nodes are implicitly not published + descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "level", Direction.Ascending, true, (string)null).ToArray(); + } + + // need to store decendants count before filtering, in order for loop to work correctly + currentPageSize = descendants.Length; + + //if specific types are declared we need to post filter them + //TODO: Update the service layer to join the cmsContentType table so we can query by content type too + IEnumerable content; + if (IndexerData.IncludeNodeTypes.Any()) + { + content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)); + } + else + { + content = descendants; + } + + AddNodesToIndex(GetSerializedContent( + SupportUnpublishedContent, + c => _serializer.Serialize(_contentService, _dataTypeService, _userService, c), + content, notPublished).WhereNotNull(), type); + + pageIndex++; + } while (currentPageSize == pageSize); } - AddNodesToIndex(mediaXElements, type); - pageIndex++; - } while (mediaXElements.Length == pageSize); + break; + case IndexTypes.Media: + var mediaParentId = -1; - break; + if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0) + { + mediaParentId = IndexerData.ParentNodeId.Value; + } + + ReindexWithXmlEntries(type, mediaParentId, + () => _contentTypeService.GetAllMediaTypes().ToArray(), + (path, pIndex, pSize) => + { + long totalMedia; + var result = _mediaService.GetPagedXmlEntries(path, pIndex, pSize, out totalMedia).ToArray(); + return new Tuple(totalMedia, result); + }, + i => _mediaService.GetById(i)); + + break; + } } - - stopwatch.Stop(); + finally + { + stopwatch.Stop(); + } + DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - End data queries - {0}, took {1}ms", type, stopwatch.ElapsedMilliseconds)); } + /// + /// Performs a reindex of a type based on looking up entries from the cmsContentXml table - but using callbacks to get this data since + /// we don't have a common underlying service interface for the media/content stuff + /// + /// + /// + /// + /// + /// + internal void ReindexWithXmlEntries( + string type, + int parentId, + Func getContentTypes, + Func> getPagedXmlEntries, + Func getContent) + where TContentType: IContentTypeComposition + { + const int pageSize = 10000; + var pageIndex = 0; + + XElement[] xElements; + + var contentTypes = getContentTypes(); + var icons = contentTypes.ToDictionary(x => x.Id, y => y.Icon); + + do + { + long total; + if (parentId == -1) + { + var pagedElements = getPagedXmlEntries("-1", pageIndex, pageSize); + total = pagedElements.Item1; + xElements = pagedElements.Item2; + } + else + { + //Get the parent + var parent = getContent(parentId); + if (parent == null) + xElements = new XElement[0]; + else + { + var pagedElements = getPagedXmlEntries(parent.Path, pageIndex, pageSize); + total = pagedElements.Item1; + xElements = pagedElements.Item2; + } + } + + //if specific types are declared we need to post filter them + //TODO: Update the service layer to join the cmsContentType table so we can query by content type too + if (IndexerData.IncludeNodeTypes.Any()) + { + var includeNodeTypeIds = contentTypes.Where(x => IndexerData.IncludeNodeTypes.Contains(x.Alias)).Select(x => x.Id); + xElements = xElements.Where(elm => includeNodeTypeIds.Contains(elm.AttributeValue("nodeType"))).ToArray(); + } + + foreach (var element in xElements) + { + if (element.Attribute("icon") == null) + { + element.Add(new XAttribute("icon", icons[element.AttributeValue("nodeType")])); + } + } + + AddNodesToIndex(xElements, type); + pageIndex++; + } while (xElements.Length == pageSize); + } + internal static IEnumerable GetSerializedContent( bool supportUnpublishdContent, Func serializer, diff --git a/src/UmbracoExamine/UmbracoExamine.csproj b/src/UmbracoExamine/UmbracoExamine.csproj index 1bc438edd7..8720849744 100644 --- a/src/UmbracoExamine/UmbracoExamine.csproj +++ b/src/UmbracoExamine/UmbracoExamine.csproj @@ -82,8 +82,9 @@ ..\Solution Items\TheFARM-Public.snk - - ..\packages\Examine.0.1.70.0\lib\Examine.dll + + ..\packages\Examine.0.1.80\lib\net45\Examine.dll + True ..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll diff --git a/src/UmbracoExamine/UmbracoExamineSearcher.cs b/src/UmbracoExamine/UmbracoExamineSearcher.cs index 7a2a80c3fe..26b02c904e 100644 --- a/src/UmbracoExamine/UmbracoExamineSearcher.cs +++ b/src/UmbracoExamine/UmbracoExamineSearcher.cs @@ -51,12 +51,41 @@ namespace UmbracoExamine /// public override string Name { - get - { - return _name; - } + get { return _name; } + } + + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + + public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) + : base(indexPath, analyzer) + { } + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer) + : base(luceneDirectory, analyzer) + { + } + + /// + /// Creates an NRT searcher + /// + /// + /// + public UmbracoExamineSearcher(IndexWriter writer, Analyzer analyzer) + : base(writer, analyzer) + { + } + + #endregion public override void Initialize(string name, System.Collections.Specialized.NameValueCollection config) { @@ -115,30 +144,6 @@ namespace UmbracoExamine } } - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - - public UmbracoExamineSearcher(DirectoryInfo indexPath, Analyzer analyzer) - : base(indexPath, analyzer) - { - } - - /// - /// Constructor to allow for creating an indexer at runtime - /// - /// - /// - - public UmbracoExamineSearcher(Lucene.Net.Store.Directory luceneDirectory, Analyzer analyzer) - : base(luceneDirectory, analyzer) - { - } - - #endregion - /// /// Used for unit tests /// diff --git a/src/UmbracoExamine/UmbracoMemberIndexer.cs b/src/UmbracoExamine/UmbracoMemberIndexer.cs index e3833da317..2e48dff64e 100644 --- a/src/UmbracoExamine/UmbracoMemberIndexer.cs +++ b/src/UmbracoExamine/UmbracoMemberIndexer.cs @@ -9,6 +9,7 @@ using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Services; using UmbracoExamine.Config; using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; using Examine; using System.IO; @@ -25,6 +26,7 @@ namespace UmbracoExamine { private readonly IMemberService _memberService; + private readonly IMemberTypeService _memberTypeService; private readonly IDataTypeService _dataTypeService; /// @@ -34,6 +36,7 @@ namespace UmbracoExamine { _dataTypeService = ApplicationContext.Current.Services.DataTypeService; _memberService = ApplicationContext.Current.Services.MemberService; + _memberTypeService = ApplicationContext.Current.Services.MemberTypeService; } /// @@ -49,6 +52,7 @@ namespace UmbracoExamine { _dataTypeService = ApplicationContext.Current.Services.DataTypeService; _memberService = ApplicationContext.Current.Services.MemberService; + _memberTypeService = ApplicationContext.Current.Services.MemberTypeService; } /// @@ -61,6 +65,8 @@ namespace UmbracoExamine /// /// /// + [Obsolete("Use the ctor specifying all dependencies instead")] + [EditorBrowsable(EditorBrowsableState.Never)] public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, IDataTypeService dataTypeService, IMemberService memberService, @@ -69,9 +75,31 @@ namespace UmbracoExamine { _dataTypeService = dataTypeService; _memberService = memberService; + _memberTypeService = ApplicationContext.Current.Services.MemberTypeService; } - + /// + /// Constructor to allow for creating an indexer at runtime + /// + /// + /// + /// + /// + /// + /// + /// + /// + public UmbracoMemberIndexer(IIndexCriteria indexerData, DirectoryInfo indexPath, IDataService dataService, + IDataTypeService dataTypeService, + IMemberService memberService, + IMemberTypeService memberTypeService, + Analyzer analyzer, bool async) + : base(indexerData, indexPath, dataService, analyzer, async) + { + _dataTypeService = dataTypeService; + _memberService = memberService; + _memberTypeService = memberTypeService; + } /// /// Ensures that the'_searchEmail' is added to the user fields so that it is indexed - without having to modify the config @@ -130,46 +158,67 @@ namespace UmbracoExamine if (SupportedTypes.Contains(type) == false) return; - const int pageSize = 1000; - var pageIndex = 0; - DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - Start data queries - {0}", type)); var stopwatch = new Stopwatch(); stopwatch.Start(); - IMember[] members; - - if (IndexerData.IncludeNodeTypes.Any()) + try { - //if there are specific node types then just index those - foreach (var nodeType in IndexerData.IncludeNodeTypes) + if (DisableXmlDocumentLookup == false) { - do + ReindexWithXmlEntries(type, -1, + () => _memberTypeService.GetAll().ToArray(), + (path, pIndex, pSize) => + { + long totalContent; + var result = _memberService.GetPagedXmlEntries(pIndex, pSize, out totalContent).ToArray(); + return new Tuple(totalContent, result); + }, + i => _memberService.GetById(i)); + } + else + { + const int pageSize = 1000; + var pageIndex = 0; + + IMember[] members; + + if (IndexerData.IncludeNodeTypes.Any()) { - long total; - members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); + //if there are specific node types then just index those + foreach (var nodeType in IndexerData.IncludeNodeTypes) + { + do + { + long total; + members = _memberService.GetAll(pageIndex, pageSize, out total, "LoginName", Direction.Ascending, true, null, nodeType).ToArray(); - AddNodesToIndex(GetSerializedMembers(members), type); + AddNodesToIndex(GetSerializedMembers(members), type); - pageIndex++; - } while (members.Length == pageSize); + pageIndex++; + } while (members.Length == pageSize); + } + } + else + { + //no node types specified, do all members + do + { + int total; + members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray(); + + AddNodesToIndex(GetSerializedMembers(members), type); + + pageIndex++; + } while (members.Length == pageSize); + } } } - else + finally { - //no node types specified, do all members - do - { - int total; - members = _memberService.GetAll(pageIndex, pageSize, out total).ToArray(); - - AddNodesToIndex(GetSerializedMembers(members), type); - - pageIndex++; - } while (members.Length == pageSize); + stopwatch.Stop(); } - - stopwatch.Stop(); + DataService.LogService.AddInfoLog(-1, string.Format("PerformIndexAll - End data queries - {0}, took {1}ms", type, stopwatch.ElapsedMilliseconds)); } @@ -189,7 +238,9 @@ namespace UmbracoExamine var fields = base.GetSpecialFieldsToIndex(allValuesForIndexing); //adds the special path property to the index - fields.Add("__key", allValuesForIndexing["__key"]); + string valuesForIndexing; + if (allValuesForIndexing.TryGetValue("__key", out valuesForIndexing)) + fields.Add("__key", valuesForIndexing); return fields; diff --git a/src/UmbracoExamine/packages.config b/src/UmbracoExamine/packages.config index 04734b9fb8..0c85a9c3ca 100644 --- a/src/UmbracoExamine/packages.config +++ b/src/UmbracoExamine/packages.config @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/umbraco.MacroEngines/packages.config b/src/umbraco.MacroEngines/packages.config index 9a785e35c3..930b7adbb0 100644 --- a/src/umbraco.MacroEngines/packages.config +++ b/src/umbraco.MacroEngines/packages.config @@ -1,6 +1,6 @@  - + diff --git a/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj b/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj index c9ee70e52c..198867a5d6 100644 --- a/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj +++ b/src/umbraco.MacroEngines/umbraco.MacroEngines.csproj @@ -45,8 +45,9 @@ false - - ..\packages\Examine.0.1.70.0\lib\Examine.dll + + ..\packages\Examine.0.1.80\lib\net45\Examine.dll + True ..\packages\HtmlAgilityPack.1.4.9\lib\Net45\HtmlAgilityPack.dll