diff --git a/src/Umbraco.Core/Models/PublicAccessEntry.cs b/src/Umbraco.Core/Models/PublicAccessEntry.cs index e93dc56e35..df2d9f9ddc 100644 --- a/src/Umbraco.Core/Models/PublicAccessEntry.cs +++ b/src/Umbraco.Core/Models/PublicAccessEntry.cs @@ -21,6 +21,10 @@ namespace Umbraco.Core.Models public PublicAccessEntry(IContent protectedNode, IContent loginNode, IContent noAccessNode, IEnumerable ruleCollection) { + if (protectedNode == null) throw new ArgumentNullException(nameof(protectedNode)); + if (loginNode == null) throw new ArgumentNullException(nameof(loginNode)); + if (noAccessNode == null) throw new ArgumentNullException(nameof(noAccessNode)); + LoginNodeId = loginNode.Id; NoAccessNodeId = noAccessNode.Id; _protectedNodeId = protectedNode.Id; diff --git a/src/Umbraco.Examine/ContentValueSetBuilder.cs b/src/Umbraco.Examine/ContentValueSetBuilder.cs index 5dd856ee7a..ec52bbbd4f 100644 --- a/src/Umbraco.Examine/ContentValueSetBuilder.cs +++ b/src/Umbraco.Examine/ContentValueSetBuilder.cs @@ -63,12 +63,12 @@ namespace Umbraco.Examine {"writerName", new object[] {c.GetWriterProfile(_userService)?.Name ?? "??"}}, {"writerID", new object[] {c.WriterId}}, {"template", new object[] {c.Template?.Id ?? 0}}, - {$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture", new object[] {0}}, + {UmbracoContentIndexer.VariesByCultureFieldName, new object[] {0}}, }; if (isVariant) { - values[$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture"] = new object[] { 1 }; + values[UmbracoContentIndexer.VariesByCultureFieldName] = new object[] { 1 }; foreach (var culture in c.AvailableCultures) { diff --git a/src/Umbraco.Examine/UmbracoContentIndexer.cs b/src/Umbraco.Examine/UmbracoContentIndexer.cs index e5f05caf43..984f70edcd 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexer.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexer.cs @@ -31,6 +31,8 @@ namespace Umbraco.Examine /// public class UmbracoContentIndexer : UmbracoExamineIndexer { + public const string VariesByCultureFieldName = UmbracoExamineIndexer.SpecialFieldPrefix + "VariesByCulture"; + public IValueSetBuilder MediaValueSetBuilder { get; } public IValueSetBuilder ContentValueSetBuilder { get; } protected IContentService ContentService { get; } @@ -304,10 +306,6 @@ namespace Umbraco.Examine mediaParentId = ParentId.Value; } - // merge note: 7.5 changes this to use mediaService.GetPagedXmlEntries but we cannot merge the - // change here as mediaService does not provide access to Xml in v8 - and actually Examine should - // not assume that Umbraco provides Xml at all. - IMedia[] media; do @@ -334,9 +332,77 @@ namespace Umbraco.Examine } } - + //TODO: We want to make a public method that iterates a data set, potentially with callbacks so that we can iterate + // a single data set once but populate multiple indexes with it. This is for Startup performance when no indexes exist. + // This could be used for any indexer of type UmbracoContentIndexer - BUT that means we need to make another interface + // for content indexers since UmbracoContentIndexer is strongly tied to lucene, so maybe we have a more generic interface + // or add to the current IUmbracoIndexer interface #endregion } + + public class ContentIndexDataSource + { + public ContentIndexDataSource(bool supportUnpublishedContent, int? parentId, + IContentService contentService, ISqlContext sqlContext, + IValueSetBuilder contentValueSetBuilder) + { + SupportUnpublishedContent = supportUnpublishedContent; + ParentId = parentId; + ContentService = contentService; + _contentValueSetBuilder = contentValueSetBuilder; + if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext)); + _publishedQuery = sqlContext.Query().Where(x => x.Published); + + } + + public bool SupportUnpublishedContent { get; } + public int? ParentId { get; } + public IContentService ContentService { get; } + + /// + /// This is a static query, it's parameters don't change so store statically + /// + private static IQuery _publishedQuery; + private readonly IValueSetBuilder _contentValueSetBuilder; + + public void Index(params IIndexer[] indexes) + { + const int pageSize = 10000; + var pageIndex = 0; + + var contentParentId = -1; + if (ParentId.HasValue && ParentId.Value > 0) + { + contentParentId = ParentId.Value; + } + IContent[] content; + + do + { + long total; + + IEnumerable descendants; + if (SupportUnpublishedContent) + { + descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total); + } + else + { + //add the published filter + //note: We will filter for published variants in the validator + descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, + _publishedQuery, Ordering.By("Path", Direction.Ascending)); + } + + content = descendants.ToArray(); + + foreach(var index in indexes) + index.IndexItems(_contentValueSetBuilder.GetValueSets(content)); + + pageIndex++; + } while (content.Length == pageSize); + } + } } diff --git a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs b/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs index 47b8a76c0f..55fd35e012 100644 --- a/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs +++ b/src/Umbraco.Examine/UmbracoContentIndexerOptions.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; namespace Umbraco.Examine { @@ -10,14 +11,42 @@ namespace Umbraco.Examine { public bool SupportUnpublishedContent { get; private set; } public bool SupportProtectedContent { get; private set; } - //TODO: We should make this a GUID! But to do that we sort of need to store the 'Path' as a comma list of GUIDs instead of int public int? ParentId { get; private set; } - public UmbracoContentIndexerOptions(bool supportUnpublishedContent, bool supportProtectedContent, int? parentId) + /// + /// Optional inclusion list of content types to index + /// + /// + /// All other types will be ignored if they do not match this list + /// + public IEnumerable IncludeContentTypes { get; private set; } + + /// + /// Optional exclusion list of content types to ignore + /// + /// + /// Any content type alias matched in this will not be included in the index + /// + public IEnumerable ExcludeContentTypes { get; private set; } + + /// + /// Creates a new + /// + /// If the index supports unpublished content + /// If the index supports protected content + /// Optional value indicating to only index content below this ID + /// Optional content type alias inclusion list + /// Optional content type alias exclusion list + public UmbracoContentIndexerOptions(bool supportUnpublishedContent, bool supportProtectedContent, + int? parentId = null, IEnumerable includeContentTypes = null, IEnumerable excludeContentTypes = null) { SupportUnpublishedContent = supportUnpublishedContent; SupportProtectedContent = supportProtectedContent; ParentId = parentId; + IncludeContentTypes = includeContentTypes; + ExcludeContentTypes = excludeContentTypes; } + + } } diff --git a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs index fcb058d268..220f082e48 100644 --- a/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs +++ b/src/Umbraco.Examine/UmbracoContentValueSetValidator.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Generic; using System.Linq; using Examine; using Examine.LuceneEngine.Providers; using Umbraco.Core; +using Umbraco.Core.Models; using Umbraco.Core.Services; namespace Umbraco.Examine @@ -15,7 +17,9 @@ namespace Umbraco.Examine { private readonly UmbracoContentIndexerOptions _options; private readonly IPublicAccessService _publicAccessService; + private const string PathKey = "path"; + public UmbracoContentValueSetValidator(UmbracoContentIndexerOptions options, IPublicAccessService publicAccessService) { _options = options; @@ -25,16 +29,31 @@ namespace Umbraco.Examine public bool Validate(ValueSet valueSet) { //check for published content - if (valueSet.Category == IndexTypes.Content - && valueSet.Values.ContainsKey(UmbracoExamineIndexer.PublishedFieldName)) + if (valueSet.Category == IndexTypes.Content && !_options.SupportUnpublishedContent) { - //fixme - variants? - - var published = valueSet.Values[UmbracoExamineIndexer.PublishedFieldName] != null && valueSet.Values[UmbracoExamineIndexer.PublishedFieldName][0].Equals(1); - //we don't support unpublished and the item is not published return false - if (_options.SupportUnpublishedContent == false && published == false) - { + if (!valueSet.Values.TryGetValue(UmbracoExamineIndexer.PublishedFieldName, out var published)) return false; + + if (!published[0].Equals(1)) + return false; + + //deal with variants, if there are unpublished variants than we need to remove them from the value set + if (valueSet.Values.TryGetValue(UmbracoContentIndexer.VariesByCultureFieldName, out var variesByCulture) + && variesByCulture.Count > 0 && variesByCulture[0].Equals(1)) + { + //so this valueset is for a content that varies by culture, now check for non-published cultures and remove those values + foreach(var publishField in valueSet.Values.Where(x => x.Key.StartsWith($"{UmbracoExamineIndexer.PublishedFieldName}_")).ToList()) + { + if (publishField.Value.Count <= 0 || !publishField.Value[0].Equals(1)) + { + //this culture is not published, so remove all of these culture values + var cultureSuffix = publishField.Key.Substring(publishField.Key.LastIndexOf('_')); + foreach (var cultureField in valueSet.Values.Where(x => x.Key.InvariantEndsWith(cultureSuffix)).ToList()) + { + valueSet.Values.Remove(cultureField.Key); + } + } + } } } @@ -45,11 +64,9 @@ namespace Umbraco.Examine if (pathValues[0].ToString().IsNullOrWhiteSpace()) return false; var path = pathValues[0].ToString(); - // Test for access if we're only indexing published content // return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content if (valueSet.Category == IndexTypes.Content - && _options.SupportUnpublishedContent == false - && _options.SupportProtectedContent == false + && !_options.SupportProtectedContent && _publicAccessService.IsProtected(path)) { return false; @@ -58,20 +75,26 @@ namespace Umbraco.Examine //check if this document is a descendent of the parent if (_options.ParentId.HasValue && _options.ParentId.Value > 0) { - if (path.IsNullOrWhiteSpace()) return false; - if (path.Contains(string.Concat(",", _options.ParentId.Value, ",")) == false) + if (!path.Contains(string.Concat(",", _options.ParentId.Value, ","))) return false; } //check for recycle bin - if (_options.SupportUnpublishedContent == false) + if (!_options.SupportUnpublishedContent) { - if (path.IsNullOrWhiteSpace()) return false; var recycleBinId = valueSet.Category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia; if (path.Contains(string.Concat(",", recycleBinId, ","))) return false; } + //check if this document is of a correct type of node type alias + if (_options.IncludeContentTypes != null && !_options.IncludeContentTypes.Contains(valueSet.ItemType)) + return false; + + //if this node type is part of our exclusion list + if (_options.ExcludeContentTypes != null && _options.ExcludeContentTypes.Contains(valueSet.ItemType)) + return false; + return true; } } diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 8262fb3cd3..10ba02b314 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -206,6 +206,7 @@ + @@ -463,7 +464,6 @@ - True True diff --git a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs index fa5a5afaf9..217ce307f0 100644 --- a/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs +++ b/src/Umbraco.Tests/UmbracoExamine/EventsTest.cs @@ -10,6 +10,7 @@ using Umbraco.Core.PropertyEditors; namespace Umbraco.Tests.UmbracoExamine { + [TestFixture] [UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)] public class EventsTest : ExamineBaseTest @@ -24,7 +25,7 @@ namespace Umbraco.Tests.UmbracoExamine using (indexer.ProcessNonAsync()) { var searcher = indexer.GetSearcher(); - + var contentService = new ExamineDemoDataContentService(); //get a node from the data repo var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]") @@ -33,9 +34,9 @@ namespace Umbraco.Tests.UmbracoExamine .First(); var valueSet = node.ConvertToValueSet(IndexTypes.Content); - indexer.IndexItems(new[] {valueSet}); + indexer.IndexItems(new[] { valueSet }); - var found = searcher.Search(searcher.CreateCriteria().Id((string) node.Attribute("id")).Compile()); + var found = searcher.Search(searcher.CreateCriteria().Id((string)node.Attribute("id")).Compile()); Assert.AreEqual(0, found.TotalItemCount); } diff --git a/src/Umbraco.Tests/UmbracoExamine/TestDataService.cs b/src/Umbraco.Tests/UmbracoExamine/TestDataService.cs deleted file mode 100644 index 856c3e99aa..0000000000 --- a/src/Umbraco.Tests/UmbracoExamine/TestDataService.cs +++ /dev/null @@ -1,32 +0,0 @@ -//using System.IO; -//using Umbraco.Tests.TestHelpers; -//using UmbracoExamine.DataServices; - -//namespace Umbraco.Tests.UmbracoExamine -//{ -// public class TestDataService : IDataService -// { - -// public TestDataService() -// { -// ContentService = new TestContentService(); -// LogService = new TestLogService(); -// MediaService = new TestMediaService(); -// } - -// #region IDataService Members - -// public IContentService ContentService { get; internal set; } - -// public ILogService LogService { get; internal set; } - -// public IMediaService MediaService { get; internal set; } - -// public string MapPath(string virtualPath) -// { -// return new DirectoryInfo(TestHelper.CurrentAssemblyDirectory) + "\\" + virtualPath.Replace("/", "\\"); -// } - -// #endregion -// } -//} diff --git a/src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs b/src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs new file mode 100644 index 0000000000..d06b2b4fff --- /dev/null +++ b/src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs @@ -0,0 +1,236 @@ +using Examine; +using NUnit.Framework; +using Umbraco.Examine; +using Moq; +using Umbraco.Core.Services; +using System.Collections.Generic; +using Umbraco.Core; +using Umbraco.Core.Models; +using System; +using System.Linq; + +namespace Umbraco.Tests.UmbracoExamine +{ + [TestFixture] + public class UmbracoContentValueSetValidatorTests + { + [Test] + public void Must_Have_Path() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(true, true, null), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsTrue(result); + } + + [Test] + public void Parent_Id() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(true, true, 555), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,444" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555,777" })); + Assert.IsTrue(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555,777,999" })); + Assert.IsTrue(result); + } + + [Test] + public void Inclusion_List() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(true, true, includeContentTypes: new List { "include-content" }), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, "include-content", new { hello = "world", path = "-1,555" })); + Assert.IsTrue(result); + } + + [Test] + public void Exclusion_List() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(true, true, excludeContentTypes: new List { "exclude-content" }), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" })); + Assert.IsTrue(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsTrue(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, "exclude-content", new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + } + + [Test] + public void Inclusion_Exclusion_List() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(true, true, + includeContentTypes: new List { "include-content", "exclude-content" }, + excludeContentTypes: new List { "exclude-content" }), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, "exclude-content", new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, "include-content", new { hello = "world", path = "-1,555" })); + Assert.IsTrue(result); + } + + [Test] + public void Recycle_Bin() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(false, true, null), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,-20,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,-20,555,777" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, + new Dictionary + { + ["hello"] = "world", + ["path"] = "-1,555", + [UmbracoExamineIndexer.PublishedFieldName] = 1 + })); + Assert.IsTrue(result); + } + + [Test] + public void Published_Only() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(false, true, null), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, + new Dictionary + { + ["hello"] = "world", + ["path"] = "-1,555", + [UmbracoExamineIndexer.PublishedFieldName] = 0 + })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, + new Dictionary + { + ["hello"] = "world", + ["path"] = "-1,555", + [UmbracoExamineIndexer.PublishedFieldName] = 1 + })); + Assert.IsTrue(result); + } + + [Test] + public void Published_Only_With_Variants() + { + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(false, true, null), + Mock.Of()); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, + new Dictionary + { + ["hello"] = "world", + ["path"] = "-1,555", + [UmbracoContentIndexer.VariesByCultureFieldName] = 1, + [UmbracoExamineIndexer.PublishedFieldName] = 0 + })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("555", IndexTypes.Content, + new Dictionary + { + ["hello"] = "world", + ["path"] = "-1,555", + [UmbracoContentIndexer.VariesByCultureFieldName] = 1, + [UmbracoExamineIndexer.PublishedFieldName] = 1 + })); + Assert.IsTrue(result); + + var valueSet = new ValueSet("555", IndexTypes.Content, + new Dictionary + { + ["hello"] = "world", + ["path"] = "-1,555", + [UmbracoContentIndexer.VariesByCultureFieldName] = 1, + [$"{UmbracoExamineIndexer.PublishedFieldName}_en-us"] = 1, + ["hello_en-us"] = "world", + ["title_en-us"] = "my title", + [$"{UmbracoExamineIndexer.PublishedFieldName}_es-es"] = 0, + ["hello_es-ES"] = "world", + ["title_es-ES"] = "my title", + [UmbracoExamineIndexer.PublishedFieldName] = 1 + }); + Assert.AreEqual(10, valueSet.Values.Count()); + Assert.IsTrue(valueSet.Values.ContainsKey($"{UmbracoExamineIndexer.PublishedFieldName}_es-es")); + Assert.IsTrue(valueSet.Values.ContainsKey("hello_es-ES")); + Assert.IsTrue(valueSet.Values.ContainsKey("title_es-ES")); + + result = validator.Validate(valueSet); + Assert.IsTrue(result); + + Assert.AreEqual(7, valueSet.Values.Count()); //filtered to 7 values (removes es-es values) + Assert.IsFalse(valueSet.Values.ContainsKey($"{UmbracoExamineIndexer.PublishedFieldName}_es-es")); + Assert.IsFalse(valueSet.Values.ContainsKey("hello_es-ES")); + Assert.IsFalse(valueSet.Values.ContainsKey("title_es-ES")); + } + + [Test] + public void Non_Protected() + { + var publicAccessService = new Mock(); + publicAccessService.Setup(x => x.IsProtected("-1,555")) + .Returns(Attempt.Succeed(new PublicAccessEntry(Guid.NewGuid(), 555, 444, 333, Enumerable.Empty()))); + publicAccessService.Setup(x => x.IsProtected("-1,777")) + .Returns(Attempt.Fail()); + var validator = new UmbracoContentValueSetValidator( + new UmbracoContentIndexerOptions(true, false, null), + publicAccessService.Object); + + var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" })); + Assert.IsFalse(result); + + result = validator.Validate(new ValueSet("777", IndexTypes.Content, new { hello = "world", path = "-1,777" })); + Assert.IsTrue(result); + } + } +}