yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates the content value set validator + tests

Browse Source
This commit is contained in:
Shannon
2018-11-27 17:43:56 +11:00
parent d3057ea2e1
commit ff3af8b7a7
9 changed files with 387 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/Umbraco.Core/Models/PublicAccessEntry.cs
View File

@@ -21,6 +21,10 @@ namespace Umbraco.Core.Models
public PublicAccessEntry(IContent protectedNode, IContent loginNode, IContent noAccessNode, IEnumerable<PublicAccessRule> ruleCollection)
{
if (protectedNode == null) throw new ArgumentNullException(nameof(protectedNode));
if (loginNode == null) throw new ArgumentNullException(nameof(loginNode));
if (noAccessNode == null) throw new ArgumentNullException(nameof(noAccessNode));
LoginNodeId = loginNode.Id;
NoAccessNodeId = noAccessNode.Id;
_protectedNodeId = protectedNode.Id;

4
src/Umbraco.Examine/ContentValueSetBuilder.cs
View File

@@ -63,12 +63,12 @@ namespace Umbraco.Examine
{"writerName", new object[] {c.GetWriterProfile(_userService)?.Name ?? "??"}},
{"writerID", new object[] {c.WriterId}},
{"template", new object[] {c.Template?.Id ?? 0}},
{$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture", new object[] {0}},
{UmbracoContentIndexer.VariesByCultureFieldName, new object[] {0}},
};
if (isVariant)
{
values[$"{UmbracoExamineIndexer.SpecialFieldPrefix}VariesByCulture"] = new object[] { 1 };
values[UmbracoContentIndexer.VariesByCultureFieldName] = new object[] { 1 };
foreach (var culture in c.AvailableCultures)
{

76
src/Umbraco.Examine/UmbracoContentIndexer.cs
View File

@@ -31,6 +31,8 @@ namespace Umbraco.Examine
/// </summary>
public class UmbracoContentIndexer : UmbracoExamineIndexer
{
public const string VariesByCultureFieldName = UmbracoExamineIndexer.SpecialFieldPrefix + "VariesByCulture";
public IValueSetBuilder<IMedia> MediaValueSetBuilder { get; }
public IValueSetBuilder<IContent> ContentValueSetBuilder { get; }
protected IContentService ContentService { get; }
@@ -304,10 +306,6 @@ namespace Umbraco.Examine
mediaParentId = ParentId.Value;
}
// merge note: 7.5 changes this to use mediaService.GetPagedXmlEntries but we cannot merge the
// change here as mediaService does not provide access to Xml in v8 - and actually Examine should
// not assume that Umbraco provides Xml at all.
IMedia[] media;
do
@@ -334,9 +332,77 @@ namespace Umbraco.Examine
}
}
//TODO: We want to make a public method that iterates a data set, potentially with callbacks so that we can iterate
// a single data set once but populate multiple indexes with it. This is for Startup performance when no indexes exist.
// This could be used for any indexer of type UmbracoContentIndexer - BUT that means we need to make another interface
// for content indexers since UmbracoContentIndexer is strongly tied to lucene, so maybe we have a more generic interface
// or add to the current IUmbracoIndexer interface
#endregion
}
public class ContentIndexDataSource
{
public ContentIndexDataSource(bool supportUnpublishedContent, int? parentId,
IContentService contentService, ISqlContext sqlContext,
IValueSetBuilder<IContent> contentValueSetBuilder)
{
SupportUnpublishedContent = supportUnpublishedContent;
ParentId = parentId;
ContentService = contentService;
_contentValueSetBuilder = contentValueSetBuilder;
if (sqlContext == null) throw new ArgumentNullException(nameof(sqlContext));
_publishedQuery = sqlContext.Query<IContent>().Where(x => x.Published);
}
public bool SupportUnpublishedContent { get; }
public int? ParentId { get; }
public IContentService ContentService { get; }
/// <summary>
/// This is a static query, it's parameters don't change so store statically
/// </summary>
private static IQuery<IContent> _publishedQuery;
private readonly IValueSetBuilder<IContent> _contentValueSetBuilder;
public void Index(params IIndexer[] indexes)
{
const int pageSize = 10000;
var pageIndex = 0;
var contentParentId = -1;
if (ParentId.HasValue && ParentId.Value > 0)
{
contentParentId = ParentId.Value;
}
IContent[] content;
do
{
long total;
IEnumerable<IContent> descendants;
if (SupportUnpublishedContent)
{
descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
}
else
{
//add the published filter
//note: We will filter for published variants in the validator
descendants = ContentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total,
_publishedQuery, Ordering.By("Path", Direction.Ascending));
}
content = descendants.ToArray();
foreach(var index in indexes)
index.IndexItems(_contentValueSetBuilder.GetValueSets(content));
pageIndex++;
} while (content.Length == pageSize);
}
}
}

33
src/Umbraco.Examine/UmbracoContentIndexerOptions.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
namespace Umbraco.Examine
{
@@ -10,14 +11,42 @@ namespace Umbraco.Examine
{
public bool SupportUnpublishedContent { get; private set; }
public bool SupportProtectedContent { get; private set; }
//TODO: We should make this a GUID! But to do that we sort of need to store the 'Path' as a comma list of GUIDs instead of int
public int? ParentId { get; private set; }
public UmbracoContentIndexerOptions(bool supportUnpublishedContent, bool supportProtectedContent, int? parentId)
/// <summary>
/// Optional inclusion list of content types to index
/// </summary>
/// <remarks>
/// All other types will be ignored if they do not match this list
/// </remarks>
public IEnumerable<string> IncludeContentTypes { get; private set; }
/// <summary>
/// Optional exclusion list of content types to ignore
/// </summary>
/// <remarks>
/// Any content type alias matched in this will not be included in the index
/// </remarks>
public IEnumerable<string> ExcludeContentTypes { get; private set; }
/// <summary>
/// Creates a new <see cref="UmbracoContentIndexerOptions"/>
/// </summary>
/// <param name="supportUnpublishedContent">If the index supports unpublished content</param>
/// <param name="supportProtectedContent">If the index supports protected content</param>
/// <param name="parentId">Optional value indicating to only index content below this ID</param>
/// <param name="includeContentTypes">Optional content type alias inclusion list</param>
/// <param name="excludeContentTypes">Optional content type alias exclusion list</param>
public UmbracoContentIndexerOptions(bool supportUnpublishedContent, bool supportProtectedContent,
int? parentId = null, IEnumerable<string> includeContentTypes = null, IEnumerable<string> excludeContentTypes = null)
{
SupportUnpublishedContent = supportUnpublishedContent;
SupportProtectedContent = supportProtectedContent;
ParentId = parentId;
IncludeContentTypes = includeContentTypes;
ExcludeContentTypes = excludeContentTypes;
}
}
}

53
src/Umbraco.Examine/UmbracoContentValueSetValidator.cs
View File

@@ -1,8 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Examine;
using Examine.LuceneEngine.Providers;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
namespace Umbraco.Examine
@@ -15,7 +17,9 @@ namespace Umbraco.Examine
{
private readonly UmbracoContentIndexerOptions _options;
private readonly IPublicAccessService _publicAccessService;
private const string PathKey = "path";
public UmbracoContentValueSetValidator(UmbracoContentIndexerOptions options, IPublicAccessService publicAccessService)
{
_options = options;
@@ -25,16 +29,31 @@ namespace Umbraco.Examine
public bool Validate(ValueSet valueSet)
{
//check for published content
if (valueSet.Category == IndexTypes.Content
&& valueSet.Values.ContainsKey(UmbracoExamineIndexer.PublishedFieldName))
if (valueSet.Category == IndexTypes.Content && !_options.SupportUnpublishedContent)
{
//fixme - variants?
var published = valueSet.Values[UmbracoExamineIndexer.PublishedFieldName] != null && valueSet.Values[UmbracoExamineIndexer.PublishedFieldName][0].Equals(1);
//we don't support unpublished and the item is not published return false
if (_options.SupportUnpublishedContent == false && published == false)
{
if (!valueSet.Values.TryGetValue(UmbracoExamineIndexer.PublishedFieldName, out var published))
return false;
if (!published[0].Equals(1))
return false;
//deal with variants, if there are unpublished variants than we need to remove them from the value set
if (valueSet.Values.TryGetValue(UmbracoContentIndexer.VariesByCultureFieldName, out var variesByCulture)
&& variesByCulture.Count > 0 && variesByCulture[0].Equals(1))
{
//so this valueset is for a content that varies by culture, now check for non-published cultures and remove those values
foreach(var publishField in valueSet.Values.Where(x => x.Key.StartsWith($"{UmbracoExamineIndexer.PublishedFieldName}_")).ToList())
{
if (publishField.Value.Count <= 0 || !publishField.Value[0].Equals(1))
{
//this culture is not published, so remove all of these culture values
var cultureSuffix = publishField.Key.Substring(publishField.Key.LastIndexOf('_'));
foreach (var cultureField in valueSet.Values.Where(x => x.Key.InvariantEndsWith(cultureSuffix)).ToList())
{
valueSet.Values.Remove(cultureField.Key);
}
}
}
}
}
@@ -45,11 +64,9 @@ namespace Umbraco.Examine
if (pathValues[0].ToString().IsNullOrWhiteSpace()) return false;
var path = pathValues[0].ToString();
// Test for access if we're only indexing published content
// return nothing if we're not supporting protected content and it is protected, and we're not supporting unpublished content
if (valueSet.Category == IndexTypes.Content
&& _options.SupportUnpublishedContent == false
&& _options.SupportProtectedContent == false
&& !_options.SupportProtectedContent
&& _publicAccessService.IsProtected(path))
{
return false;
@@ -58,20 +75,26 @@ namespace Umbraco.Examine
//check if this document is a descendent of the parent
if (_options.ParentId.HasValue && _options.ParentId.Value > 0)
{
if (path.IsNullOrWhiteSpace()) return false;
if (path.Contains(string.Concat(",", _options.ParentId.Value, ",")) == false)
if (!path.Contains(string.Concat(",", _options.ParentId.Value, ",")))
return false;
}
//check for recycle bin
if (_options.SupportUnpublishedContent == false)
if (!_options.SupportUnpublishedContent)
{
if (path.IsNullOrWhiteSpace()) return false;
var recycleBinId = valueSet.Category == IndexTypes.Content ? Constants.System.RecycleBinContent : Constants.System.RecycleBinMedia;
if (path.Contains(string.Concat(",", recycleBinId, ",")))
return false;
}
//check if this document is of a correct type of node type alias
if (_options.IncludeContentTypes != null && !_options.IncludeContentTypes.Contains(valueSet.ItemType))
return false;
//if this node type is part of our exclusion list
if (_options.ExcludeContentTypes != null && _options.ExcludeContentTypes.Contains(valueSet.ItemType))
return false;
return true;
}
}

2
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -206,6 +206,7 @@
<Compile Include="Testing\Objects\Accessors\TestUmbracoContextAccessor.cs" />
<Compile Include="CoreThings\UdiTests.cs" />
<Compile Include="UmbracoExamine\RandomIdRamDirectory.cs" />
<Compile Include="UmbracoExamine\UmbracoContentValueSetValidatorTests.cs" />
<Compile Include="Web\AngularIntegration\AngularAntiForgeryTests.cs" />
<Compile Include="Web\AngularIntegration\ContentModelSerializationTests.cs" />
<Compile Include="Web\AngularIntegration\JsInitializationTests.cs" />
@@ -463,7 +464,6 @@
<Compile Include="UmbracoExamine\IndexInitializer.cs" />
<Compile Include="UmbracoExamine\IndexTest.cs" />
<Compile Include="UmbracoExamine\SearchTests.cs" />
<Compile Include="UmbracoExamine\TestDataService.cs" />
<Compile Include="UmbracoExamine\TestFiles.Designer.cs">
<AutoGen>True</AutoGen>
<DesignTime>True</DesignTime>

7
src/Umbraco.Tests/UmbracoExamine/EventsTest.cs
View File

@@ -10,6 +10,7 @@ using Umbraco.Core.PropertyEditors;
namespace Umbraco.Tests.UmbracoExamine
{
[TestFixture]
[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
public class EventsTest : ExamineBaseTest
@@ -24,7 +25,7 @@ namespace Umbraco.Tests.UmbracoExamine
using (indexer.ProcessNonAsync())
{
var searcher = indexer.GetSearcher();
var contentService = new ExamineDemoDataContentService();
//get a node from the data repo
var node = contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
@@ -33,9 +34,9 @@ namespace Umbraco.Tests.UmbracoExamine
.First();
var valueSet = node.ConvertToValueSet(IndexTypes.Content);
indexer.IndexItems(new[] {valueSet});
indexer.IndexItems(new[] { valueSet });
var found = searcher.Search(searcher.CreateCriteria().Id((string) node.Attribute("id")).Compile());
var found = searcher.Search(searcher.CreateCriteria().Id((string)node.Attribute("id")).Compile());
Assert.AreEqual(0, found.TotalItemCount);
}

32
src/Umbraco.Tests/UmbracoExamine/TestDataService.cs
View File

@@ -1,32 +0,0 @@
//using System.IO;
//using Umbraco.Tests.TestHelpers;
//using UmbracoExamine.DataServices;
//namespace Umbraco.Tests.UmbracoExamine
//{
// public class TestDataService : IDataService
// {
// public TestDataService()
// {
// ContentService = new TestContentService();
// LogService = new TestLogService();
// MediaService = new TestMediaService();
// }
// #region IDataService Members
// public IContentService ContentService { get; internal set; }
// public ILogService LogService { get; internal set; }
// public IMediaService MediaService { get; internal set; }
// public string MapPath(string virtualPath)
// {
// return new DirectoryInfo(TestHelper.CurrentAssemblyDirectory) + "\\" + virtualPath.Replace("/", "\\");
// }
// #endregion
// }
//}

236
src/Umbraco.Tests/UmbracoExamine/UmbracoContentValueSetValidatorTests.cs Normal file
View File

@@ -0,0 +1,236 @@
using Examine;
using NUnit.Framework;
using Umbraco.Examine;
using Moq;
using Umbraco.Core.Services;
using System.Collections.Generic;
using Umbraco.Core;
using Umbraco.Core.Models;
using System;
using System.Linq;
namespace Umbraco.Tests.UmbracoExamine
{
[TestFixture]
public class UmbracoContentValueSetValidatorTests
{
[Test]
public void Must_Have_Path()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(true, true, null),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsTrue(result);
}
[Test]
public void Parent_Id()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(true, true, 555),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,444" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555,777" }));
Assert.IsTrue(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555,777,999" }));
Assert.IsTrue(result);
}
[Test]
public void Inclusion_List()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(true, true, includeContentTypes: new List<string> { "include-content" }),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, "include-content", new { hello = "world", path = "-1,555" }));
Assert.IsTrue(result);
}
[Test]
public void Exclusion_List()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(true, true, excludeContentTypes: new List<string> { "exclude-content" }),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" }));
Assert.IsTrue(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsTrue(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, "exclude-content", new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
}
[Test]
public void Inclusion_Exclusion_List()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(true, true,
includeContentTypes: new List<string> { "include-content", "exclude-content" },
excludeContentTypes: new List<string> { "exclude-content" }),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, "test-content", new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, "exclude-content", new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, "include-content", new { hello = "world", path = "-1,555" }));
Assert.IsTrue(result);
}
[Test]
public void Recycle_Bin()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(false, true, null),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,-20,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,-20,555,777" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content,
new Dictionary<string, object>
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoExamineIndexer.PublishedFieldName] = 1
}));
Assert.IsTrue(result);
}
[Test]
public void Published_Only()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(false, true, null),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content,
new Dictionary<string, object>
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoExamineIndexer.PublishedFieldName] = 0
}));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content,
new Dictionary<string, object>
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoExamineIndexer.PublishedFieldName] = 1
}));
Assert.IsTrue(result);
}
[Test]
public void Published_Only_With_Variants()
{
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(false, true, null),
Mock.Of<IPublicAccessService>());
var result = validator.Validate(new ValueSet("555", IndexTypes.Content,
new Dictionary<string, object>
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoContentIndexer.VariesByCultureFieldName] = 1,
[UmbracoExamineIndexer.PublishedFieldName] = 0
}));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("555", IndexTypes.Content,
new Dictionary<string, object>
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoContentIndexer.VariesByCultureFieldName] = 1,
[UmbracoExamineIndexer.PublishedFieldName] = 1
}));
Assert.IsTrue(result);
var valueSet = new ValueSet("555", IndexTypes.Content,
new Dictionary<string, object>
{
["hello"] = "world",
["path"] = "-1,555",
[UmbracoContentIndexer.VariesByCultureFieldName] = 1,
[$"{UmbracoExamineIndexer.PublishedFieldName}_en-us"] = 1,
["hello_en-us"] = "world",
["title_en-us"] = "my title",
[$"{UmbracoExamineIndexer.PublishedFieldName}_es-es"] = 0,
["hello_es-ES"] = "world",
["title_es-ES"] = "my title",
[UmbracoExamineIndexer.PublishedFieldName] = 1
});
Assert.AreEqual(10, valueSet.Values.Count());
Assert.IsTrue(valueSet.Values.ContainsKey($"{UmbracoExamineIndexer.PublishedFieldName}_es-es"));
Assert.IsTrue(valueSet.Values.ContainsKey("hello_es-ES"));
Assert.IsTrue(valueSet.Values.ContainsKey("title_es-ES"));
result = validator.Validate(valueSet);
Assert.IsTrue(result);
Assert.AreEqual(7, valueSet.Values.Count()); //filtered to 7 values (removes es-es values)
Assert.IsFalse(valueSet.Values.ContainsKey($"{UmbracoExamineIndexer.PublishedFieldName}_es-es"));
Assert.IsFalse(valueSet.Values.ContainsKey("hello_es-ES"));
Assert.IsFalse(valueSet.Values.ContainsKey("title_es-ES"));
}
[Test]
public void Non_Protected()
{
var publicAccessService = new Mock<IPublicAccessService>();
publicAccessService.Setup(x => x.IsProtected("-1,555"))
.Returns(Attempt.Succeed(new PublicAccessEntry(Guid.NewGuid(), 555, 444, 333, Enumerable.Empty<PublicAccessRule>())));
publicAccessService.Setup(x => x.IsProtected("-1,777"))
.Returns(Attempt.Fail<PublicAccessEntry>());
var validator = new UmbracoContentValueSetValidator(
new UmbracoContentIndexerOptions(true, false, null),
publicAccessService.Object);
var result = validator.Validate(new ValueSet("555", IndexTypes.Content, new { hello = "world", path = "-1,555" }));
Assert.IsFalse(result);
result = validator.Validate(new ValueSet("777", IndexTypes.Content, new { hello = "world", path = "-1,777" }));
Assert.IsTrue(result);
}
}
}