yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backports some changes from v8 so that we can perform a nice strongly typed query with paging - this is then used for the content indexer to index content via the db for published content instead of the xml cache system. This was already done in v8 but have no backported the logic and fixed up the unit tests. When merging with v8 we will most likely just keep all v8 stuff and discard these changes, but we'll need to compare just in case. All tests pass and re-indexing is working as expected. Also updated the paging count from 1000 to 5000 for reindexing.

Browse Source
This commit is contained in:
Shannon
2016-05-26 15:30:40 +02:00
parent def8ee4962
commit 628ce5ea1f
13 changed files with 239 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
View File

@@ -771,22 +771,22 @@ namespace Umbraco.Core.Persistence.Repositories
/// <param name="filter">Search text filter</param>
/// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
public IEnumerable<IContent> GetPagedResultsByQuery(IQuery<IContent> query, long pageIndex, int pageSize, out long totalRecords,
string orderBy, Direction orderDirection, bool orderBySystemField, string filter = "")
string orderBy, Direction orderDirection, bool orderBySystemField, IQuery<IContent> filter = null)
{
//NOTE: This uses the GetBaseQuery method but that does not take into account the required 'newest' field which is
// what we always require for a paged result, so we'll ensure it's included in the filter
var args = new List<object>();
var sbWhere = new StringBuilder("AND (cmsDocument.newest = 1)");
if (filter.IsNullOrWhiteSpace() == false)
var filterSql = new Sql().Append("AND (cmsDocument.newest = 1)");
if (filter != null)
{
sbWhere.Append(" AND (cmsDocument." + SqlSyntax.GetQuotedColumnName("text") + " LIKE @" + args.Count + ")");
args.Add("%" + filter + "%");
foreach (var filterClaus in filter.GetWhereClauses())
{
filterSql.Append(string.Format("AND ({0})", filterClaus.Item1), filterClaus.Item2);
}
}
Func<Tuple<string, object[]>> filterCallback = () => new Tuple<string, object[]>(sbWhere.ToString(), args.ToArray());
Func<Tuple<string, object[]>> filterCallback = () => new Tuple<string, object[]>(filterSql.SQL, filterSql.Arguments);
return GetPagedResultsByQuery<DocumentDto, Content>(query, pageIndex, pageSize, out totalRecords,
new Tuple<string, string>("cmsDocument", "nodeId"),

4
src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs
View File

@@ -85,9 +85,9 @@ namespace Umbraco.Core.Persistence.Repositories
/// <param name="orderBy">Field to order by</param>
/// <param name="orderDirection">Direction to order by</param>
/// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
/// <param name="filter">Search text filter</param>
/// <param name="filter"></param>
/// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
IEnumerable<IContent> GetPagedResultsByQuery(IQuery<IContent> query, long pageIndex, int pageSize, out long totalRecords,
string orderBy, Direction orderDirection, bool orderBySystemField, string filter = "");
string orderBy, Direction orderDirection, bool orderBySystemField, IQuery<IContent> filter = null);
}
}

22
src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
View File

@@ -235,13 +235,29 @@ namespace Umbraco.Core.Persistence.Repositories
private Sql GetFilteredSqlForPagedResults(Sql sql, Func<Tuple<string, object[]>> defaultFilter = null)
{
//copy to var so that the original isn't changed
var filteredSql = new Sql(sql.SQL, sql.Arguments);
Sql filteredSql;
// Apply filter
if (defaultFilter != null)
{
var filterResult = defaultFilter();
filteredSql.Append(filterResult.Item1, filterResult.Item2);
//NOTE: this is certainly strange - NPoco handles this much better but we need to re-create the sql
// instance a couple of times to get the parameter order correct, for some reason the first
// time the arguments don't show up correctly but the SQL argument parameter names are actually updated
// accordingly - so we re-create it again. In v8 we don't need to do this and it's already taken care of.
filteredSql = new Sql(sql.SQL, sql.Arguments);
var args = filteredSql.Arguments.Concat(filterResult.Item2).ToArray();
filteredSql = new Sql(
string.Format("{0} {1}", filteredSql.SQL, filterResult.Item1),
args);
filteredSql = new Sql(filteredSql.SQL, args);
}
else
{
//copy to var so that the original isn't changed
filteredSql = new Sql(sql.SQL, sql.Arguments);
}
return filteredSql;
}

46
src/Umbraco.Core/Services/ContentService.cs
View File

@@ -536,7 +536,12 @@ namespace Umbraco.Core.Services
{
query.Where(x => x.ParentId == id);
}
var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, orderBySystemField, filter);
IQuery<IContent> filterQuery = null;
if (filter.IsNullOrWhiteSpace() == false)
{
filterQuery = Query<IContent>.Builder.Where(x => x.Name.Contains(filter));
}
var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, orderBySystemField, filterQuery);
return contents;
}
@@ -593,12 +598,49 @@ namespace Umbraco.Core.Services
{
query.Where(x => x.Path.SqlContains(string.Format(",{0},", id), TextColumnType.NVarchar));
}
var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, orderBySystemField, filter);
IQuery<IContent> filterQuery = null;
if (filter.IsNullOrWhiteSpace() == false)
{
filterQuery = Query<IContent>.Builder.Where(x => x.Name.Contains(filter));
}
var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, orderBySystemField, filterQuery);
return contents;
}
}
/// <summary>
/// Gets a collection of <see cref="IContent"/> objects by Parent Id
/// </summary>
/// <param name="id">Id of the Parent to retrieve Descendants from</param>
/// <param name="pageIndex">Page number</param>
/// <param name="pageSize">Page size</param>
/// <param name="totalChildren">Total records query would return without paging</param>
/// <param name="orderBy">Field to order by</param>
/// <param name="orderDirection">Direction to order by</param>
/// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
/// <param name="filter">Search filter</param>
/// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
public IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy, Direction orderDirection, bool orderBySystemField, IQuery<IContent> filter)
{
Mandate.ParameterCondition(pageIndex >= 0, "pageIndex");
Mandate.ParameterCondition(pageSize > 0, "pageSize");
using (var repository = RepositoryFactory.CreateContentRepository(UowProvider.GetUnitOfWork()))
{
var query = Query<IContent>.Builder;
//if the id is System Root, then just get all
if (id != Constants.System.Root)
{
query.Where(x => x.Path.SqlContains(string.Format(",{0},", id), TextColumnType.NVarchar));
}
var contents = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, orderBySystemField, filter);
return contents;
}
}
/// <summary>
/// Gets a collection of <see cref="IContent"/> objects by its name or partial name
/// </summary>

16
src/Umbraco.Core/Services/IContentService.cs
View File

@@ -4,6 +4,7 @@ using System.ComponentModel;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Membership;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Publishing;
namespace Umbraco.Core.Services
@@ -268,6 +269,21 @@ namespace Umbraco.Core.Services
IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
string orderBy, Direction orderDirection, bool orderBySystemField, string filter);
/// <summary>
/// Gets a collection of <see cref="IContent"/> objects by Parent Id
/// </summary>
/// <param name="id">Id of the Parent to retrieve Descendants from</param>
/// <param name="pageIndex">Page number</param>
/// <param name="pageSize">Page size</param>
/// <param name="totalRecords">Total records query would return without paging</param>
/// <param name="orderBy">Field to order by</param>
/// <param name="orderDirection">Direction to order by</param>
/// <param name="orderBySystemField">Flag to indicate when ordering by system field</param>
/// <param name="filter"></param>
/// <returns>An Enumerable list of <see cref="IContent"/> objects</returns>
IEnumerable<IContent> GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords,
string orderBy, Direction orderDirection, bool orderBySystemField, IQuery<IContent> filter);
/// <summary>
/// Gets a collection of an <see cref="IContent"/> objects versions by its Id
/// </summary>

8
src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
View File

@@ -685,8 +685,10 @@ namespace Umbraco.Tests.Persistence.Repositories
{
// Act
var query = Query<IContent>.Builder.Where(x => x.Level == 2);
var filterQuery = Query<IContent>.Builder.Where(x => x.Name.Contains("Page 2"));
long totalRecords;
var result = repository.GetPagedResultsByQuery(query, 0, 1, out totalRecords, "Name", Direction.Ascending, true, "Page 2");
var result = repository.GetPagedResultsByQuery(query, 0, 1, out totalRecords, "Name", Direction.Ascending, true, filterQuery);
// Assert
Assert.That(totalRecords, Is.EqualTo(1));
@@ -706,8 +708,10 @@ namespace Umbraco.Tests.Persistence.Repositories
{
// Act
var query = Query<IContent>.Builder.Where(x => x.Level == 2);
var filterQuery = Query<IContent>.Builder.Where(x => x.Name.Contains("Page"));
long totalRecords;
var result = repository.GetPagedResultsByQuery(query, 0, 1, out totalRecords, "Name", Direction.Ascending, true, "Page");
var result = repository.GetPagedResultsByQuery(query, 0, 1, out totalRecords, "Name", Direction.Ascending, true, filterQuery);
// Assert
Assert.That(totalRecords, Is.EqualTo(2));

20
src/Umbraco.Tests/PublishedContent/LegacyExamineBackedMediaTests.cs
View File

@@ -7,14 +7,17 @@ using NUnit.Framework;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.UmbracoExamine;
using umbraco.MacroEngines;
using Umbraco.Core;
using Umbraco.Core.Logging;
using Umbraco.Core.Persistence.Mappers;
namespace Umbraco.Tests.PublishedContent
{
public class LegacyExamineBackedMediaTests : ExamineBaseTest
{
public override void TestSetup()
public override void Initialize()
{
base.TestSetup();
base.Initialize();
var settings = SettingsForTests.GenerateMockSettings();
var contentMock = Mock.Get(settings.Content);
@@ -22,13 +25,8 @@ namespace Umbraco.Tests.PublishedContent
contentMock.Setup(x => x.UmbracoLibraryCacheDuration).Returns(1800);
SettingsForTests.ConfigureSettings(settings);
}
public override void TestTearDown()
{
SettingsForTests.Reset();
base.TestTearDown();
}
[Test]
public void Ensure_Children_Are_Sorted()
{
@@ -47,7 +45,9 @@ namespace Umbraco.Tests.PublishedContent
var children = backedMedia.ChildrenAsList.Value;
var currSort = 0;
for (var i = 0; i < children.Count(); i++)
Assert.Greater(children.Count, 0);
for (var i = 0; i < children.Count; i++)
{
Assert.GreaterOrEqual(children[i].SortOrder, currSort);
currSort = children[i].SortOrder;

28
src/Umbraco.Tests/UmbracoExamine/EventsTest.cs
View File

@@ -45,19 +45,21 @@ namespace Umbraco.Tests.UmbracoExamine
private static UmbracoContentIndexer _indexer;
private Lucene.Net.Store.Directory _luceneDir;
public override void TestSetup()
{
base.TestSetup();
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
}
public override void Initialize()
{
base.Initialize();
public override void TestTearDown()
{
base.TestTearDown();
_luceneDir.Dispose();
}
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
}
public override void TearDown()
{
base.TearDown();
_luceneDir.Dispose();
}
}
}

28
src/Umbraco.Tests/UmbracoExamine/ExamineBaseTest.cs
View File

@@ -1,5 +1,9 @@
using NUnit.Framework;
using Moq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Logging;
using Umbraco.Core.ObjectResolution;
using Umbraco.Core.Persistence.Mappers;
using Umbraco.Core.Strings;
using Umbraco.Tests.TestHelpers;
using UmbracoExamine;
@@ -7,29 +11,27 @@ using UmbracoExamine;
namespace Umbraco.Tests.UmbracoExamine
{
[TestFixture]
public abstract class ExamineBaseTest : BaseUmbracoConfigurationTest
public abstract class ExamineBaseTest : BaseDatabaseFactoryTest
{
[SetUp]
public virtual void TestSetup()
/// <summary>
/// sets up resolvers before resolution is frozen
/// </summary>
protected override void FreezeResolution()
{
UmbracoExamineSearcher.DisableInitializationCheck = true;
BaseUmbracoIndexer.DisableInitializationCheck = true;
ShortStringHelperResolver.Current = new ShortStringHelperResolver(new DefaultShortStringHelper(SettingsForTests.GetDefault()));
Resolution.Freeze();
base.FreezeResolution();
}
[TearDown]
public virtual void TestTearDown()
public override void TearDown()
{
base.TearDown();
UmbracoExamineSearcher.DisableInitializationCheck = null;
BaseUmbracoIndexer.DisableInitializationCheck = null;
//reset all resolvers
ResolverCollection.ResetAll();
//reset resolution itself (though this should be taken care of by resetting any of the resolvers above)
Resolution.Reset();
}
}
}

45
src/Umbraco.Tests/UmbracoExamine/IndexInitializer.cs
View File

@@ -10,6 +10,7 @@ using Moq;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Membership;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Services;
using UmbracoExamine;
using UmbracoExamine.Config;
@@ -41,7 +42,48 @@ namespace Umbraco.Tests.UmbracoExamine
}
if (contentService == null)
{
contentService = Mock.Of<IContentService>();
long longTotalRecs;
int intTotalRecs;
var allRecs = dataService.ContentService.GetLatestContentByXPath("//*[@isDoc]")
.Root
.Elements()
.Select(x => Mock.Of<IContent>(
m =>
m.Id == (int)x.Attribute("id") &&
m.ParentId == (int)x.Attribute("parentID") &&
m.Level == (int)x.Attribute("level") &&
m.CreatorId == 0 &&
m.SortOrder == (int)x.Attribute("sortOrder") &&
m.CreateDate == (DateTime)x.Attribute("createDate") &&
m.UpdateDate == (DateTime)x.Attribute("updateDate") &&
m.Name == (string)x.Attribute("nodeName") &&
m.Path == (string)x.Attribute("path") &&
m.Properties == new PropertyCollection() &&
m.ContentType == Mock.Of<IContentType>(mt =>
mt.Alias == x.Name.LocalName &&
mt.Id == (int)x.Attribute("nodeType") &&
mt.Icon == "test")))
.ToArray();
contentService = Mock.Of<IContentService>(
x => x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<string>(), It.IsAny<Direction>(), It.IsAny<bool>(), It.IsAny<string>())
==
allRecs
&& x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<string>(), It.IsAny<Direction>(), It.IsAny<string>())
==
allRecs
&& x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<int>(), It.IsAny<int>(), out intTotalRecs, It.IsAny<string>(), It.IsAny<Direction>(), It.IsAny<string>())
==
allRecs
&& x.GetPagedDescendants(
It.IsAny<int>(), It.IsAny<long>(), It.IsAny<int>(), out longTotalRecs, It.IsAny<string>(), It.IsAny<Direction>(), It.IsAny<bool>(), It.IsAny<IQuery<IContent>>())
==
allRecs);
}
if (userService == null)
{
@@ -86,7 +128,6 @@ namespace Umbraco.Tests.UmbracoExamine
It.IsAny<int>(), It.IsAny<int>(), It.IsAny<int>(), out intTotalRecs, It.IsAny<string>(), It.IsAny<Direction>(), It.IsAny<string>())
==
allRecs);
}
if (dataTypeService == null)
{

31
src/Umbraco.Tests/UmbracoExamine/IndexTest.cs
View File

@@ -142,8 +142,6 @@ namespace Umbraco.Tests.UmbracoExamine
{
var s = (IndexSearcher)_searcher.GetSearcher();
//first delete all 'Content' (not media). This is done by directly manipulating the index with the Lucene API, not examine!
var contentTerm = new Term(LuceneIndexer.IndexTypeFieldName, IndexTypes.Content);
@@ -207,23 +205,24 @@ namespace Umbraco.Tests.UmbracoExamine
private Lucene.Net.Store.Directory _luceneDir;
public override void TestTearDown()
{
base.TestTearDown();
_luceneDir.Dispose();
public override void TearDown()
{
base.TearDown();
_luceneDir.Dispose();
UmbracoExamineSearcher.DisableInitializationCheck = null;
BaseUmbracoIndexer.DisableInitializationCheck = null;
}
public override void TestSetup()
{
base.TestSetup();
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
}
}
public override void Initialize()
{
base.Initialize();
_luceneDir = new RAMDirectory();
_indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir);
_indexer.RebuildIndex();
_searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir);
}
#endregion
}

30
src/UmbracoExamine/BaseUmbracoIndexer.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Net;
using System.Security;
@@ -361,10 +362,11 @@ namespace UmbracoExamine
/// <param name="type"></param>
protected override void PerformIndexAll(string type)
{
//NOTE: the logic below is ONLY used for published content, for media and members and non-published content, this method is overridden
//NOTE: the logic below is NOT used, this method is overridden
// and we query directly against the umbraco service layer.
// This is here for backwards compat only.
if (!SupportedTypes.Contains(type))
if (SupportedTypes.Contains(type) == false)
return;
var xPath = "//*[(number(@id) > 0 and (@isDoc or @nodeTypeAlias)){0}]"; //we'll add more filters to this below if needed
@@ -417,16 +419,11 @@ namespace UmbracoExamine
AddNodesToIndex(xPath, type);
}
/// <summary>
/// Returns an XDocument for the entire tree stored for the IndexType specified.
/// </summary>
/// <param name="xPath">The xpath to the node.</param>
/// <param name="type">The type of data to request from the data service.</param>
/// <returns>Either the Content or Media xml. If the type is not of those specified null is returned</returns>
[Obsolete("This method is not be used, it will be removed in future versions")]
[EditorBrowsable(EditorBrowsableState.Never)]
protected virtual XDocument GetXDocument(string xPath, string type)
{
//TODO: We need to get rid of this! it will now only ever be called for published content - but we're keeping the other
// logic here for backwards compatibility in case inheritors are calling this for some reason.
//TODO: We need to get rid of this! This does not get called by our code
if (type == IndexTypes.Content)
{
@@ -447,12 +444,9 @@ namespace UmbracoExamine
}
#endregion
#region Private
/// <summary>
/// Adds all nodes with the given xPath root.
/// </summary>
/// <param name="xPath">The x path.</param>
/// <param name="type">The type.</param>
[Obsolete("This method is not be used, it will be removed in future versions")]
[EditorBrowsable(EditorBrowsableState.Never)]
private void AddNodesToIndex(string xPath, string type)
{
// Get all the nodes of nodeTypeAlias == nodeTypeAlias
@@ -476,9 +470,5 @@ namespace UmbracoExamine
}
}
#endregion
}
}

62
src/UmbracoExamine/UmbracoContentIndexer.cs
View File

@@ -25,6 +25,7 @@ using UmbracoExamine.Config;
using Examine.LuceneEngine.Providers;
using Lucene.Net.Analysis;
using umbraco.BasePages;
using Umbraco.Core.Persistence.Querying;
using IContentService = Umbraco.Core.Services.IContentService;
using UmbracoExamine.LocalStorage;
using IMediaService = Umbraco.Core.Services.IMediaService;
@@ -348,50 +349,55 @@ namespace UmbracoExamine
protected override void PerformIndexAll(string type)
{
const int pageSize = 1000;
const int pageSize = 5000;
var pageIndex = 0;
switch (type)
{
case IndexTypes.Content:
if (this.SupportUnpublishedContent == false)
var contentParentId = -1;
if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
{
//use the base implementation which will use the published XML cache to perform the lookups
base.PerformIndexAll(type);
contentParentId = IndexerData.ParentNodeId.Value;
}
else
IContent[] content;
do
{
var contentParentId = -1;
if (IndexerData.ParentNodeId.HasValue && IndexerData.ParentNodeId.Value > 0)
long total;
IEnumerable<IContent> descendants;
if (SupportUnpublishedContent)
{
contentParentId = IndexerData.ParentNodeId.Value;
descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
}
IContent[] content;
do
else
{
long total;
var descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total);
//add the published filter
var qry = Query<IContent>.Builder.Where(x => x.Published == true);
//if specific types are declared we need to post filter them
//TODO: Update the service layer to join the cmsContentType table so we can query by content type too
if (IndexerData.IncludeNodeTypes.Any())
{
content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray();
}
else
{
content = descendants.ToArray();
}
descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "Path", Direction.Ascending, true, qry);
}
AddNodesToIndex(GetSerializedContent(content), type);
pageIndex++;
//if specific types are declared we need to post filter them
//TODO: Update the service layer to join the cmsContentType table so we can query by content type too
if (IndexerData.IncludeNodeTypes.Any())
{
content = descendants.Where(x => IndexerData.IncludeNodeTypes.Contains(x.ContentType.Alias)).ToArray();
}
else
{
content = descendants.ToArray();
}
AddNodesToIndex(GetSerializedContent(content), type);
pageIndex++;
} while (content.Length == pageSize);
} while (content.Length == pageSize);
}
break;
case IndexTypes.Media: