yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch origin/dev-v7 into temp-u4-6994

Browse Source
This commit is contained in:
Stephan
2016-11-14 18:31:36 +01:00
parent feb120cbeb 1c036b1a11
commit bbe1cb1813
6 changed files with 158 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
View File

@@ -165,86 +165,50 @@ namespace Umbraco.Core.Persistence.Repositories
#region Overrides of VersionableRepositoryBase<IContent>
public void RebuildXmlStructures(Func<IContent, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
public void RebuildXmlStructures(Func<IContent, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
{
// the previous way of doing this was to run it all in one big transaction,
// and to bulk-insert groups of xml rows - which works, until the transaction
// times out - and besides, because v7 transactions are ReadCommited, it does
// not bring much safety - so this reverts to updating each record individually,
// and it may be slower in the end, but should be more resilient.
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
var baseId = 0;
var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
while (true)
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var subQuery = new Sql()
.Select("id")
.From<NodeDto>(SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// get the next group of nodes
var query = GetBaseQuery(false);
if (contentTypeIdsA.Length > 0)
query = query
.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
query = query
.Where<NodeDto>(x => x.NodeId > baseId && x.Trashed == false)
.Where<DocumentDto>(x => x.Published)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
var subQuery = new Sql()
.Select("umbracoNode.id as nodeId")
.From<ContentDto>(SqlSyntax)
.InnerJoin<NodeDto>(SqlSyntax)
.On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
.WhereIn<ContentDto>(dto => dto.ContentTypeId, contentTypeIds, SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// no more nodes, break
if (xmlItems.Count == 0) break;
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
foreach (var xmlItem in xmlItems)
{
var query = Query<IContent>.Builder.Where(x => x.Published == true);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
try
{
//copy local
var id = contentTypeId;
var query = Query<IContent>.Builder.Where(x => x.Published == true && x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
// InsertOrUpdate tries to update first, which is good since it is what
// should happen in most cases, then it tries to insert, and it should work
// unless the node has been deleted, and we just report the exception
Database.InsertOrUpdate(xmlItem);
}
catch (Exception e)
{
Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
}
}
tr.Complete();
baseId = xmlItems.Last().NodeId;
}
}
private void RebuildXmlStructuresProcessQuery(Func<IContent, XElement> serializer, IQuery<IContent> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
long total;
var processed = 0;
do
{
//NOTE: This is an important call, we cannot simply make a call to:
// GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending);
// because that method is used to query 'latest' content items where in this case we don't necessarily
// want latest content items because a pulished content item might not actually be the latest.
// see: http://issues.umbraco.org/issue/U4-6322 & http://issues.umbraco.org/issue/U4-5982
var descendants = GetPagedResultsByQuery<DocumentDto, Content>(query, pageIndex, pageSize, out total,
new Tuple<string, string>("cmsDocument", "nodeId"),
sql => ProcessQuery(sql), "Path", Direction.Ascending, true);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToDataString() });
//bulk insert it into the database
var count = Database.BulkInsertRecords(xmlItems, tr, SqlSyntax);
processed += count;
pageIndex++;
} while (processed < total);
}
public override IEnumerable<IContent> GetAllVersions(int id)
{

95
src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
View File

@@ -227,80 +227,51 @@ namespace Umbraco.Core.Persistence.Repositories
return media;
}
public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
{
// the previous way of doing this was to run it all in one big transaction,
// and to bulk-insert groups of xml rows - which works, until the transaction
// times out - and besides, because v7 transactions are ReadCommited, it does
// not bring much safety - so this reverts to updating each record individually,
// and it may be slower in the end, but should be more resilient.
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
var baseId = 0;
var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
while (true)
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var subQuery = new Sql()
.Select("id")
.From<NodeDto>(SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// get the next group of nodes
var query = GetBaseQuery(false);
if (contentTypeIdsA.Length > 0)
query = query
.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
query = query
.Where<NodeDto>(x => x.NodeId > baseId)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
var subQuery = new Sql()
.Select("umbracoNode.id as nodeId")
.From<ContentDto>(SqlSyntax)
.InnerJoin<NodeDto>(SqlSyntax)
.On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
.WhereIn<ContentDto>(dto => dto.ContentTypeId, contentTypeIds, SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// no more nodes, break
if (xmlItems.Count == 0) break;
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
foreach (var xmlItem in xmlItems)
{
var query = Query<IMedia>.Builder;
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
try
{
//copy local
var id = contentTypeId;
var query = Query<IMedia>.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
// InsertOrUpdate tries to update first, which is good since it is what
// should happen in most cases, then it tries to insert, and it should work
// unless the node has been deleted, and we just report the exception
Database.InsertOrUpdate(xmlItem);
}
catch (Exception e)
{
Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
}
}
tr.Complete();
baseId = xmlItems.Last().NodeId;
}
}
private void RebuildXmlStructuresProcessQuery(Func<IMedia, XElement> serializer, IQuery<IMedia> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = long.MinValue;
var processed = 0;
do
{
var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending, true);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToDataString() }).ToArray();
//bulk insert it into the database
Database.BulkInsertRecords(xmlItems, tr);
processed += xmlItems.Length;
pageIndex++;
} while (processed < total);
}
public void AddOrUpdateContentXml(IMedia content, Func<IMedia, XElement> xml)
{
_contentXmlRepository.AddOrUpdate(new ContentXmlEntity<IMedia>(content, xml));

95
src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
View File

@@ -383,80 +383,51 @@ namespace Umbraco.Core.Persistence.Repositories
return ProcessQuery(sql, true);
}
public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
{
// the previous way of doing this was to run it all in one big transaction,
// and to bulk-insert groups of xml rows - which works, until the transaction
// times out - and besides, because v7 transactions are ReadCommited, it does
// not bring much safety - so this reverts to updating each record individually,
// and it may be slower in the end, but should be more resilient.
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
var baseId = 0;
var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
while (true)
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var subQuery = new Sql()
.Select("id")
.From<NodeDto>(SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// get the next group of nodes
var query = GetBaseQuery(false);
if (contentTypeIdsA.Length > 0)
query = query
.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
query = query
.Where<NodeDto>(x => x.NodeId > baseId)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
var subQuery = new Sql()
.Select("umbracoNode.id as nodeId")
.From<ContentDto>(SqlSyntax)
.InnerJoin<NodeDto>(SqlSyntax)
.On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
.WhereIn<ContentDto>(dto => dto.ContentTypeId, contentTypeIds, SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// no more nodes, break
if (xmlItems.Count == 0) break;
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
foreach (var xmlItem in xmlItems)
{
var query = Query<IMember>.Builder;
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
try
{
//copy local
var id = contentTypeId;
var query = Query<IMember>.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
// InsertOrUpdate tries to update first, which is good since it is what
// should happen in most cases, then it tries to insert, and it should work
// unless the node has been deleted, and we just report the exception
Database.InsertOrUpdate(xmlItem);
}
catch (Exception e)
{
Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
}
}
tr.Complete();
baseId = xmlItems.Last().NodeId;
}
}
private void RebuildXmlStructuresProcessQuery(Func<IMember, XElement> serializer, IQuery<IMember> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = long.MinValue;
var processed = 0;
do
{
var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending, true);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToDataString() }).ToArray();
//bulk insert it into the database
Database.BulkInsertRecords(xmlItems, tr);
processed += xmlItems.Length;
pageIndex++;
} while (processed < total);
}
public override IMember GetByVersion(Guid versionId)
{
var sql = GetBaseQuery(false);

36
src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
View File

@@ -101,7 +101,7 @@ namespace Umbraco.Tests.Persistence.Repositories
for (int i = 0; i < allCreated.Count; i++)
{
allCreated[i].Name = "blah" + i;
//IMPORTANT testing note here: We need to changed the published state here so that
//IMPORTANT testing note here: We need to changed the published state here so that
// it doesn't automatically think this is simply publishing again - this forces the latest
// version to be Saved and not published
allCreated[i].ChangePublishedState(PublishedState.Saved);
@@ -109,7 +109,7 @@ namespace Umbraco.Tests.Persistence.Repositories
}
unitOfWork.Commit();
//delete all xml
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
@@ -156,7 +156,7 @@ namespace Umbraco.Tests.Persistence.Repositories
}
unitOfWork.Commit();
//delete all xml
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
@@ -221,7 +221,7 @@ namespace Umbraco.Tests.Persistence.Repositories
}
unitOfWork.Commit();
//delete all xml
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
@@ -234,10 +234,10 @@ namespace Umbraco.Tests.Persistence.Repositories
/// <summary>
/// This test ensures that when property values using special database fields are saved, the actual data in the
/// object being stored is also transformed in the same way as the data being stored in the database is.
/// Before you would see that ex: a decimal value being saved as 100 or "100", would be that exact value in the
/// Before you would see that ex: a decimal value being saved as 100 or "100", would be that exact value in the
/// object, but the value saved to the database was actually 100.000000.
/// When querying the database for the value again - the value would then differ from what is in the object.
/// This caused inconsistencies between saving+publishing and simply saving and then publishing, due to the former
/// When querying the database for the value again - the value would then differ from what is in the object.
/// This caused inconsistencies between saving+publishing and simply saving and then publishing, due to the former
/// sending the non-transformed data directly on to publishing.
/// </summary>
[Test]
@@ -269,10 +269,10 @@ namespace Umbraco.Tests.Persistence.Repositories
var contentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage", propertyTypeCollection);
contentTypeRepository.AddOrUpdate(contentType);
unitOfWork.Commit();
// Int and decimal values are passed in as strings as they would be from the backoffice UI
var textpage = MockedContent.CreateSimpleContentWithSpecialDatabaseTypes(contentType, "test@umbraco.org", -1, "100", "150", dateValue);
// Act
repository.AddOrUpdate(textpage);
unitOfWork.Commit();
@@ -280,7 +280,7 @@ namespace Umbraco.Tests.Persistence.Repositories
// Assert
Assert.That(contentType.HasIdentity, Is.True);
Assert.That(textpage.HasIdentity, Is.True);
var persistedTextpage = repository.Get(textpage.Id);
Assert.That(persistedTextpage.Name, Is.EqualTo(textpage.Name));
Assert.AreEqual(100m, persistedTextpage.GetValue(decimalPropertyAlias));
@@ -342,12 +342,12 @@ namespace Umbraco.Tests.Persistence.Repositories
contentTypeRepository.AddOrUpdate(contentType);
repository.AddOrUpdate(textpage);
unitOfWork.Commit();
// Assert
Assert.That(contentType.HasIdentity, Is.True);
Assert.That(textpage.HasIdentity, Is.True);
}
}
@@ -631,9 +631,9 @@ namespace Umbraco.Tests.Persistence.Repositories
// Act
var query = Query<IContent>.Builder.Where(x => x.Name.Contains("Text"));
long totalRecords;
try
{
{
DatabaseContext.Database.EnableSqlTrace = true;
DatabaseContext.Database.EnableSqlCount();
@@ -643,14 +643,14 @@ namespace Umbraco.Tests.Persistence.Repositories
Assert.AreEqual(2, result.Count());
result = repository.GetPagedResultsByQuery(query, 1, 2, out totalRecords, "title", Direction.Ascending, false);
Assert.AreEqual(1, result.Count());
}
finally
{
{
DatabaseContext.Database.EnableSqlTrace = false;
DatabaseContext.Database.DisableSqlCount();
}
}
}
}
@@ -761,7 +761,7 @@ namespace Umbraco.Tests.Persistence.Repositories
// Act
var query = Query<IContent>.Builder.Where(x => x.Level == 2);
var filterQuery = Query<IContent>.Builder.Where(x => x.Name.Contains("Page 2"));
long totalRecords;
var result = repository.GetPagedResultsByQuery(query, 0, 1, out totalRecords, "Name", Direction.Ascending, true, filterQuery);

38
src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using Moq;
@@ -59,7 +60,7 @@ namespace Umbraco.Tests.Persistence.Repositories
}
unitOfWork.Commit();
//delete all xml
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
@@ -69,6 +70,39 @@ namespace Umbraco.Tests.Persistence.Repositories
}
}
[Test]
public void Rebuild_Some_Xml_Structures()
{
var provider = new PetaPocoUnitOfWorkProvider(Logger);
var unitOfWork = provider.GetUnitOfWork();
MediaTypeRepository mediaTypeRepository;
using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository))
{
var mediaType = mediaTypeRepository.Get(1032);
IMedia img50 = null;
for (var i = 0; i < 100; i++)
{
var image = MockedMedia.CreateMediaImage(mediaType, -1);
repository.AddOrUpdate(image);
if (i == 50) img50 = image;
}
unitOfWork.Commit();
// assume this works (see other test)
repository.RebuildXmlStructures(media => new XElement("test"), 10);
//delete some xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml WHERE nodeId < " + img50.Id);
Assert.AreEqual(50, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
repository.RebuildXmlStructures(media => new XElement("test"), 10);
Assert.AreEqual(103, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
}
}
[Test]
public void Rebuild_All_Xml_Structures_For_Content_Type()
{
@@ -99,7 +133,7 @@ namespace Umbraco.Tests.Persistence.Repositories
}
unitOfWork.Commit();
//delete all xml
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));

8
src/Umbraco.Tests/Persistence/Repositories/MemberRepositoryTest.cs
View File

@@ -54,7 +54,7 @@ namespace Umbraco.Tests.Persistence.Repositories
using (var repository = CreateRepository(unitOfWork, out memberTypeRepository, out memberGroupRepository))
{
var memberType1 = CreateTestMemberType();
for (var i = 0; i < 100; i++)
{
var member = MockedMember.CreateSimpleMember(memberType1, "blah" + i, "blah" + i + "@example.com", "blah", "blah" + i);
@@ -103,7 +103,7 @@ namespace Umbraco.Tests.Persistence.Repositories
}
unitOfWork.Commit();
//delete all xml
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
@@ -216,7 +216,7 @@ namespace Umbraco.Tests.Persistence.Repositories
var sut = repository.Get(member.Id);
Assert.That(sut, Is.Not.Null);
Assert.That(sut.HasIdentity, Is.True);
Assert.That(sut.HasIdentity, Is.True);
Assert.That(sut.Properties.Any(x => x.HasIdentity == false || x.Id == 0), Is.False);
Assert.That(sut.Name, Is.EqualTo("Johnny Hefty"));
Assert.That(sut.Email, Is.EqualTo("johnny@example.com"));
@@ -350,7 +350,7 @@ namespace Umbraco.Tests.Persistence.Repositories
{
memberType = MockedContentTypes.CreateSimpleMemberType();
memberTypeRepository.AddOrUpdate(memberType);
unitOfWork.Commit();
unitOfWork.Commit();
}
var member = MockedMember.CreateSimpleMember(memberType, name ?? "Johnny Hefty", email ?? "johnny@example.com", password ?? "123", username ?? "hefty", key);