yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch origin/dev-v7 into temp-U4-9105

Browse Source
This commit is contained in:
Stephan
2016-11-14 18:45:56 +01:00
parent 2bc07ad4cc 1c036b1a11
commit 539e7c80bf
6 changed files with 157 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
View File

@@ -179,86 +179,51 @@ namespace Umbraco.Core.Persistence.Repositories
#region Overrides of VersionableRepositoryBase<IContent>
public void RebuildXmlStructures(Func<IContent, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
public void RebuildXmlStructures(Func<IContent, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
{
// the previous way of doing this was to run it all in one big transaction,
// and to bulk-insert groups of xml rows - which works, until the transaction
// times out - and besides, because v7 transactions are ReadCommited, it does
// not bring much safety - so this reverts to updating each record individually,
// and it may be slower in the end, but should be more resilient.
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
var baseId = 0;
var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
while (true)
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var subQuery = new Sql()
.Select("id")
.From<NodeDto>(SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// get the next group of nodes
var query = GetBaseQuery(false);
if (contentTypeIdsA.Length > 0)
query = query
.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
query = query
.Where<NodeDto>(x => x.NodeId > baseId && x.Trashed == false)
.Where<DocumentDto>(x => x.Published)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
var subQuery = new Sql()
.Select("umbracoNode.id as nodeId")
.From<ContentDto>(SqlSyntax)
.InnerJoin<NodeDto>(SqlSyntax)
.On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
.WhereIn<ContentDto>(dto => dto.ContentTypeId, contentTypeIds, SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// no more nodes, break
if (xmlItems.Count == 0) break;
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
foreach (var xmlItem in xmlItems)
{
RebuildXmlStructuresProcessQuery(serializer, _publishedQuery, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
try
{
//copy local
var id = contentTypeId;
var query = Query<IContent>.Builder.Where(x => x.Published == true && x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
// should happen in most cases, then it tries to insert, and it should work
// unless the node has been deleted, and we just report the exception
Database.InsertOrUpdate(xmlItem);
}
catch (Exception e)
{
Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
}
}
tr.Complete();
baseId = xmlItems.Last().NodeId;
}
}
private void RebuildXmlStructuresProcessQuery(Func<IContent, XElement> serializer, IQuery<IContent> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = long.MinValue;
var processed = 0;
do
{
//NOTE: This is an important call, we cannot simply make a call to:
// GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending);
// because that method is used to query 'latest' content items where in this case we don't necessarily
// want latest content items because a pulished content item might not actually be the latest.
// see: http://issues.umbraco.org/issue/U4-6322 & http://issues.umbraco.org/issue/U4-5982
var descendants = GetPagedResultsByQuery<DocumentDto, Content>(query, pageIndex, pageSize, out total,
new Tuple<string, string>("cmsDocument", "nodeId"),
ProcessQuery, "Path", Direction.Ascending, true);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToDataString() });
//bulk insert it into the database
var count = Database.BulkInsertRecords(xmlItems, tr, SqlSyntax);
processed += count;
pageIndex++;
} while (processed < total);
}
public override IContent GetByVersion(Guid versionId)
{
var sql = GetBaseQuery(false);

95
src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
View File

@@ -163,80 +163,51 @@ namespace Umbraco.Core.Persistence.Repositories
return media;
}
public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
{
// the previous way of doing this was to run it all in one big transaction,
// and to bulk-insert groups of xml rows - which works, until the transaction
// times out - and besides, because v7 transactions are ReadCommited, it does
// not bring much safety - so this reverts to updating each record individually,
// and it may be slower in the end, but should be more resilient.
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
var baseId = 0;
var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
while (true)
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var subQuery = new Sql()
.Select("id")
.From<NodeDto>(SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// get the next group of nodes
var query = GetBaseQuery(false);
if (contentTypeIdsA.Length > 0)
query = query
.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
query = query
.Where<NodeDto>(x => x.NodeId > baseId)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
var subQuery = new Sql()
.Select("umbracoNode.id as nodeId")
.From<ContentDto>(SqlSyntax)
.InnerJoin<NodeDto>(SqlSyntax)
.On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
.WhereIn<ContentDto>(dto => dto.ContentTypeId, contentTypeIds, SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// no more nodes, break
if (xmlItems.Count == 0) break;
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
foreach (var xmlItem in xmlItems)
{
var query = Query<IMedia>.Builder;
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
try
{
//copy local
var id = contentTypeId;
var query = Query<IMedia>.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
// InsertOrUpdate tries to update first, which is good since it is what
// should happen in most cases, then it tries to insert, and it should work
// unless the node has been deleted, and we just report the exception
Database.InsertOrUpdate(xmlItem);
}
catch (Exception e)
{
Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
}
}
tr.Complete();
baseId = xmlItems.Last().NodeId;
}
}
private void RebuildXmlStructuresProcessQuery(Func<IMedia, XElement> serializer, IQuery<IMedia> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = long.MinValue;
var processed = 0;
do
{
var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending, true);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToDataString() }).ToArray();
//bulk insert it into the database
Database.BulkInsertRecords(xmlItems, tr);
processed += xmlItems.Length;
pageIndex++;
} while (processed < total);
}
public void AddOrUpdateContentXml(IMedia content, Func<IMedia, XElement> xml)
{
_contentXmlRepository.AddOrUpdate(new ContentXmlEntity<IMedia>(content, xml));

95
src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
View File

@@ -380,80 +380,51 @@ namespace Umbraco.Core.Persistence.Repositories
#region Overrides of VersionableRepositoryBase<IMembershipUser>
public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 200, IEnumerable<int> contentTypeIds = null)
{
// the previous way of doing this was to run it all in one big transaction,
// and to bulk-insert groups of xml rows - which works, until the transaction
// times out - and besides, because v7 transactions are ReadCommited, it does
// not bring much safety - so this reverts to updating each record individually,
// and it may be slower in the end, but should be more resilient.
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
var baseId = 0;
var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray();
while (true)
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var subQuery = new Sql()
.Select("id")
.From<NodeDto>(SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// get the next group of nodes
var query = GetBaseQuery(false);
if (contentTypeIdsA.Length > 0)
query = query
.WhereIn<ContentDto>(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
query = query
.Where<NodeDto>(x => x.NodeId > baseId)
.OrderBy<NodeDto>(x => x.NodeId, SqlSyntax);
var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize))
.Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() })
.ToList();
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
var subQuery = new Sql()
.Select("umbracoNode.id as nodeId")
.From<ContentDto>(SqlSyntax)
.InnerJoin<NodeDto>(SqlSyntax)
.On<ContentDto, NodeDto>(SqlSyntax, left => left.NodeId, right => right.NodeId)
.WhereIn<ContentDto>(dto => dto.ContentTypeId, contentTypeIds, SqlSyntax)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId);
// no more nodes, break
if (xmlItems.Count == 0) break;
var deleteSql = SqlSyntax.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
foreach (var xmlItem in xmlItems)
{
var query = Query<IMember>.Builder;
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
try
{
//copy local
var id = contentTypeId;
var query = Query<IMember>.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
// InsertOrUpdate tries to update first, which is good since it is what
// should happen in most cases, then it tries to insert, and it should work
// unless the node has been deleted, and we just report the exception
Database.InsertOrUpdate(xmlItem);
}
catch (Exception e)
{
Logger.Error<MediaRepository>("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
}
}
tr.Complete();
baseId = xmlItems.Last().NodeId;
}
}
private void RebuildXmlStructuresProcessQuery(Func<IMember, XElement> serializer, IQuery<IMember> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = long.MinValue;
var processed = 0;
do
{
var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending, true);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToDataString() }).ToArray();
//bulk insert it into the database
Database.BulkInsertRecords(xmlItems, tr);
processed += xmlItems.Length;
pageIndex++;
} while (processed < total);
}
public override IMember GetByVersion(Guid versionId)
{
var sql = GetBaseQuery(false);