yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes: U4-5055 Umbraco update fails with large set of media content due to timeout in RebuildXmlStructures. Adds RebuildXmlStructures to content, media, member services as public APIs (though people won't really use them), the underlying repositories now rebuild these structures using a single transactions but queried by pages of 5000 which should reduce memory overhead if there's a ton of media, etc... Added tests for this too. Added CountPublished to ContentService too.

Browse Source
This commit is contained in:
Shannon
2014-10-23 18:31:08 +10:00
parent 238f5fc5f3
commit 44a39e7ca6
16 changed files with 625 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
View File

@@ -168,7 +168,86 @@ namespace Umbraco.Core.Persistence.Repositories
#endregion
#region Overrides of VersionableRepositoryBase<IContent>
public void RebuildXmlStructures(Func<IContent, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
{
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<DocumentDto>()
.On<ContentXmlDto, DocumentDto>(left => left.NodeId, right => right.NodeId);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
foreach (var id in contentTypeIds)
{
var id1 = id;
var subQuery = new Sql()
.Select("cmsDocument.nodeId")
.From<DocumentDto>()
.InnerJoin<ContentDto>()
.On<DocumentDto, ContentDto>(left => left.NodeId, right => right.NodeId)
.Where<DocumentDto>(dto => dto.Published)
.Where<ContentDto>(dto => dto.ContentTypeId == id1);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
{
var query = Query<IContent>.Builder.Where(x => x.Published == true);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
{
//copy local
var id = contentTypeId;
var query = Query<IContent>.Builder.Where(x => x.Published == true && x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
}
tr.Complete();
}
}
private void RebuildXmlStructuresProcessQuery(Func<IContent, XElement> serializer, IQuery<IContent> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = int.MinValue;
var processed = 0;
do
{
var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToString(SaveOptions.None) }).ToArray();
//bulk insert it into the database
Database.BulkInsertRecords(xmlItems, tr);
processed += xmlItems.Length;
pageIndex++;
} while (processed < total);
}
public override IContent GetByVersion(Guid versionId)
{
var sql = GetBaseQuery(false);
@@ -538,6 +617,14 @@ namespace Umbraco.Core.Persistence.Repositories
}
}
public int CountPublished()
{
var sql = GetBaseQuery(true).Where<NodeDto>(x => x.Trashed == false)
.Where<DocumentDto>(x => x.Published == true)
.Where<DocumentDto>(x => x.Newest == true);
return Database.ExecuteScalar<int>(sql);
}
public void ReplaceContentPermissions(EntityPermissionSet permissionSet)
{
var repo = new PermissionRepository<IContent>(UnitOfWork, _cacheHelper);

9
src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs
View File

@@ -11,6 +11,15 @@ namespace Umbraco.Core.Persistence.Repositories
{
public interface IContentRepository : IRepositoryVersionable<int, IContent>, IRecycleBinRepository<IContent>
{
/// <summary>
/// Get the count of published items
/// </summary>
/// <returns></returns>
/// <remarks>
/// We require this on the repo because the IQuery{IContent} cannot supply the 'newest' parameter
/// </remarks>
int CountPublished();
/// <summary>
/// Used to bulk update the permissions set for a content item. This will replace all permissions
/// assigned to an entity with a list of user id & permission pairs.

1
src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs
View File

@@ -9,6 +9,7 @@ namespace Umbraco.Core.Persistence.Repositories
{
public interface IMediaRepository : IRepositoryVersionable<int, IMedia>, IRecycleBinRepository<IMedia>
{
/// <summary>
/// Used to add/update published xml for the media item
/// </summary>

10
src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Xml.Linq;
using Umbraco.Core.Models.EntityBase;
namespace Umbraco.Core.Persistence.Repositories
@@ -12,6 +13,15 @@ namespace Umbraco.Core.Persistence.Repositories
public interface IRepositoryVersionable<TId, TEntity> : IRepositoryQueryable<TId, TEntity>
where TEntity : IAggregateRoot
{
/// <summary>
/// Rebuilds the xml structures for all TEntity if no content type ids are specified, otherwise rebuilds the xml structures
/// for only the content types specified
/// </summary>
/// <param name="serializer">The serializer to convert TEntity to Xml</param>
/// <param name="groupSize">Structures will be rebuilt in chunks of this size</param>
/// <param name="contentTypeIds"></param>
void RebuildXmlStructures(Func<TEntity, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null);
/// <summary>
/// Get the total count of entities
/// </summary>

85
src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
View File

@@ -16,6 +16,7 @@ using Umbraco.Core.Persistence.Factories;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Persistence.SqlSyntax;
using Umbraco.Core.Persistence.UnitOfWork;
using Umbraco.Core.Services;
namespace Umbraco.Core.Persistence.Repositories
{
@@ -172,6 +173,90 @@ namespace Umbraco.Core.Persistence.Repositories
return media;
}
public void RebuildXmlStructures(Func<IMedia, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
{
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == mediaObjectType);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
foreach (var id in contentTypeIds)
{
var id1 = id;
var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.InnerJoin<ContentDto>()
.On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == mediaObjectType)
.Where<ContentDto>(dto => dto.ContentTypeId == id1);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
{
var query = Query<IMedia>.Builder;
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
{
//copy local
var id = contentTypeId;
var query = Query<IMedia>.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
}
tr.Complete();
}
}
private void RebuildXmlStructuresProcessQuery(Func<IMedia, XElement> serializer, IQuery<IMedia> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = int.MinValue;
var processed = 0;
do
{
var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToString(SaveOptions.None) }).ToArray();
//bulk insert it into the database
Database.BulkInsertRecords(xmlItems, tr);
processed += xmlItems.Length;
pageIndex++;
} while (processed < total);
}
public void AddOrUpdateContentXml(IMedia content, Func<IMedia, XElement> xml)
{
var contentExists = Database.ExecuteScalar<int>("SELECT COUNT(nodeId) FROM cmsContentXml WHERE nodeId = @Id", new { Id = content.Id }) != 0;

84
src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
View File

@@ -418,6 +418,90 @@ namespace Umbraco.Core.Persistence.Repositories
#region Overrides of VersionableRepositoryBase<IMembershipUser>
public void RebuildXmlStructures(Func<IMember, XElement> serializer, int groupSize = 5000, IEnumerable<int> contentTypeIds = null)
{
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = Database.GetTransaction())
{
//Remove all the data first, if anything fails after this it's no problem the transaction will be reverted
if (contentTypeIds == null)
{
var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == memberObjectType);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
else
{
foreach (var id in contentTypeIds)
{
var id1 = id;
var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.InnerJoin<ContentDto>()
.On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == memberObjectType)
.Where<ContentDto>(dto => dto.ContentTypeId == id1);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
Database.Execute(deleteSql);
}
}
//now insert the data, again if something fails here, the whole transaction is reversed
if (contentTypeIds == null)
{
var query = Query<IMember>.Builder;
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
else
{
foreach (var contentTypeId in contentTypeIds)
{
//copy local
var id = contentTypeId;
var query = Query<IMember>.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false);
RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize);
}
}
tr.Complete();
}
}
private void RebuildXmlStructuresProcessQuery(Func<IMember, XElement> serializer, IQuery<IMember> query, Transaction tr, int pageSize)
{
var pageIndex = 0;
var total = int.MinValue;
var processed = 0;
do
{
var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending);
var xmlItems = (from descendant in descendants
let xml = serializer(descendant)
select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToString(SaveOptions.None) }).ToArray();
//bulk insert it into the database
Database.BulkInsertRecords(xmlItems, tr);
processed += xmlItems.Length;
pageIndex++;
} while (processed < total);
}
public override IMember GetByVersion(Guid versionId)
{
var sql = GetBaseQuery(false);

110
src/Umbraco.Core/Services/ContentService.cs
View File

@@ -77,6 +77,15 @@ namespace Umbraco.Core.Services
_userService = userService;
}
public int CountPublished(string contentTypeAlias = null)
{
var uow = _uowProvider.GetUnitOfWork();
using (var repository = _repositoryFactory.CreateContentRepository(uow))
{
return repository.CountPublished();
}
}
public int Count(string contentTypeAlias = null)
{
var uow = _uowProvider.GetUnitOfWork();
@@ -1460,6 +1469,27 @@ namespace Umbraco.Core.Services
return true;
}
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all documents
/// </summary>
/// <param name="contentTypeIds">
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all content
/// </param>
public void RebuildXmlStructures(params int[] contentTypeIds)
{
var uow = _uowProvider.GetUnitOfWork();
using (var repository = _repositoryFactory.CreateContentRepository(uow))
{
repository.RebuildXmlStructures(
content => _entitySerializer.Serialize(this, _dataTypeService, _userService, content),
contentTypeIds: contentTypeIds.Length == 0 ? null : contentTypeIds);
}
Audit.Add(AuditTypes.Publish, "ContentService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1);
}
#region Internal Methods
/// <summary>
@@ -1558,86 +1588,6 @@ namespace Umbraco.Core.Services
}
}
//TODO: WE should make a base class for ContentService and MediaService to share!
// currently we have this logic duplicated (nearly the same) for media types and soon to be member types
//TODO: This needs to be put into the ContentRepository, all CUD logic!
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all documents
/// </summary>
/// <param name="contentTypeIds">
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all content
/// </param>
/// <returns>True if publishing succeeded, otherwise False</returns>
private void RebuildXmlStructures(params int[] contentTypeIds)
{
using (new WriteLock(Locker))
{
var list = new List<IContent>();
var uow = _uowProvider.GetUnitOfWork();
//First we're going to get the data that needs to be inserted before clearing anything, this
//ensures that we don't accidentally leave the content xml table empty if something happens
//during the lookup process.
list.AddRange(contentTypeIds.Any() == false
? GetAllPublished()
: contentTypeIds.SelectMany(GetPublishedContentOfContentType));
var xmlItems = new List<ContentXmlDto>();
foreach (var c in list)
{
var xml = _entitySerializer.Serialize(this, _dataTypeService, _userService, c);
xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) });
}
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = uow.Database.GetTransaction())
{
if (contentTypeIds.Any() == false)
{
//Remove all Document records from the cmsContentXml table (DO NOT REMOVE Media/Members!) (based on inner join of cmsDocument)
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<DocumentDto>()
.On<ContentXmlDto, DocumentDto>(left => left.NodeId, right => right.NodeId);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
uow.Database.Execute(deleteSql);
}
else
{
foreach (var id in contentTypeIds)
{
//first we'll clear out the data from the cmsContentXml table for this type
var id1 = id;
var subQuery = new Sql()
.Select("cmsDocument.nodeId")
.From<DocumentDto>()
.InnerJoin<ContentDto>()
.On<DocumentDto, ContentDto>(left => left.NodeId, right => right.NodeId)
.Where<DocumentDto>(dto => dto.Published)
.Where<ContentDto>(dto => dto.ContentTypeId == id1);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
uow.Database.Execute(deleteSql);
}
}
//bulk insert it into the database
uow.Database.BulkInsertRecords(xmlItems, tr);
tr.Complete();
}
Audit.Add(AuditTypes.Publish, "RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1);
}
}
/// <summary>
/// Publishes a <see cref="IContent"/> object and all its children
/// </summary>

2
src/Umbraco.Core/Services/EntityXmlSerializer.cs
View File

@@ -83,7 +83,7 @@ namespace Umbraco.Core.Services
}
/// <summary>
/// Exports an <see cref="IMedia"/> item to xml as an <see cref="XElement"/>
/// Exports an <see cref="IMember"/> item to xml as an <see cref="XElement"/>
/// </summary>
/// <param name="dataTypeService"></param>
/// <param name="member">Member to export</param>

10
src/Umbraco.Core/Services/IContentService.cs
View File

@@ -12,6 +12,16 @@ namespace Umbraco.Core.Services
/// </summary>
public interface IContentService : IService
{
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all documents
/// </summary>
/// <param name="contentTypeIds">
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all content
/// </param>
void RebuildXmlStructures(params int[] contentTypeIds);
int CountPublished(string contentTypeAlias = null);
int Count(string contentTypeAlias = null);
int CountChildren(int parentId, string contentTypeAlias = null);
int CountDescendants(int parentId, string contentTypeAlias = null);

9
src/Umbraco.Core/Services/IMediaService.cs
View File

@@ -10,6 +10,15 @@ namespace Umbraco.Core.Services
/// </summary>
public interface IMediaService : IService
{
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all media
/// </summary>
/// <param name="contentTypeIds">
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all media
/// </param>
void RebuildXmlStructures(params int[] contentTypeIds);
int Count(string contentTypeAlias = null);
int CountChildren(int parentId, string contentTypeAlias = null);
int CountDescendants(int parentId, string contentTypeAlias = null);

9
src/Umbraco.Core/Services/IMemberService.cs
View File

@@ -12,6 +12,15 @@ namespace Umbraco.Core.Services
/// </summary>
public interface IMemberService : IMembershipMemberService
{
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all documents
/// </summary>
/// <param name="contentTypeIds">
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all content
/// </param>
void RebuildXmlStructures(params int[] contentTypeIds);
/// <summary>
/// Gets a list of paged <see cref="IMember"/> objects
/// </summary>

92
src/Umbraco.Core/Services/MediaService.cs
View File

@@ -637,7 +637,7 @@ namespace Umbraco.Core.Services
{
return;
}
media.ParentId = parentId;
if (media.Trashed)
{
@@ -661,7 +661,7 @@ namespace Umbraco.Core.Services
var parentLevel = media.Level;
var parentTrashed = media.Trashed;
var updatedDescendants = UpdatePropertiesOnChildren(children, parentPath, parentLevel, parentTrashed, moveInfo);
Save(updatedDescendants, userId,
Save(updatedDescendants, userId,
//no events!
false);
}
@@ -708,7 +708,7 @@ namespace Umbraco.Core.Services
media.ChangeTrashedState(true, Constants.System.RecycleBinMedia);
repository.AddOrUpdate(media);
//Loop through descendants to update their trash state, but ensuring structure by keeping the ParentId
foreach (var descendant in descendants)
{
@@ -1037,8 +1037,6 @@ namespace Umbraco.Core.Services
return true;
}
//TODO: This needs to be put into the MediaRepository, all CUD logic!
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all media
@@ -1047,85 +1045,17 @@ namespace Umbraco.Core.Services
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all media
/// </param>
/// <returns>True if publishing succeeded, otherwise False</returns>
internal void RebuildXmlStructures(params int[] contentTypeIds)
public void RebuildXmlStructures(params int[] contentTypeIds)
{
using (new WriteLock(Locker))
var uow = _uowProvider.GetUnitOfWork();
using (var repository = _repositoryFactory.CreateMediaRepository(uow))
{
var list = new List<IMedia>();
var uow = _uowProvider.GetUnitOfWork();
//First we're going to get the data that needs to be inserted before clearing anything, this
//ensures that we don't accidentally leave the content xml table empty if something happens
//during the lookup process.
if (contentTypeIds.Any() == false)
{
var rootMedia = GetRootMedia();
foreach (var media in rootMedia)
{
list.Add(media);
list.AddRange(GetDescendants(media));
}
}
else
{
list.AddRange(contentTypeIds.SelectMany(i => GetMediaOfMediaType(i).Where(media => media.Trashed == false)));
}
var xmlItems = new List<ContentXmlDto>();
foreach (var c in list)
{
var xml = _entitySerializer.Serialize(this, _dataTypeService, _userService, c);
xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) });
}
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = uow.Database.GetTransaction())
{
if (contentTypeIds.Any() == false)
{
var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == mediaObjectType);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
uow.Database.Execute(deleteSql);
}
else
{
foreach (var id in contentTypeIds)
{
var id1 = id;
var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.InnerJoin<ContentDto>()
.On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == mediaObjectType)
.Where<ContentDto>(dto => dto.ContentTypeId == id1);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
uow.Database.Execute(deleteSql);
}
}
//bulk insert it into the database
uow.Database.BulkInsertRecords(xmlItems, tr);
tr.Complete();
}
Audit.Add(AuditTypes.Publish, "RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1);
repository.RebuildXmlStructures(
media => _entitySerializer.Serialize(this, _dataTypeService, _userService, media),
contentTypeIds: contentTypeIds.Length == 0 ? null : contentTypeIds);
}
Audit.Add(AuditTypes.Publish, "MediaService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1);
}
/// <summary>

104
src/Umbraco.Core/Services/MemberService.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Threading;
using System.Web.Security;
using System.Xml.Linq;
using Umbraco.Core.Auditing;
using Umbraco.Core.Configuration;
using Umbraco.Core.Events;
using Umbraco.Core.Models;
@@ -589,6 +590,27 @@ namespace Umbraco.Core.Services
}
}
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all members
/// </summary>
/// <param name="memberTypeIds">
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all members = USE WITH CARE!
/// </param>
/// <returns>True if publishing succeeded, otherwise False</returns>
public void RebuildXmlStructures(params int[] memberTypeIds)
{
var uow = _uowProvider.GetUnitOfWork();
using (var repository = _repositoryFactory.CreateMemberRepository(uow))
{
repository.RebuildXmlStructures(
member => _entitySerializer.Serialize(_dataTypeService, member),
contentTypeIds: memberTypeIds.Length == 0 ? null : memberTypeIds);
}
Audit.Add(AuditTypes.Publish, "MemberService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1);
}
#endregion
#region IMembershipMemberService Implementation
@@ -1151,88 +1173,6 @@ namespace Umbraco.Core.Services
return contentType;
}
}
/// <summary>
/// Rebuilds all xml content in the cmsContentXml table for all members
/// </summary>
/// <param name="memberTypeIds">
/// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures
/// for all members = USE WITH CARE!
/// </param>
/// <returns>True if publishing succeeded, otherwise False</returns>
internal void RebuildXmlStructures(params int[] memberTypeIds)
{
using (new WriteLock(Locker))
{
var list = new List<IMember>();
var uow = _uowProvider.GetUnitOfWork();
//First we're going to get the data that needs to be inserted before clearing anything, this
//ensures that we don't accidentally leave the content xml table empty if something happens
//during the lookup process.
if (memberTypeIds.Any() == false)
{
list.AddRange(GetAllMembers());
}
else
{
list.AddRange(memberTypeIds.SelectMany(GetMembersByMemberType));
}
var xmlItems = new List<ContentXmlDto>();
foreach (var c in list)
{
var xml = _entitySerializer.Serialize(_dataTypeService, c);
xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) });
}
//Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too.
using (var tr = uow.Database.GetTransaction())
{
if (memberTypeIds.Any() == false)
{
//Remove all member records from the cmsContentXml table (DO NOT REMOVE Content/Media!)
var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == memberObjectType);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
uow.Database.Execute(deleteSql);
}
else
{
foreach (var id in memberTypeIds)
{
var id1 = id;
var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member);
var subQuery = new Sql()
.Select("DISTINCT cmsContentXml.nodeId")
.From<ContentXmlDto>()
.InnerJoin<NodeDto>()
.On<ContentXmlDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.InnerJoin<ContentDto>()
.On<ContentDto, NodeDto>(left => left.NodeId, right => right.NodeId)
.Where<NodeDto>(dto => dto.NodeObjectType == memberObjectType)
.Where<ContentDto>(dto => dto.ContentTypeId == id1);
var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery);
uow.Database.Execute(deleteSql);
}
}
//bulk insert it into the database
uow.Database.BulkInsertRecords(xmlItems, tr);
tr.Complete();
}
}
}
#region Event Handlers

113
src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Models;
@@ -46,6 +47,118 @@ namespace Umbraco.Tests.Persistence.Repositories
return repository;
}
[Test]
public void Rebuild_All_Xml_Structures()
{
var provider = new PetaPocoUnitOfWorkProvider();
var unitOfWork = provider.GetUnitOfWork();
ContentTypeRepository contentTypeRepository;
using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
{
var contentType1 = MockedContentTypes.CreateSimpleContentType("Textpage1", "Textpage1");
contentTypeRepository.AddOrUpdate(contentType1);
var allCreated = new List<IContent>();
for (var i = 0; i < 100; i++)
{
//These will be non-published so shouldn't show up
var c1 = MockedContent.CreateSimpleContent(contentType1);
repository.AddOrUpdate(c1);
allCreated.Add(c1);
}
for (var i = 0; i < 100; i++)
{
var c1 = MockedContent.CreateSimpleContent(contentType1);
c1.ChangePublishedState(PublishedState.Published);
repository.AddOrUpdate(c1);
allCreated.Add(c1);
}
unitOfWork.Commit();
//now create some versions of this content - this shouldn't affect the xml structures saved
for (int i = 0; i < allCreated.Count; i++)
{
allCreated[i].Name = "blah" + i;
repository.AddOrUpdate(allCreated[i]);
}
unitOfWork.Commit();
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
repository.RebuildXmlStructures(media => new XElement("test"), 10);
Assert.AreEqual(100, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
}
}
[Test]
public void Rebuild_All_Xml_Structures_For_Content_Type()
{
var provider = new PetaPocoUnitOfWorkProvider();
var unitOfWork = provider.GetUnitOfWork();
ContentTypeRepository contentTypeRepository;
using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
{
var contentType1 = MockedContentTypes.CreateSimpleContentType("Textpage1", "Textpage1");
var contentType2 = MockedContentTypes.CreateSimpleContentType("Textpage2", "Textpage2");
var contentType3 = MockedContentTypes.CreateSimpleContentType("Textpage3", "Textpage3");
contentTypeRepository.AddOrUpdate(contentType1);
contentTypeRepository.AddOrUpdate(contentType2);
contentTypeRepository.AddOrUpdate(contentType3);
var allCreated = new List<IContent>();
for (var i = 0; i < 30; i++)
{
//These will be non-published so shouldn't show up
var c1 = MockedContent.CreateSimpleContent(contentType1);
repository.AddOrUpdate(c1);
allCreated.Add(c1);
}
for (var i = 0; i < 30; i++)
{
var c1 = MockedContent.CreateSimpleContent(contentType1);
c1.ChangePublishedState(PublishedState.Published);
repository.AddOrUpdate(c1);
allCreated.Add(c1);
}
for (var i = 0; i < 30; i++)
{
var c1 = MockedContent.CreateSimpleContent(contentType2);
c1.ChangePublishedState(PublishedState.Published);
repository.AddOrUpdate(c1);
allCreated.Add(c1);
}
for (var i = 0; i < 30; i++)
{
var c1 = MockedContent.CreateSimpleContent(contentType3);
c1.ChangePublishedState(PublishedState.Published);
repository.AddOrUpdate(c1);
allCreated.Add(c1);
}
unitOfWork.Commit();
//now create some versions of this content - this shouldn't affect the xml structures saved
for (int i = 0; i < allCreated.Count; i++)
{
allCreated[i].Name = "blah" + i;
repository.AddOrUpdate(allCreated[i]);
}
unitOfWork.Commit();
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
repository.RebuildXmlStructures(media => new XElement("test"), 10, contentTypeIds: new[] { contentType1.Id, contentType2.Id });
Assert.AreEqual(60, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
}
}
[Test]
public void Ensures_Permissions_Are_Set_If_Parent_Entity_Permissions_Exist()
{

70
src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Xml.Linq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Models;
@@ -9,6 +10,7 @@ using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.Caching;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Persistence.Repositories;
using Umbraco.Core.Persistence.SqlSyntax;
using Umbraco.Core.Persistence.UnitOfWork;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.TestHelpers.Entities;
@@ -38,6 +40,74 @@ namespace Umbraco.Tests.Persistence.Repositories
return repository;
}
[Test]
public void Rebuild_All_Xml_Structures()
{
var provider = new PetaPocoUnitOfWorkProvider();
var unitOfWork = provider.GetUnitOfWork();
MediaTypeRepository mediaTypeRepository;
using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository))
{
var mediaType = mediaTypeRepository.Get(1032);
for (var i = 0; i < 100; i++)
{
var image = MockedMedia.CreateMediaImage(mediaType, -1);
repository.AddOrUpdate(image);
}
unitOfWork.Commit();
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
repository.RebuildXmlStructures(media => new XElement("test"), 10);
Assert.AreEqual(103, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
}
}
[Test]
public void Rebuild_All_Xml_Structures_For_Content_Type()
{
var provider = new PetaPocoUnitOfWorkProvider();
var unitOfWork = provider.GetUnitOfWork();
MediaTypeRepository mediaTypeRepository;
using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository))
{
var imageMediaType = mediaTypeRepository.Get(1032);
var fileMediaType = mediaTypeRepository.Get(1033);
var folderMediaType = mediaTypeRepository.Get(1031);
for (var i = 0; i < 30; i++)
{
var image = MockedMedia.CreateMediaImage(imageMediaType, -1);
repository.AddOrUpdate(image);
}
for (var i = 0; i < 30; i++)
{
var file = MockedMedia.CreateMediaFile(fileMediaType, -1);
repository.AddOrUpdate(file);
}
for (var i = 0; i < 30; i++)
{
var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1);
repository.AddOrUpdate(folder);
}
unitOfWork.Commit();
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
repository.RebuildXmlStructures(media => new XElement("test"), 10, contentTypeIds: new[] {1032, 1033});
Assert.AreEqual(62, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
}
}
[Test]
public void Can_Instantiate_Repository_From_Resolver()
{

75
src/Umbraco.Tests/Persistence/Repositories/MemberRepositoryTest.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Xml.Linq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Models;
@@ -39,6 +40,76 @@ namespace Umbraco.Tests.Persistence.Repositories
return repository;
}
[Test]
public void Rebuild_All_Xml_Structures()
{
var provider = new PetaPocoUnitOfWorkProvider();
var unitOfWork = provider.GetUnitOfWork();
MemberTypeRepository memberTypeRepository;
MemberGroupRepository memberGroupRepository;
using (var repository = CreateRepository(unitOfWork, out memberTypeRepository, out memberGroupRepository))
{
var memberType1 = CreateTestMemberType();
for (var i = 0; i < 100; i++)
{
var member = MockedMember.CreateSimpleMember(memberType1, "blah" + i, "blah" + i + "@example.com", "blah", "blah" + i);
repository.AddOrUpdate(member);
}
unitOfWork.Commit();
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
repository.RebuildXmlStructures(media => new XElement("test"), 10);
Assert.AreEqual(100, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
}
}
[Test]
public void Rebuild_All_Xml_Structures_For_Content_Type()
{
var provider = new PetaPocoUnitOfWorkProvider();
var unitOfWork = provider.GetUnitOfWork();
MemberTypeRepository memberTypeRepository;
MemberGroupRepository memberGroupRepository;
using (var repository = CreateRepository(unitOfWork, out memberTypeRepository, out memberGroupRepository))
{
var memberType1 = CreateTestMemberType("mt1");
var memberType2 = CreateTestMemberType("mt2");
var memberType3 = CreateTestMemberType("mt3");
for (var i = 0; i < 30; i++)
{
var member = MockedMember.CreateSimpleMember(memberType1, "b1lah" + i, "b1lah" + i + "@example.com", "b1lah", "b1lah" + i);
repository.AddOrUpdate(member);
}
for (var i = 0; i < 30; i++)
{
var member = MockedMember.CreateSimpleMember(memberType2, "b2lah" + i, "b2lah" + i + "@example.com", "b2lah", "b2lah" + i);
repository.AddOrUpdate(member);
}
for (var i = 0; i < 30; i++)
{
var member = MockedMember.CreateSimpleMember(memberType3, "b3lah" + i, "b3lah" + i + "@example.com", "b3lah", "b3lah" + i);
repository.AddOrUpdate(member);
}
unitOfWork.Commit();
//delete all xml
unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
repository.RebuildXmlStructures(media => new XElement("test"), 10, contentTypeIds: new[] { memberType1.Id, memberType2.Id });
Assert.AreEqual(60, unitOfWork.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM cmsContentXml"));
}
}
[Test]
public void Can_Instantiate_Repository_From_Resolver()
{
@@ -297,7 +368,7 @@ namespace Umbraco.Tests.Persistence.Repositories
}
}
private IMemberType CreateTestMemberType()
private IMemberType CreateTestMemberType(string alias = null)
{
var provider = new PetaPocoUnitOfWorkProvider();
var unitOfWork = provider.GetUnitOfWork();
@@ -305,7 +376,7 @@ namespace Umbraco.Tests.Persistence.Repositories
MemberGroupRepository memberGroupRepository;
using (var repository = CreateRepository(unitOfWork, out memberTypeRepository, out memberGroupRepository))
{
var memberType = MockedContentTypes.CreateSimpleMemberType();
var memberType = MockedContentTypes.CreateSimpleMemberType(alias);
memberTypeRepository.AddOrUpdate(memberType);
unitOfWork.Commit();
return memberType;