From 44a39e7ca6765bbf2f35c2f62d741447b49a2efa Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 23 Oct 2014 18:31:08 +1000 Subject: [PATCH] Fixes: U4-5055 Umbraco update fails with large set of media content due to timeout in RebuildXmlStructures. Adds RebuildXmlStructures to content, media, member services as public APIs (though people won't really use them), the underlying repositories now rebuild these structures using a single transactions but queried by pages of 5000 which should reduce memory overhead if there's a ton of media, etc... Added tests for this too. Added CountPublished to ContentService too. --- .../Repositories/ContentRepository.cs | 89 +++++++++++++- .../Interfaces/IContentRepository.cs | 9 ++ .../Interfaces/IMediaRepository.cs | 1 + .../Interfaces/IRepositoryVersionable.cs | 10 ++ .../Repositories/MediaRepository.cs | 85 +++++++++++++ .../Repositories/MemberRepository.cs | 84 +++++++++++++ src/Umbraco.Core/Services/ContentService.cs | 110 +++++------------ .../Services/EntityXmlSerializer.cs | 2 +- src/Umbraco.Core/Services/IContentService.cs | 10 ++ src/Umbraco.Core/Services/IMediaService.cs | 9 ++ src/Umbraco.Core/Services/IMemberService.cs | 9 ++ src/Umbraco.Core/Services/MediaService.cs | 92 ++------------ src/Umbraco.Core/Services/MemberService.cs | 104 ++++------------ .../Repositories/ContentRepositoryTest.cs | 113 ++++++++++++++++++ .../Repositories/MediaRepositoryTest.cs | 70 +++++++++++ .../Repositories/MemberRepositoryTest.cs | 75 +++++++++++- 16 files changed, 625 insertions(+), 247 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 2abb9fe9de..49e4673e5c 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -168,7 +168,86 @@ namespace Umbraco.Core.Persistence.Repositories #endregion #region Overrides of VersionableRepositoryBase - + + public void RebuildXmlStructures(Func serializer, int groupSize = 5000, IEnumerable contentTypeIds = null) + { + + //Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too. + using (var tr = Database.GetTransaction()) + { + //Remove all the data first, if anything fails after this it's no problem the transaction will be reverted + if (contentTypeIds == null) + { + var subQuery = new Sql() + .Select("DISTINCT cmsContentXml.nodeId") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId); + + var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); + Database.Execute(deleteSql); + } + else + { + foreach (var id in contentTypeIds) + { + var id1 = id; + var subQuery = new Sql() + .Select("cmsDocument.nodeId") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .Where(dto => dto.Published) + .Where(dto => dto.ContentTypeId == id1); + + var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); + Database.Execute(deleteSql); + } + } + + //now insert the data, again if something fails here, the whole transaction is reversed + if (contentTypeIds == null) + { + var query = Query.Builder.Where(x => x.Published == true); + RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize); + } + else + { + foreach (var contentTypeId in contentTypeIds) + { + //copy local + var id = contentTypeId; + var query = Query.Builder.Where(x => x.Published == true && x.ContentTypeId == id && x.Trashed == false); + RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize); + } + } + + tr.Complete(); + } + } + + private void RebuildXmlStructuresProcessQuery(Func serializer, IQuery query, Transaction tr, int pageSize) + { + var pageIndex = 0; + var total = int.MinValue; + var processed = 0; + do + { + var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending); + + var xmlItems = (from descendant in descendants + let xml = serializer(descendant) + select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToString(SaveOptions.None) }).ToArray(); + + //bulk insert it into the database + Database.BulkInsertRecords(xmlItems, tr); + + processed += xmlItems.Length; + + pageIndex++; + } while (processed < total); + } + public override IContent GetByVersion(Guid versionId) { var sql = GetBaseQuery(false); @@ -538,6 +617,14 @@ namespace Umbraco.Core.Persistence.Repositories } } + public int CountPublished() + { + var sql = GetBaseQuery(true).Where(x => x.Trashed == false) + .Where(x => x.Published == true) + .Where(x => x.Newest == true); + return Database.ExecuteScalar(sql); + } + public void ReplaceContentPermissions(EntityPermissionSet permissionSet) { var repo = new PermissionRepository(UnitOfWork, _cacheHelper); diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs index e52789c3ac..ac0fcebbe7 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IContentRepository.cs @@ -11,6 +11,15 @@ namespace Umbraco.Core.Persistence.Repositories { public interface IContentRepository : IRepositoryVersionable, IRecycleBinRepository { + /// + /// Get the count of published items + /// + /// + /// + /// We require this on the repo because the IQuery{IContent} cannot supply the 'newest' parameter + /// + int CountPublished(); + /// /// Used to bulk update the permissions set for a content item. This will replace all permissions /// assigned to an entity with a list of user id & permission pairs. diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs index e571269b98..f84844c177 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IMediaRepository.cs @@ -9,6 +9,7 @@ namespace Umbraco.Core.Persistence.Repositories { public interface IMediaRepository : IRepositoryVersionable, IRecycleBinRepository { + /// /// Used to add/update published xml for the media item /// diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs index 81783ccfbd..229a6fc0ef 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Xml.Linq; using Umbraco.Core.Models.EntityBase; namespace Umbraco.Core.Persistence.Repositories @@ -12,6 +13,15 @@ namespace Umbraco.Core.Persistence.Repositories public interface IRepositoryVersionable : IRepositoryQueryable where TEntity : IAggregateRoot { + /// + /// Rebuilds the xml structures for all TEntity if no content type ids are specified, otherwise rebuilds the xml structures + /// for only the content types specified + /// + /// The serializer to convert TEntity to Xml + /// Structures will be rebuilt in chunks of this size + /// + void RebuildXmlStructures(Func serializer, int groupSize = 5000, IEnumerable contentTypeIds = null); + /// /// Get the total count of entities /// diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index ab01af1aff..7a487acaf4 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -16,6 +16,7 @@ using Umbraco.Core.Persistence.Factories; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.SqlSyntax; using Umbraco.Core.Persistence.UnitOfWork; +using Umbraco.Core.Services; namespace Umbraco.Core.Persistence.Repositories { @@ -172,6 +173,90 @@ namespace Umbraco.Core.Persistence.Repositories return media; } + public void RebuildXmlStructures(Func serializer, int groupSize = 5000, IEnumerable contentTypeIds = null) + { + + //Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too. + using (var tr = Database.GetTransaction()) + { + //Remove all the data first, if anything fails after this it's no problem the transaction will be reverted + if (contentTypeIds == null) + { + var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media); + var subQuery = new Sql() + .Select("DISTINCT cmsContentXml.nodeId") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .Where(dto => dto.NodeObjectType == mediaObjectType); + + var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); + Database.Execute(deleteSql); + } + else + { + foreach (var id in contentTypeIds) + { + var id1 = id; + var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media); + var subQuery = new Sql() + .Select("DISTINCT cmsContentXml.nodeId") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .Where(dto => dto.NodeObjectType == mediaObjectType) + .Where(dto => dto.ContentTypeId == id1); + + var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); + Database.Execute(deleteSql); + } + } + + //now insert the data, again if something fails here, the whole transaction is reversed + if (contentTypeIds == null) + { + var query = Query.Builder; + RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize); + } + else + { + foreach (var contentTypeId in contentTypeIds) + { + //copy local + var id = contentTypeId; + var query = Query.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false); + RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize); + } + } + + tr.Complete(); + } + } + + private void RebuildXmlStructuresProcessQuery(Func serializer, IQuery query, Transaction tr, int pageSize) + { + var pageIndex = 0; + var total = int.MinValue; + var processed = 0; + do + { + var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending); + + var xmlItems = (from descendant in descendants + let xml = serializer(descendant) + select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToString(SaveOptions.None) }).ToArray(); + + //bulk insert it into the database + Database.BulkInsertRecords(xmlItems, tr); + + processed += xmlItems.Length; + + pageIndex++; + } while (processed < total); + } + public void AddOrUpdateContentXml(IMedia content, Func xml) { var contentExists = Database.ExecuteScalar("SELECT COUNT(nodeId) FROM cmsContentXml WHERE nodeId = @Id", new { Id = content.Id }) != 0; diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index 4158fddfa4..d5c65bd85b 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -418,6 +418,90 @@ namespace Umbraco.Core.Persistence.Repositories #region Overrides of VersionableRepositoryBase + public void RebuildXmlStructures(Func serializer, int groupSize = 5000, IEnumerable contentTypeIds = null) + { + + //Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too. + using (var tr = Database.GetTransaction()) + { + //Remove all the data first, if anything fails after this it's no problem the transaction will be reverted + if (contentTypeIds == null) + { + var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member); + var subQuery = new Sql() + .Select("DISTINCT cmsContentXml.nodeId") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .Where(dto => dto.NodeObjectType == memberObjectType); + + var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); + Database.Execute(deleteSql); + } + else + { + foreach (var id in contentTypeIds) + { + var id1 = id; + var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member); + var subQuery = new Sql() + .Select("DISTINCT cmsContentXml.nodeId") + .From() + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .InnerJoin() + .On(left => left.NodeId, right => right.NodeId) + .Where(dto => dto.NodeObjectType == memberObjectType) + .Where(dto => dto.ContentTypeId == id1); + + var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); + Database.Execute(deleteSql); + } + } + + //now insert the data, again if something fails here, the whole transaction is reversed + if (contentTypeIds == null) + { + var query = Query.Builder; + RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize); + } + else + { + foreach (var contentTypeId in contentTypeIds) + { + //copy local + var id = contentTypeId; + var query = Query.Builder.Where(x => x.ContentTypeId == id && x.Trashed == false); + RebuildXmlStructuresProcessQuery(serializer, query, tr, groupSize); + } + } + + tr.Complete(); + } + } + + private void RebuildXmlStructuresProcessQuery(Func serializer, IQuery query, Transaction tr, int pageSize) + { + var pageIndex = 0; + var total = int.MinValue; + var processed = 0; + do + { + var descendants = GetPagedResultsByQuery(query, pageIndex, pageSize, out total, "Path", Direction.Ascending); + + var xmlItems = (from descendant in descendants + let xml = serializer(descendant) + select new ContentXmlDto { NodeId = descendant.Id, Xml = xml.ToString(SaveOptions.None) }).ToArray(); + + //bulk insert it into the database + Database.BulkInsertRecords(xmlItems, tr); + + processed += xmlItems.Length; + + pageIndex++; + } while (processed < total); + } + public override IMember GetByVersion(Guid versionId) { var sql = GetBaseQuery(false); diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index f70f1295ee..00dd4ddb0e 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -77,6 +77,15 @@ namespace Umbraco.Core.Services _userService = userService; } + public int CountPublished(string contentTypeAlias = null) + { + var uow = _uowProvider.GetUnitOfWork(); + using (var repository = _repositoryFactory.CreateContentRepository(uow)) + { + return repository.CountPublished(); + } + } + public int Count(string contentTypeAlias = null) { var uow = _uowProvider.GetUnitOfWork(); @@ -1460,6 +1469,27 @@ namespace Umbraco.Core.Services return true; } + /// + /// Rebuilds all xml content in the cmsContentXml table for all documents + /// + /// + /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures + /// for all content + /// + public void RebuildXmlStructures(params int[] contentTypeIds) + { + var uow = _uowProvider.GetUnitOfWork(); + using (var repository = _repositoryFactory.CreateContentRepository(uow)) + { + repository.RebuildXmlStructures( + content => _entitySerializer.Serialize(this, _dataTypeService, _userService, content), + contentTypeIds: contentTypeIds.Length == 0 ? null : contentTypeIds); + } + + Audit.Add(AuditTypes.Publish, "ContentService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); + + } + #region Internal Methods /// @@ -1558,86 +1588,6 @@ namespace Umbraco.Core.Services } } - //TODO: WE should make a base class for ContentService and MediaService to share! - // currently we have this logic duplicated (nearly the same) for media types and soon to be member types - - //TODO: This needs to be put into the ContentRepository, all CUD logic! - - /// - /// Rebuilds all xml content in the cmsContentXml table for all documents - /// - /// - /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures - /// for all content - /// - /// True if publishing succeeded, otherwise False - private void RebuildXmlStructures(params int[] contentTypeIds) - { - using (new WriteLock(Locker)) - { - var list = new List(); - - var uow = _uowProvider.GetUnitOfWork(); - - //First we're going to get the data that needs to be inserted before clearing anything, this - //ensures that we don't accidentally leave the content xml table empty if something happens - //during the lookup process. - - list.AddRange(contentTypeIds.Any() == false - ? GetAllPublished() - : contentTypeIds.SelectMany(GetPublishedContentOfContentType)); - - var xmlItems = new List(); - foreach (var c in list) - { - var xml = _entitySerializer.Serialize(this, _dataTypeService, _userService, c); - xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }); - } - - //Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too. - using (var tr = uow.Database.GetTransaction()) - { - if (contentTypeIds.Any() == false) - { - //Remove all Document records from the cmsContentXml table (DO NOT REMOVE Media/Members!) (based on inner join of cmsDocument) - var subQuery = new Sql() - .Select("DISTINCT cmsContentXml.nodeId") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId); - - var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); - uow.Database.Execute(deleteSql); - } - else - { - foreach (var id in contentTypeIds) - { - //first we'll clear out the data from the cmsContentXml table for this type - var id1 = id; - var subQuery = new Sql() - .Select("cmsDocument.nodeId") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .Where(dto => dto.Published) - .Where(dto => dto.ContentTypeId == id1); - - var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); - uow.Database.Execute(deleteSql); - } - } - - //bulk insert it into the database - uow.Database.BulkInsertRecords(xmlItems, tr); - - tr.Complete(); - } - - Audit.Add(AuditTypes.Publish, "RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); - } - } - /// /// Publishes a object and all its children /// diff --git a/src/Umbraco.Core/Services/EntityXmlSerializer.cs b/src/Umbraco.Core/Services/EntityXmlSerializer.cs index a3adb82964..df29d02ca4 100644 --- a/src/Umbraco.Core/Services/EntityXmlSerializer.cs +++ b/src/Umbraco.Core/Services/EntityXmlSerializer.cs @@ -83,7 +83,7 @@ namespace Umbraco.Core.Services } /// - /// Exports an item to xml as an + /// Exports an item to xml as an /// /// /// Member to export diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index c85647ccf9..f5579896f4 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -12,6 +12,16 @@ namespace Umbraco.Core.Services /// public interface IContentService : IService { + /// + /// Rebuilds all xml content in the cmsContentXml table for all documents + /// + /// + /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures + /// for all content + /// + void RebuildXmlStructures(params int[] contentTypeIds); + + int CountPublished(string contentTypeAlias = null); int Count(string contentTypeAlias = null); int CountChildren(int parentId, string contentTypeAlias = null); int CountDescendants(int parentId, string contentTypeAlias = null); diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs index 62b94cc34e..b77046ddc1 100644 --- a/src/Umbraco.Core/Services/IMediaService.cs +++ b/src/Umbraco.Core/Services/IMediaService.cs @@ -10,6 +10,15 @@ namespace Umbraco.Core.Services /// public interface IMediaService : IService { + /// + /// Rebuilds all xml content in the cmsContentXml table for all media + /// + /// + /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures + /// for all media + /// + void RebuildXmlStructures(params int[] contentTypeIds); + int Count(string contentTypeAlias = null); int CountChildren(int parentId, string contentTypeAlias = null); int CountDescendants(int parentId, string contentTypeAlias = null); diff --git a/src/Umbraco.Core/Services/IMemberService.cs b/src/Umbraco.Core/Services/IMemberService.cs index 49a3612f30..249a744081 100644 --- a/src/Umbraco.Core/Services/IMemberService.cs +++ b/src/Umbraco.Core/Services/IMemberService.cs @@ -12,6 +12,15 @@ namespace Umbraco.Core.Services /// public interface IMemberService : IMembershipMemberService { + /// + /// Rebuilds all xml content in the cmsContentXml table for all documents + /// + /// + /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures + /// for all content + /// + void RebuildXmlStructures(params int[] contentTypeIds); + /// /// Gets a list of paged objects /// diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index 4266817435..f2c5526e8d 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -637,7 +637,7 @@ namespace Umbraco.Core.Services { return; } - + media.ParentId = parentId; if (media.Trashed) { @@ -661,7 +661,7 @@ namespace Umbraco.Core.Services var parentLevel = media.Level; var parentTrashed = media.Trashed; var updatedDescendants = UpdatePropertiesOnChildren(children, parentPath, parentLevel, parentTrashed, moveInfo); - Save(updatedDescendants, userId, + Save(updatedDescendants, userId, //no events! false); } @@ -708,7 +708,7 @@ namespace Umbraco.Core.Services media.ChangeTrashedState(true, Constants.System.RecycleBinMedia); repository.AddOrUpdate(media); - + //Loop through descendants to update their trash state, but ensuring structure by keeping the ParentId foreach (var descendant in descendants) { @@ -1037,8 +1037,6 @@ namespace Umbraco.Core.Services return true; } - - //TODO: This needs to be put into the MediaRepository, all CUD logic! /// /// Rebuilds all xml content in the cmsContentXml table for all media @@ -1047,85 +1045,17 @@ namespace Umbraco.Core.Services /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures /// for all media /// - /// True if publishing succeeded, otherwise False - internal void RebuildXmlStructures(params int[] contentTypeIds) + public void RebuildXmlStructures(params int[] contentTypeIds) { - using (new WriteLock(Locker)) + var uow = _uowProvider.GetUnitOfWork(); + using (var repository = _repositoryFactory.CreateMediaRepository(uow)) { - var list = new List(); - - var uow = _uowProvider.GetUnitOfWork(); - - //First we're going to get the data that needs to be inserted before clearing anything, this - //ensures that we don't accidentally leave the content xml table empty if something happens - //during the lookup process. - - if (contentTypeIds.Any() == false) - { - var rootMedia = GetRootMedia(); - foreach (var media in rootMedia) - { - list.Add(media); - list.AddRange(GetDescendants(media)); - } - } - else - { - list.AddRange(contentTypeIds.SelectMany(i => GetMediaOfMediaType(i).Where(media => media.Trashed == false))); - } - - var xmlItems = new List(); - foreach (var c in list) - { - var xml = _entitySerializer.Serialize(this, _dataTypeService, _userService, c); - xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }); - } - - //Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too. - using (var tr = uow.Database.GetTransaction()) - { - if (contentTypeIds.Any() == false) - { - var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media); - var subQuery = new Sql() - .Select("DISTINCT cmsContentXml.nodeId") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .Where(dto => dto.NodeObjectType == mediaObjectType); - - var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); - uow.Database.Execute(deleteSql); - } - else - { - foreach (var id in contentTypeIds) - { - var id1 = id; - var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media); - var subQuery = new Sql() - .Select("DISTINCT cmsContentXml.nodeId") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .Where(dto => dto.NodeObjectType == mediaObjectType) - .Where(dto => dto.ContentTypeId == id1); - - var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); - uow.Database.Execute(deleteSql); - } - } - - //bulk insert it into the database - uow.Database.BulkInsertRecords(xmlItems, tr); - - tr.Complete(); - } - - Audit.Add(AuditTypes.Publish, "RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); + repository.RebuildXmlStructures( + media => _entitySerializer.Serialize(this, _dataTypeService, _userService, media), + contentTypeIds: contentTypeIds.Length == 0 ? null : contentTypeIds); } + + Audit.Add(AuditTypes.Publish, "MediaService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); } /// diff --git a/src/Umbraco.Core/Services/MemberService.cs b/src/Umbraco.Core/Services/MemberService.cs index 0dece12390..5aa94fb409 100644 --- a/src/Umbraco.Core/Services/MemberService.cs +++ b/src/Umbraco.Core/Services/MemberService.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading; using System.Web.Security; using System.Xml.Linq; +using Umbraco.Core.Auditing; using Umbraco.Core.Configuration; using Umbraco.Core.Events; using Umbraco.Core.Models; @@ -589,6 +590,27 @@ namespace Umbraco.Core.Services } } + /// + /// Rebuilds all xml content in the cmsContentXml table for all members + /// + /// + /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures + /// for all members = USE WITH CARE! + /// + /// True if publishing succeeded, otherwise False + public void RebuildXmlStructures(params int[] memberTypeIds) + { + var uow = _uowProvider.GetUnitOfWork(); + using (var repository = _repositoryFactory.CreateMemberRepository(uow)) + { + repository.RebuildXmlStructures( + member => _entitySerializer.Serialize(_dataTypeService, member), + contentTypeIds: memberTypeIds.Length == 0 ? null : memberTypeIds); + } + + Audit.Add(AuditTypes.Publish, "MemberService.RebuildXmlStructures completed, the xml has been regenerated in the database", 0, -1); + } + #endregion #region IMembershipMemberService Implementation @@ -1151,88 +1173,6 @@ namespace Umbraco.Core.Services return contentType; } } - - /// - /// Rebuilds all xml content in the cmsContentXml table for all members - /// - /// - /// Only rebuild the xml structures for the content type ids passed in, if none then rebuilds the structures - /// for all members = USE WITH CARE! - /// - /// True if publishing succeeded, otherwise False - internal void RebuildXmlStructures(params int[] memberTypeIds) - { - using (new WriteLock(Locker)) - { - var list = new List(); - - var uow = _uowProvider.GetUnitOfWork(); - - //First we're going to get the data that needs to be inserted before clearing anything, this - //ensures that we don't accidentally leave the content xml table empty if something happens - //during the lookup process. - - if (memberTypeIds.Any() == false) - { - list.AddRange(GetAllMembers()); - } - else - { - list.AddRange(memberTypeIds.SelectMany(GetMembersByMemberType)); - } - - var xmlItems = new List(); - foreach (var c in list) - { - var xml = _entitySerializer.Serialize(_dataTypeService, c); - xmlItems.Add(new ContentXmlDto { NodeId = c.Id, Xml = xml.ToString(SaveOptions.None) }); - } - - //Ok, now we need to remove the data and re-insert it, we'll do this all in one transaction too. - using (var tr = uow.Database.GetTransaction()) - { - if (memberTypeIds.Any() == false) - { - //Remove all member records from the cmsContentXml table (DO NOT REMOVE Content/Media!) - var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member); - var subQuery = new Sql() - .Select("DISTINCT cmsContentXml.nodeId") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .Where(dto => dto.NodeObjectType == memberObjectType); - - var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); - uow.Database.Execute(deleteSql); - } - else - { - foreach (var id in memberTypeIds) - { - var id1 = id; - var memberObjectType = Guid.Parse(Constants.ObjectTypes.Member); - var subQuery = new Sql() - .Select("DISTINCT cmsContentXml.nodeId") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .Where(dto => dto.NodeObjectType == memberObjectType) - .Where(dto => dto.ContentTypeId == id1); - - var deleteSql = SqlSyntaxContext.SqlSyntaxProvider.GetDeleteSubquery("cmsContentXml", "nodeId", subQuery); - uow.Database.Execute(deleteSql); - } - } - - //bulk insert it into the database - uow.Database.BulkInsertRecords(xmlItems, tr); - - tr.Complete(); - } - } - } #region Event Handlers diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs index 6cdc192c63..86c26fd0e5 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Xml.Linq; using NUnit.Framework; using Umbraco.Core; using Umbraco.Core.Models; @@ -46,6 +47,118 @@ namespace Umbraco.Tests.Persistence.Repositories return repository; } + [Test] + public void Rebuild_All_Xml_Structures() + { + var provider = new PetaPocoUnitOfWorkProvider(); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + + var contentType1 = MockedContentTypes.CreateSimpleContentType("Textpage1", "Textpage1"); + contentTypeRepository.AddOrUpdate(contentType1); + var allCreated = new List(); + + for (var i = 0; i < 100; i++) + { + //These will be non-published so shouldn't show up + var c1 = MockedContent.CreateSimpleContent(contentType1); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + for (var i = 0; i < 100; i++) + { + var c1 = MockedContent.CreateSimpleContent(contentType1); + c1.ChangePublishedState(PublishedState.Published); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + unitOfWork.Commit(); + + //now create some versions of this content - this shouldn't affect the xml structures saved + for (int i = 0; i < allCreated.Count; i++) + { + allCreated[i].Name = "blah" + i; + repository.AddOrUpdate(allCreated[i]); + } + unitOfWork.Commit(); + + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + + repository.RebuildXmlStructures(media => new XElement("test"), 10); + + Assert.AreEqual(100, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + + [Test] + public void Rebuild_All_Xml_Structures_For_Content_Type() + { + var provider = new PetaPocoUnitOfWorkProvider(); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + var contentType1 = MockedContentTypes.CreateSimpleContentType("Textpage1", "Textpage1"); + var contentType2 = MockedContentTypes.CreateSimpleContentType("Textpage2", "Textpage2"); + var contentType3 = MockedContentTypes.CreateSimpleContentType("Textpage3", "Textpage3"); + contentTypeRepository.AddOrUpdate(contentType1); + contentTypeRepository.AddOrUpdate(contentType2); + contentTypeRepository.AddOrUpdate(contentType3); + + var allCreated = new List(); + + for (var i = 0; i < 30; i++) + { + //These will be non-published so shouldn't show up + var c1 = MockedContent.CreateSimpleContent(contentType1); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + for (var i = 0; i < 30; i++) + { + var c1 = MockedContent.CreateSimpleContent(contentType1); + c1.ChangePublishedState(PublishedState.Published); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + for (var i = 0; i < 30; i++) + { + var c1 = MockedContent.CreateSimpleContent(contentType2); + c1.ChangePublishedState(PublishedState.Published); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + for (var i = 0; i < 30; i++) + { + var c1 = MockedContent.CreateSimpleContent(contentType3); + c1.ChangePublishedState(PublishedState.Published); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + unitOfWork.Commit(); + + //now create some versions of this content - this shouldn't affect the xml structures saved + for (int i = 0; i < allCreated.Count; i++) + { + allCreated[i].Name = "blah" + i; + repository.AddOrUpdate(allCreated[i]); + } + unitOfWork.Commit(); + + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + + repository.RebuildXmlStructures(media => new XElement("test"), 10, contentTypeIds: new[] { contentType1.Id, contentType2.Id }); + + Assert.AreEqual(60, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + [Test] public void Ensures_Permissions_Are_Set_If_Parent_Entity_Permissions_Exist() { diff --git a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs index 26cc919c5c..b76bdd37bc 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Xml.Linq; using NUnit.Framework; using Umbraco.Core; using Umbraco.Core.Models; @@ -9,6 +10,7 @@ using Umbraco.Core.Persistence; using Umbraco.Core.Persistence.Caching; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.Repositories; +using Umbraco.Core.Persistence.SqlSyntax; using Umbraco.Core.Persistence.UnitOfWork; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.TestHelpers.Entities; @@ -38,6 +40,74 @@ namespace Umbraco.Tests.Persistence.Repositories return repository; } + [Test] + public void Rebuild_All_Xml_Structures() + { + var provider = new PetaPocoUnitOfWorkProvider(); + var unitOfWork = provider.GetUnitOfWork(); + MediaTypeRepository mediaTypeRepository; + using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository)) + { + + var mediaType = mediaTypeRepository.Get(1032); + + for (var i = 0; i < 100; i++) + { + var image = MockedMedia.CreateMediaImage(mediaType, -1); + repository.AddOrUpdate(image); + } + unitOfWork.Commit(); + + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + + repository.RebuildXmlStructures(media => new XElement("test"), 10); + + Assert.AreEqual(103, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + + [Test] + public void Rebuild_All_Xml_Structures_For_Content_Type() + { + var provider = new PetaPocoUnitOfWorkProvider(); + var unitOfWork = provider.GetUnitOfWork(); + MediaTypeRepository mediaTypeRepository; + using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository)) + { + + var imageMediaType = mediaTypeRepository.Get(1032); + var fileMediaType = mediaTypeRepository.Get(1033); + var folderMediaType = mediaTypeRepository.Get(1031); + + for (var i = 0; i < 30; i++) + { + var image = MockedMedia.CreateMediaImage(imageMediaType, -1); + repository.AddOrUpdate(image); + } + for (var i = 0; i < 30; i++) + { + var file = MockedMedia.CreateMediaFile(fileMediaType, -1); + repository.AddOrUpdate(file); + } + for (var i = 0; i < 30; i++) + { + var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1); + repository.AddOrUpdate(folder); + } + unitOfWork.Commit(); + + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + + repository.RebuildXmlStructures(media => new XElement("test"), 10, contentTypeIds: new[] {1032, 1033}); + + Assert.AreEqual(62, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + [Test] public void Can_Instantiate_Repository_From_Resolver() { diff --git a/src/Umbraco.Tests/Persistence/Repositories/MemberRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/MemberRepositoryTest.cs index 08b89d245b..dec18ef4e5 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/MemberRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/MemberRepositoryTest.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Xml.Linq; using NUnit.Framework; using Umbraco.Core; using Umbraco.Core.Models; @@ -39,6 +40,76 @@ namespace Umbraco.Tests.Persistence.Repositories return repository; } + [Test] + public void Rebuild_All_Xml_Structures() + { + var provider = new PetaPocoUnitOfWorkProvider(); + var unitOfWork = provider.GetUnitOfWork(); + MemberTypeRepository memberTypeRepository; + MemberGroupRepository memberGroupRepository; + using (var repository = CreateRepository(unitOfWork, out memberTypeRepository, out memberGroupRepository)) + { + var memberType1 = CreateTestMemberType(); + + for (var i = 0; i < 100; i++) + { + var member = MockedMember.CreateSimpleMember(memberType1, "blah" + i, "blah" + i + "@example.com", "blah", "blah" + i); + repository.AddOrUpdate(member); + } + unitOfWork.Commit(); + + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + + repository.RebuildXmlStructures(media => new XElement("test"), 10); + + Assert.AreEqual(100, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + + [Test] + public void Rebuild_All_Xml_Structures_For_Content_Type() + { + var provider = new PetaPocoUnitOfWorkProvider(); + var unitOfWork = provider.GetUnitOfWork(); + MemberTypeRepository memberTypeRepository; + MemberGroupRepository memberGroupRepository; + using (var repository = CreateRepository(unitOfWork, out memberTypeRepository, out memberGroupRepository)) + { + + var memberType1 = CreateTestMemberType("mt1"); + var memberType2 = CreateTestMemberType("mt2"); + var memberType3 = CreateTestMemberType("mt3"); + + for (var i = 0; i < 30; i++) + { + var member = MockedMember.CreateSimpleMember(memberType1, "b1lah" + i, "b1lah" + i + "@example.com", "b1lah", "b1lah" + i); + repository.AddOrUpdate(member); + } + for (var i = 0; i < 30; i++) + { + var member = MockedMember.CreateSimpleMember(memberType2, "b2lah" + i, "b2lah" + i + "@example.com", "b2lah", "b2lah" + i); + repository.AddOrUpdate(member); + } + for (var i = 0; i < 30; i++) + { + var member = MockedMember.CreateSimpleMember(memberType3, "b3lah" + i, "b3lah" + i + "@example.com", "b3lah", "b3lah" + i); + repository.AddOrUpdate(member); + } + unitOfWork.Commit(); + + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + Assert.AreEqual(0, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + + repository.RebuildXmlStructures(media => new XElement("test"), 10, contentTypeIds: new[] { memberType1.Id, memberType2.Id }); + + Assert.AreEqual(60, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + + [Test] public void Can_Instantiate_Repository_From_Resolver() { @@ -297,7 +368,7 @@ namespace Umbraco.Tests.Persistence.Repositories } } - private IMemberType CreateTestMemberType() + private IMemberType CreateTestMemberType(string alias = null) { var provider = new PetaPocoUnitOfWorkProvider(); var unitOfWork = provider.GetUnitOfWork(); @@ -305,7 +376,7 @@ namespace Umbraco.Tests.Persistence.Repositories MemberGroupRepository memberGroupRepository; using (var repository = CreateRepository(unitOfWork, out memberTypeRepository, out memberGroupRepository)) { - var memberType = MockedContentTypes.CreateSimpleMemberType(); + var memberType = MockedContentTypes.CreateSimpleMemberType(alias); memberTypeRepository.AddOrUpdate(memberType); unitOfWork.Commit(); return memberType;