From 5bc66c5b7fe2bdf311370ece1b998dfff530c5ff Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Wed, 7 Dec 2016 14:18:27 +0100 Subject: [PATCH 1/7] Bumps the version --- build/UmbracoVersion.txt | 2 +- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/UmbracoVersion.txt b/build/UmbracoVersion.txt index df03fe8381..345fce6961 100644 --- a/build/UmbracoVersion.txt +++ b/build/UmbracoVersion.txt @@ -1,2 +1,2 @@ # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta) -7.5.5 \ No newline at end of file +7.5.6 \ No newline at end of file diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index b3b5633864..6821a737af 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.5.5")] -[assembly: AssemblyInformationalVersion("7.5.5")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.5.6")] +[assembly: AssemblyInformationalVersion("7.5.6")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index ccf4aecf13..0b3a17c8d6 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.5.5"); + private static readonly Version Version = new Version("7.5.6"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index c8c90c2ed5..8424c750c7 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -2415,9 +2415,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7550 + 7560 / - http://localhost:7550 + http://localhost:7560 False False From 85f054e3c1a70bc76279d8cbec9d6bc1dfc91ef0 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 20 Jan 2017 09:54:59 +0100 Subject: [PATCH 2/7] No need to warn if attribute is not present Adds some logging about indexing being done.. the number of committed items seems to differ a little from the number of items in the index, not sure wh --- .../WebServices/ExamineManagementApiController.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs index 9c50468465..79e7fe8f53 100644 --- a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs +++ b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs @@ -166,6 +166,8 @@ namespace Umbraco.Web.WebServices var msg = ValidateLuceneIndexer(indexerName, out indexer); if (msg.IsSuccessStatusCode) { + LogHelper.Info(string.Format("Rebuilding index '{0}'", indexerName)); + //remove it in case there's a handler there alraedy indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; //now add a single handler @@ -201,6 +203,8 @@ namespace Umbraco.Web.WebServices //ensure it's not listening anymore indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; + LogHelper.Info(string.Format("Rebuilding index '{0}' done, {1} items committed (can differ from the number of items in the index)", indexer.Name, indexer.CommitCount)); + var cacheKey = "temp_indexing_op_" + indexer.Name; ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(cacheKey); } @@ -266,7 +270,10 @@ namespace Umbraco.Web.WebServices var val = p.GetValue(indexer, null); if (val == null) { - LogHelper.Warn("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name); + // Do not warn for new new attribute that is optional + if(string.Equals(p.Name, "DirectoryFactory", StringComparison.InvariantCultureIgnoreCase) == false) + LogHelper.Warn("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name); + val = string.Empty; } indexerModel.ProviderProperties.Add(p.Name, val.ToString()); From 12922f841361c6884d266401d0cb548a89d84504 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Fri, 20 Jan 2017 09:54:59 +0100 Subject: [PATCH 3/7] No need to warn if attribute is not present Adds some logging about indexing being done.. the number of committed items seems to differ a little from the number of items in the index, not sure wh --- .../WebServices/ExamineManagementApiController.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs index 9c50468465..79e7fe8f53 100644 --- a/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs +++ b/src/Umbraco.Web/WebServices/ExamineManagementApiController.cs @@ -166,6 +166,8 @@ namespace Umbraco.Web.WebServices var msg = ValidateLuceneIndexer(indexerName, out indexer); if (msg.IsSuccessStatusCode) { + LogHelper.Info(string.Format("Rebuilding index '{0}'", indexerName)); + //remove it in case there's a handler there alraedy indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; //now add a single handler @@ -201,6 +203,8 @@ namespace Umbraco.Web.WebServices //ensure it's not listening anymore indexer.IndexOperationComplete -= Indexer_IndexOperationComplete; + LogHelper.Info(string.Format("Rebuilding index '{0}' done, {1} items committed (can differ from the number of items in the index)", indexer.Name, indexer.CommitCount)); + var cacheKey = "temp_indexing_op_" + indexer.Name; ApplicationContext.Current.ApplicationCache.RuntimeCache.ClearCacheItem(cacheKey); } @@ -266,7 +270,10 @@ namespace Umbraco.Web.WebServices var val = p.GetValue(indexer, null); if (val == null) { - LogHelper.Warn("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name); + // Do not warn for new new attribute that is optional + if(string.Equals(p.Name, "DirectoryFactory", StringComparison.InvariantCultureIgnoreCase) == false) + LogHelper.Warn("Property value was null when setting up property on indexer: " + indexer.Name + " property: " + p.Name); + val = string.Empty; } indexerModel.ProviderProperties.Add(p.Name, val.ToString()); From 2a4e73c65021cf9c22e0b68001030d559ac159cd Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 23 Jan 2017 00:40:24 +1100 Subject: [PATCH 4/7] Reduced allocations for the Media/Content/Property Factories since we don't need to create new objects every time we want to map values. Changes default sort order for paging from Path to umbracoNode.id since we have no index on Path and it doesn't make a lot of sense. Fixes obsolete warnings for various Sql usages. Reduces the amount of DeepClone calls required when looking up a content type for a content item, even though these are cached they are still deep cloned out of the cache. Fixes the main issue of having nearly 100,000 rows of unsorted property data and then having to query those rows for every document being built, the re-iteration of these rows causes a lot of overhead and is unecessary, instead we ensure the property data set and the document data set is sorted by node id, then use a stored index to continue looking up the property data for the next content item found. --- .../Persistence/Factories/ContentFactory.cs | 17 +- .../Persistence/Factories/MediaFactory.cs | 13 +- .../Persistence/Factories/PropertyFactory.cs | 16 +- .../Repositories/ContentRepository.cs | 52 +++--- .../Repositories/MediaRepository.cs | 60 +++---- .../Repositories/VersionableRepositoryBase.cs | 150 +++++++++++------- src/Umbraco.Core/Services/ContentService.cs | 4 +- src/Umbraco.Core/Services/IContentService.cs | 4 +- src/Umbraco.Core/Services/IMediaService.cs | 4 +- src/Umbraco.Core/Services/MediaService.cs | 4 +- 10 files changed, 199 insertions(+), 125 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Factories/ContentFactory.cs b/src/Umbraco.Core/Persistence/Factories/ContentFactory.cs index 5dcec8fed0..88de5a1d6c 100644 --- a/src/Umbraco.Core/Persistence/Factories/ContentFactory.cs +++ b/src/Umbraco.Core/Persistence/Factories/ContentFactory.cs @@ -27,15 +27,15 @@ namespace Umbraco.Core.Persistence.Factories #region Implementation of IEntityFactory - public IContent BuildEntity(DocumentDto dto) + public static IContent BuildEntity(DocumentDto dto, IContentType contentType) { - var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, _contentType); + var content = new Content(dto.Text, dto.ContentVersionDto.ContentDto.NodeDto.ParentId, contentType); try { content.DisableChangeTracking(); - content.Id = _id; + content.Id = dto.NodeId; content.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId; content.Name = dto.Text; content.NodeName = dto.ContentVersionDto.ContentDto.NodeDto.Text; @@ -49,8 +49,8 @@ namespace Umbraco.Core.Persistence.Factories content.Published = dto.Published; content.CreateDate = dto.ContentVersionDto.ContentDto.NodeDto.CreateDate; content.UpdateDate = dto.ContentVersionDto.VersionDate; - content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?) null; - content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?) null; + content.ExpireDate = dto.ExpiresDate.HasValue ? dto.ExpiresDate.Value : (DateTime?)null; + content.ReleaseDate = dto.ReleaseDate.HasValue ? dto.ReleaseDate.Value : (DateTime?)null; content.Version = dto.ContentVersionDto.VersionId; content.PublishedState = dto.Published ? PublishedState.Published : PublishedState.Unpublished; content.PublishedVersionGuid = dto.DocumentPublishedReadOnlyDto == null ? default(Guid) : dto.DocumentPublishedReadOnlyDto.VersionId; @@ -64,6 +64,13 @@ namespace Umbraco.Core.Persistence.Factories { content.EnableChangeTracking(); } + + } + + [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")] + public IContent BuildEntity(DocumentDto dto) + { + return BuildEntity(dto, _contentType); } public DocumentDto BuildDto(IContent entity) diff --git a/src/Umbraco.Core/Persistence/Factories/MediaFactory.cs b/src/Umbraco.Core/Persistence/Factories/MediaFactory.cs index 0fcb654cb7..5729bb125e 100644 --- a/src/Umbraco.Core/Persistence/Factories/MediaFactory.cs +++ b/src/Umbraco.Core/Persistence/Factories/MediaFactory.cs @@ -27,15 +27,15 @@ namespace Umbraco.Core.Persistence.Factories #region Implementation of IEntityFactory - public IMedia BuildEntity(ContentVersionDto dto) + public static IMedia BuildEntity(ContentVersionDto dto, IMediaType contentType) { - var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, _contentType); + var media = new Models.Media(dto.ContentDto.NodeDto.Text, dto.ContentDto.NodeDto.ParentId, contentType); try { media.DisableChangeTracking(); - media.Id = _id; + media.Id = dto.NodeId; media.Key = dto.ContentDto.NodeDto.UniqueId; media.Path = dto.ContentDto.NodeDto.Path; media.CreatorId = dto.ContentDto.NodeDto.UserId.Value; @@ -55,6 +55,13 @@ namespace Umbraco.Core.Persistence.Factories { media.EnableChangeTracking(); } + + } + + [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")] + public IMedia BuildEntity(ContentVersionDto dto) + { + return BuildEntity(dto, _contentType); } public ContentVersionDto BuildDto(IMedia entity) diff --git a/src/Umbraco.Core/Persistence/Factories/PropertyFactory.cs b/src/Umbraco.Core/Persistence/Factories/PropertyFactory.cs index 446bd426ad..f202d8c321 100644 --- a/src/Umbraco.Core/Persistence/Factories/PropertyFactory.cs +++ b/src/Umbraco.Core/Persistence/Factories/PropertyFactory.cs @@ -30,11 +30,11 @@ namespace Umbraco.Core.Persistence.Factories _updateDate = updateDate; } - public IEnumerable BuildEntity(PropertyDataDto[] dtos) + public static IEnumerable BuildEntity(IReadOnlyCollection dtos, PropertyType[] compositionTypeProperties, DateTime createDate, DateTime updateDate) { var properties = new List(); - foreach (var propertyType in _compositionTypeProperties) + foreach (var propertyType in compositionTypeProperties) { var propertyDataDto = dtos.LastOrDefault(x => x.PropertyTypeId == propertyType.Id); var property = propertyDataDto == null @@ -47,8 +47,8 @@ namespace Umbraco.Core.Persistence.Factories //on initial construction we don't want to have dirty properties tracked property.DisableChangeTracking(); - property.CreateDate = _createDate; - property.UpdateDate = _updateDate; + property.CreateDate = createDate; + property.UpdateDate = updateDate; // http://issues.umbraco.org/issue/U4-1946 property.ResetDirtyProperties(false); properties.Add(property); @@ -57,12 +57,18 @@ namespace Umbraco.Core.Persistence.Factories { property.EnableChangeTracking(); } - + } return properties; } + [Obsolete("Use the static method instead, there's no reason to allocate one of these classes everytime we want to map values")] + public IEnumerable BuildEntity(PropertyDataDto[] dtos) + { + return BuildEntity(dtos, _compositionTypeProperties, _createDate, _updateDate); + } + public IEnumerable BuildDto(IEnumerable properties) { var propertyDataDtos = new List(); diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 294f869c3f..b25db8afb2 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories { var sql = GetBaseQuery(false) .Where(GetBaseWhereClause(), new { Id = id }) - .Where(x => x.Newest) + .Where(x => x.Newest, SqlSyntax) .OrderByDescending(x => x.VersionDate, SqlSyntax); var dto = Database.Fetch(SqlSyntax.SelectTop(sql, 1)).FirstOrDefault(); @@ -77,7 +77,7 @@ namespace Umbraco.Core.Persistence.Repositories } //we only want the newest ones with this method - sql.Where(x => x.Newest); + sql.Where(x => x.Newest, SqlSyntax); return ProcessQuery(sql); } @@ -87,9 +87,9 @@ namespace Umbraco.Core.Persistence.Repositories var sqlClause = GetBaseQuery(false); var translator = new SqlTranslator(sqlClause, query); var sql = translator.Translate() - .Where(x => x.Newest) - .OrderByDescending(x => x.VersionDate) - .OrderBy(x => x.SortOrder); + .Where(x => x.Newest, SqlSyntax) + .OrderByDescending(x => x.VersionDate, SqlSyntax) + .OrderBy(x => x.SortOrder, SqlSyntax); return ProcessQuery(sql); } @@ -183,8 +183,8 @@ namespace Umbraco.Core.Persistence.Repositories query = query .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); query = query - .Where(x => x.NodeId > baseId && x.Trashed == false) - .Where(x => x.Published) + .Where(x => x.NodeId > baseId && x.Trashed == false, SqlSyntax) + .Where(x => x.Published, SqlSyntax) .OrderBy(x => x.NodeId, SqlSyntax); var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize)) .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() }) @@ -222,7 +222,7 @@ namespace Umbraco.Core.Persistence.Repositories { var sql = GetBaseQuery(false); sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId }); - sql.OrderByDescending(x => x.VersionDate); + sql.OrderByDescending(x => x.VersionDate, SqlSyntax); var dto = Database.Fetch(sql).FirstOrDefault(); @@ -238,10 +238,10 @@ namespace Umbraco.Core.Persistence.Repositories { var sql = new Sql() .Select("*") - .From() - .InnerJoin().On(left => left.VersionId, right => right.VersionId) - .Where(x => x.VersionId == versionId) - .Where(x => x.Newest != true); + .From(SqlSyntax) + .InnerJoin(SqlSyntax).On(SqlSyntax, left => left.VersionId, right => right.VersionId) + .Where(x => x.VersionId == versionId, SqlSyntax) + .Where(x => x.Newest != true, SqlSyntax); var dto = Database.Fetch(sql).FirstOrDefault(); if (dto == null) return; @@ -848,7 +848,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", var content = new IContent[dtos.Count]; var defs = new List(); var templateIds = new List(); - + + //track the looked up content types, even though the content types are cached + // they still need to be deep cloned out of the cache and we don't want to add + // the overhead of deep cloning them on every item in this loop + var contentTypes = new Dictionary(); + for (var i = 0; i < dtos.Count; i++) { var dto = dtos[i]; @@ -867,9 +872,19 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", // else, need to fetch from the database // content type repository is full-cache so OK to get each one independently - var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); - var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId); - content[i] = factory.BuildEntity(dto); + + IContentType contentType; + if (contentTypes.ContainsKey(dto.ContentVersionDto.ContentDto.ContentTypeId)) + { + contentType = contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId]; + } + else + { + contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); + contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType; + } + + content[i] = ContentFactory.BuildEntity(dto, contentType); // need template if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0) @@ -910,7 +925,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 - ((Entity) cc).ResetDirtyProperties(false); + cc.ResetDirtyProperties(false); } return content; @@ -927,8 +942,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", { var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); - var factory = new ContentFactory(contentType, NodeObjectTypeId, dto.NodeId); - var content = factory.BuildEntity(dto); + var content = ContentFactory.BuildEntity(dto, contentType); //Check if template id is set on DocumentDto, and get ITemplate if it is. if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0) diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index 2687848fa5..bf034bd8ff 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -76,7 +76,7 @@ namespace Umbraco.Core.Persistence.Repositories var sqlClause = GetBaseQuery(false); var translator = new SqlTranslator(sqlClause, query); var sql = translator.Translate() - .OrderBy(x => x.SortOrder); + .OrderBy(x => x.SortOrder, SqlSyntax); return ProcessQuery(sql); } @@ -89,12 +89,12 @@ namespace Umbraco.Core.Persistence.Repositories { var sql = new Sql(); sql.Select(isCount ? "COUNT(*)" : "*") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .Where(x => x.NodeObjectType == NodeObjectTypeId); + .From(SqlSyntax) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax) + .Where(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax); return sql; } @@ -148,6 +148,11 @@ namespace Umbraco.Core.Persistence.Repositories var content = new IMedia[dtos.Count]; var defs = new List(); + //track the looked up content types, even though the content types are cached + // they still need to be deep cloned out of the cache and we don't want to add + // the overhead of deep cloning them on every item in this loop + var contentTypes = new Dictionary(); + for (var i = 0; i < dtos.Count; i++) { var dto = dtos[i]; @@ -165,9 +170,19 @@ namespace Umbraco.Core.Persistence.Repositories // else, need to fetch from the database // content type repository is full-cache so OK to get each one independently - var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); - var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId); - content[i] = factory.BuildEntity(dto); + + IMediaType contentType; + if (contentTypes.ContainsKey(dto.ContentDto.ContentTypeId)) + { + contentType = contentTypes[dto.ContentDto.ContentTypeId]; + } + else + { + contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); + contentTypes[dto.ContentDto.ContentTypeId] = contentType; + } + + content[i] = MediaFactory.BuildEntity(dto, contentType); // need properties defs.Add(new DocumentDefinition( @@ -195,7 +210,7 @@ namespace Umbraco.Core.Persistence.Repositories //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 - ((Entity) cc).ResetDirtyProperties(false); + cc.ResetDirtyProperties(false); } return content; @@ -205,26 +220,16 @@ namespace Umbraco.Core.Persistence.Repositories { var sql = GetBaseQuery(false); sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId }); - sql.OrderByDescending(x => x.VersionDate); + sql.OrderByDescending(x => x.VersionDate, SqlSyntax); var dto = Database.Fetch(sql).FirstOrDefault(); if (dto == null) return null; - var mediaType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); + var content = CreateMediaFromDto(dto, versionId, sql); - var factory = new MediaFactory(mediaType, NodeObjectTypeId, dto.NodeId); - var media = factory.BuildEntity(dto); - - var properties = GetPropertyCollection(sql, new[] { new DocumentDefinition(dto.NodeId, dto.VersionId, media.UpdateDate, media.CreateDate, mediaType) }); - - media.Properties = properties[dto.NodeId]; - - //on initial construction we don't want to have dirty properties tracked - // http://issues.umbraco.org/issue/U4-1946 - ((Entity)media).ResetDirtyProperties(false); - return media; + return content; } public void RebuildXmlStructures(Func serializer, int groupSize = 200, IEnumerable contentTypeIds = null) @@ -245,7 +250,7 @@ namespace Umbraco.Core.Persistence.Repositories query = query .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); query = query - .Where(x => x.NodeId > baseId) + .Where(x => x.NodeId > baseId, SqlSyntax) .OrderBy(x => x.NodeId, SqlSyntax); var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize)) .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() }) @@ -512,9 +517,8 @@ namespace Umbraco.Core.Persistence.Repositories private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql) { var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); - - var factory = new MediaFactory(contentType, NodeObjectTypeId, dto.NodeId); - var media = factory.BuildEntity(dto); + + var media = MediaFactory.BuildEntity(dto, contentType); var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType); diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index 19f4b8fdfb..16bad74612 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Diagnostics; using System.Linq; using System.Text; @@ -405,11 +406,15 @@ namespace Umbraco.Core.Persistence.Repositories } } - //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique. - // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column - // is empty for many nodes) - // see: http://issues.umbraco.org/issue/U4-8831 - sortedSql.OrderBy("umbracoNode.id"); + if (orderBySystemField && orderBy != "umbracoNode.id") + { + //no matter what we always MUST order the result also by umbracoNode.id to ensure that all records being ordered by are unique. + // if we do not do this then we end up with issues where we are ordering by a field that has duplicate values (i.e. the 'text' column + // is empty for many nodes) + // see: http://issues.umbraco.org/issue/U4-8831 + sortedSql.OrderBy("umbracoNode.id"); + } + return sortedSql; @@ -511,9 +516,9 @@ namespace Umbraco.Core.Persistence.Repositories protected IDictionary GetPropertyCollection( Sql docSql, - IEnumerable documentDefs) + IReadOnlyCollection documentDefs) { - if (documentDefs.Any() == false) return new Dictionary(); + if (documentDefs.Count == 0) return new Dictionary(); //we need to parse the original SQL statement and reduce the columns to just cmsContent.nodeId, cmsContentVersion.VersionId so that we can use // the statement to go get the property data for all of the items by using an inner join @@ -524,6 +529,15 @@ namespace Umbraco.Core.Persistence.Repositories parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal)); } + //It's Important with the sort order here! We require this to be sorted by node id, + // this is required because this data set can be huge depending on the page size. Due + // to it's size we need to be smart about iterating over the property values to build + // the document. Before we used to use Linq to get the property data for a given content node + // and perform a Distinct() call. This kills performance because that would mean if we had 7000 nodes + // and on each iteration we will perform a lookup on potentially 100,000 property rows against the node + // id which turns out to be a crazy amount of iterations. Instead we know it's sorted by this value we'll + // keep an index stored of the rows being read so we never have to re-iterate the entire data set + // on each document iteration. var propSql = new Sql(@"SELECT cmsPropertyData.* FROM cmsPropertyData INNER JOIN cmsPropertyType @@ -531,8 +545,8 @@ ON cmsPropertyData.propertytypeid = cmsPropertyType.id INNER JOIN (" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId -LEFT OUTER JOIN cmsDataTypePreValues -ON cmsPropertyType.dataTypeId = cmsDataTypePreValues.datatypeNodeId", docSql.Arguments); +ORDER BY contentNodeId, propertytypeid +", docSql.Arguments); var allPropertyData = Database.Fetch(propSql); @@ -556,59 +570,81 @@ WHERE EXISTS( }); var result = new Dictionary(); - var propertiesWithTagSupport = new Dictionary(); + //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time + var resolvedCompositionProperties = new Dictionary(); + var propertyDataSetIndex = 0; - //iterate each definition grouped by it's content type - this will mean less property type iterations while building - // up the property collections - foreach (var compositionGroup in documentDefs.GroupBy(x => x.Composition)) + //This must be sorted by node id because this is how we are sorting the query to lookup property types above, + // which allows us to more efficiently iterate over the large data set of property values + foreach (var def in documentDefs.OrderBy(x => x.Id)) { - var compositionProperties = compositionGroup.Key.CompositionPropertyTypes.ToArray(); - - foreach (var def in compositionGroup) + //get the resolved proeprties from our local cache, or resolve them and put them in cache + PropertyType[] compositionProperties; + if (resolvedCompositionProperties.ContainsKey(def.Composition.Id)) { - var propertyDataDtos = allPropertyData.Where(x => x.NodeId == def.Id).Distinct(); - - var propertyFactory = new PropertyFactory(compositionProperties, def.Version, def.Id, def.CreateDate, def.VersionDate); - var properties = propertyFactory.BuildEntity(propertyDataDtos.ToArray()).ToArray(); - - foreach (var property in properties) - { - //NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck - var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias); - - var tagSupport = propertiesWithTagSupport.ContainsKey(property.PropertyType.PropertyEditorAlias) - ? propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] - : TagExtractor.GetAttribute(editor); - - if (tagSupport != null) - { - //add to local cache so we don't need to reflect next time for this property editor alias - propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport; - - //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up - var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId) - .Distinct() - .ToArray(); - - var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder)); - - var preVals = new PreValueCollection(asDictionary); - - var contentPropData = new ContentPropertyData(property.Value, - preVals, - new Dictionary()); - - TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport); - } - } - - if (result.ContainsKey(def.Id)) - { - Logger.Warn>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name); - } - result[def.Id] = new PropertyCollection(properties); + compositionProperties = resolvedCompositionProperties[def.Composition.Id]; } + else + { + compositionProperties = def.Composition.CompositionPropertyTypes.ToArray(); + resolvedCompositionProperties[def.Composition.Id] = compositionProperties; + } + + var propertyDataDtos = new List(); + + for (var i = propertyDataSetIndex; i < allPropertyData.Count; i++) + { + if (allPropertyData[i].NodeId == def.Id) + { + propertyDataDtos.Add(allPropertyData[i]); + } + else + { + //the node id has changed so we need to exit the loop and store the index + propertyDataSetIndex = i; + break; + } + } + + var properties = PropertyFactory.BuildEntity(propertyDataDtos, compositionProperties, def.CreateDate, def.VersionDate).ToArray(); + + foreach (var property in properties) + { + //NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck + var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias); + + var tagSupport = propertiesWithTagSupport.ContainsKey(property.PropertyType.PropertyEditorAlias) + ? propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] + : TagExtractor.GetAttribute(editor); + + if (tagSupport != null) + { + //add to local cache so we don't need to reflect next time for this property editor alias + propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport; + + //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up + var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId) + .Distinct() + .ToArray(); + + var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder)); + + var preVals = new PreValueCollection(asDictionary); + + var contentPropData = new ContentPropertyData(property.Value, + preVals, + new Dictionary()); + + TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport); + } + } + + if (result.ContainsKey(def.Id)) + { + Logger.Warn>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name); + } + result[def.Id] = new PropertyCollection(properties); } return result; diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index 6f2007912f..d5658b9f9a 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -585,7 +585,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] - public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") { long total; var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter); @@ -604,7 +604,7 @@ namespace Umbraco.Core.Services /// Direction to order by /// Search text filter /// An Enumerable list of objects - public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") { return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter); } diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index 7722bf9c65..6d73546571 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -259,7 +259,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, - string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id @@ -273,7 +273,7 @@ namespace Umbraco.Core.Services /// Search text filter /// An Enumerable list of objects IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords, - string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs index d25ddf7f58..a02ac43b93 100644 --- a/src/Umbraco.Core/Services/IMediaService.cs +++ b/src/Umbraco.Core/Services/IMediaService.cs @@ -169,7 +169,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, - string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id @@ -183,7 +183,7 @@ namespace Umbraco.Core.Services /// Search text filter /// An Enumerable list of objects IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords, - string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index 9a3a30c8bd..f4e963c658 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -451,7 +451,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] - public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") { long total; var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter); @@ -470,7 +470,7 @@ namespace Umbraco.Core.Services /// Direction to order by /// Search text filter /// An Enumerable list of objects - public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "Path", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") { return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter); } From 44fc8be49ef773d8af4d22ba2668452c51635692 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 24 Jan 2017 00:35:17 +1100 Subject: [PATCH 5/7] Changes back to "path" default sorting but updates the UmbracoContentIndexer to ensure it's sorted by umbracoNode.id --- src/Umbraco.Core/Services/ContentService.cs | 4 ++-- src/Umbraco.Core/Services/IContentService.cs | 4 ++-- src/Umbraco.Core/Services/IMediaService.cs | 4 ++-- src/Umbraco.Core/Services/MediaService.cs | 4 ++-- src/UmbracoExamine/UmbracoContentIndexer.cs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index d5658b9f9a..32665910ad 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -585,7 +585,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] - public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "") { long total; var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter); @@ -604,7 +604,7 @@ namespace Umbraco.Core.Services /// Direction to order by /// Search text filter /// An Enumerable list of objects - public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "") { return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter); } diff --git a/src/Umbraco.Core/Services/IContentService.cs b/src/Umbraco.Core/Services/IContentService.cs index 6d73546571..ff515169d9 100644 --- a/src/Umbraco.Core/Services/IContentService.cs +++ b/src/Umbraco.Core/Services/IContentService.cs @@ -259,7 +259,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, - string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id @@ -273,7 +273,7 @@ namespace Umbraco.Core.Services /// Search text filter /// An Enumerable list of objects IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords, - string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs index a02ac43b93..d4218764b7 100644 --- a/src/Umbraco.Core/Services/IMediaService.cs +++ b/src/Umbraco.Core/Services/IMediaService.cs @@ -169,7 +169,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, - string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id @@ -183,7 +183,7 @@ namespace Umbraco.Core.Services /// Search text filter /// An Enumerable list of objects IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalRecords, - string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = ""); + string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = ""); /// /// Gets a collection of objects by Parent Id diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index f4e963c658..a67ee8285d 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -451,7 +451,7 @@ namespace Umbraco.Core.Services [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] - public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "") { long total; var result = GetPagedDescendants(id, Convert.ToInt64(pageIndex), pageSize, out total, orderBy, orderDirection, true, filter); @@ -470,7 +470,7 @@ namespace Umbraco.Core.Services /// Direction to order by /// Search text filter /// An Enumerable list of objects - public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "umbracoNode.id", Direction orderDirection = Direction.Ascending, string filter = "") + public IEnumerable GetPagedDescendants(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy = "path", Direction orderDirection = Direction.Ascending, string filter = "") { return GetPagedDescendants(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter); } diff --git a/src/UmbracoExamine/UmbracoContentIndexer.cs b/src/UmbracoExamine/UmbracoContentIndexer.cs index f7e4c45f50..e7df641122 100644 --- a/src/UmbracoExamine/UmbracoContentIndexer.cs +++ b/src/UmbracoExamine/UmbracoContentIndexer.cs @@ -497,7 +497,7 @@ namespace UmbracoExamine IContent[] descendants; if (SupportUnpublishedContent) { - descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total).ToArray(); + descendants = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out total, "umbracoNode.id").ToArray(); } else { From 24c053d6713a081a17bbc95fa1e9bbcb2501e954 Mon Sep 17 00:00:00 2001 From: Mads Rasmussen Date: Mon, 23 Jan 2017 16:11:09 +0100 Subject: [PATCH 6/7] Fixes: U4-9427 Unable to install local packages in various browsers --- .../views/install-local.controller.js | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/packager/views/install-local.controller.js b/src/Umbraco.Web.UI.Client/src/views/packager/views/install-local.controller.js index e34bc48ecd..9ce2506e76 100644 --- a/src/Umbraco.Web.UI.Client/src/views/packager/views/install-local.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/packager/views/install-local.controller.js @@ -34,18 +34,24 @@ fields: {}, file: file }).progress(function (evt) { + + // hack: in some browsers the progress event is called after success + // this prevents the UI from going back to a uploading state + if(vm.zipFile.uploadStatus !== "done" && vm.zipFile.uploadStatus !== "error") { - // set view state to uploading - vm.state = 'uploading'; + // set view state to uploading + vm.state = 'uploading'; - // calculate progress in percentage - var progressPercentage = parseInt(100.0 * evt.loaded / evt.total, 10); + // calculate progress in percentage + var progressPercentage = parseInt(100.0 * evt.loaded / evt.total, 10); - // set percentage property on file - vm.zipFile.uploadProgress = progressPercentage; + // set percentage property on file + vm.zipFile.uploadProgress = progressPercentage; - // set uploading status on file - vm.zipFile.uploadStatus = "uploading"; + // set uploading status on file + vm.zipFile.uploadStatus = "uploading"; + + } }).success(function (data, status, headers, config) { From c7b505fd908d926c194b78907b02e93c7b043c88 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 24 Jan 2017 16:04:26 +1100 Subject: [PATCH 7/7] Updates the paging logic so that the ContentRepository uses 2x queries: A full query and a query for getting Ids. This is important because the query for fetching ids is used for paging and for getting property data and it can be much much faster than the full query which was previously used for both. There's not really any changes to the media/members respositories since their full queries don't have certain outer joins that make them run really slow --- .../Persistence/Repositories/BaseQueryType.cs | 9 + .../Repositories/ContentRepository.cs | 174 +++++++++---- .../Repositories/MediaRepository.cs | 15 +- .../Repositories/MemberRepository.cs | 31 ++- .../Repositories/VersionableRepositoryBase.cs | 246 +++++++++--------- src/Umbraco.Core/Umbraco.Core.csproj | 1 + 6 files changed, 285 insertions(+), 191 deletions(-) create mode 100644 src/Umbraco.Core/Persistence/Repositories/BaseQueryType.cs diff --git a/src/Umbraco.Core/Persistence/Repositories/BaseQueryType.cs b/src/Umbraco.Core/Persistence/Repositories/BaseQueryType.cs new file mode 100644 index 0000000000..4579fd98fb --- /dev/null +++ b/src/Umbraco.Core/Persistence/Repositories/BaseQueryType.cs @@ -0,0 +1,9 @@ +namespace Umbraco.Core.Persistence.Repositories +{ + internal enum BaseQueryType + { + Full, + Ids, + Count + } +} \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index b25db8afb2..b02f5304e6 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -70,63 +70,90 @@ namespace Umbraco.Core.Persistence.Repositories protected override IEnumerable PerformGetAll(params int[] ids) { - var sql = GetBaseQuery(false); - if (ids.Any()) + Func translate = s => { - sql.Where("umbracoNode.id in (@ids)", new { ids }); - } + if (ids.Any()) + { + s.Where("umbracoNode.id in (@ids)", new { ids }); + } + //we only want the newest ones with this method + s.Where(x => x.Newest, SqlSyntax); + return s; + }; + + var sqlBaseFull = GetBaseQuery(BaseQueryType.Full); + var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids); - //we only want the newest ones with this method - sql.Where(x => x.Newest, SqlSyntax); - - return ProcessQuery(sql); + return ProcessQuery(translate(sqlBaseFull), translate(sqlBaseIds)); } protected override IEnumerable PerformGetByQuery(IQuery query) { - var sqlClause = GetBaseQuery(false); - var translator = new SqlTranslator(sqlClause, query); - var sql = translator.Translate() - .Where(x => x.Newest, SqlSyntax) - .OrderByDescending(x => x.VersionDate, SqlSyntax) - .OrderBy(x => x.SortOrder, SqlSyntax); + var sqlBaseFull = GetBaseQuery(BaseQueryType.Full); + var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids); - return ProcessQuery(sql); + Func, Sql> translate = (translator) => + { + return translator.Translate() + .Where(x => x.Newest, SqlSyntax) + .OrderByDescending(x => x.VersionDate, SqlSyntax) + .OrderBy(x => x.SortOrder, SqlSyntax); + }; + + var translatorFull = new SqlTranslator(sqlBaseFull, query); + var translatorIds = new SqlTranslator(sqlBaseIds, query); + + return ProcessQuery(translate(translatorFull), translate(translatorIds)); } #endregion #region Overrides of PetaPocoRepositoryBase - - protected override Sql GetBaseQuery(bool isCount) + protected override Sql GetBaseQuery(BaseQueryType queryType) { - var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)", + var sql = new Sql(); + sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsDocument.nodeId" : "*")) + .From(SqlSyntax) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.VersionId, right => right.VersionId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId); + + if (queryType == BaseQueryType.Full) + { + //The only reason we apply this left outer join is to be able to pull back the DocumentPublishedReadOnlyDto + //information with the entire data set, so basically this will get both the latest document and also it's published + //version if it has one. When performing a count or when just retrieving Ids like in paging, this is unecessary + //and causes huge performance overhead for the SQL server, especially when sorting the result. + //To fix this perf overhead we'd need another index on : + // CREATE NON CLUSTERED INDEX ON cmsDocument.node + cmsDocument.published + + var sqlx = string.Format("LEFT OUTER JOIN {0} {1} ON ({1}.{2}={0}.{2} AND {1}.{3}=1)", SqlSyntax.GetQuotedTableName("cmsDocument"), SqlSyntax.GetQuotedTableName("cmsDocument2"), SqlSyntax.GetQuotedColumnName("nodeId"), SqlSyntax.GetQuotedColumnName("published")); - var sql = new Sql(); - sql.Select(isCount ? "COUNT(*)" : "*") - .From() - .InnerJoin() - .On(left => left.VersionId, right => right.VersionId) - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - // cannot do this because PetaPoco does not know how to alias the table //.LeftOuterJoin() //.On(left => left.NodeId, right => right.NodeId) // so have to rely on writing our own SQL - .Append(sqlx/*, new { @published = true }*/) + sql.Append(sqlx /*, new { @published = true }*/); + } + + sql.Where(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax); - .Where(x => x.NodeObjectType == NodeObjectTypeId); return sql; } + protected override Sql GetBaseQuery(bool isCount) + { + return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full); + } + protected override string GetBaseWhereClause() { return "umbracoNode.id = @Id"; @@ -173,20 +200,32 @@ namespace Umbraco.Core.Persistence.Repositories // not bring much safety - so this reverts to updating each record individually, // and it may be slower in the end, but should be more resilient. - var baseId = 0; var contentTypeIdsA = contentTypeIds == null ? new int[0] : contentTypeIds.ToArray(); + + Func translate = (bId, sql) => + { + if (contentTypeIdsA.Length > 0) + { + sql.WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); + } + + sql + .Where(x => x.NodeId > bId && x.Trashed == false, SqlSyntax) + .Where(x => x.Published, SqlSyntax) + .OrderBy(x => x.NodeId, SqlSyntax); + + return sql; + }; + + var baseId = 0; + while (true) { // get the next group of nodes - var query = GetBaseQuery(false); - if (contentTypeIdsA.Length > 0) - query = query - .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); - query = query - .Where(x => x.NodeId > baseId && x.Trashed == false, SqlSyntax) - .Where(x => x.Published, SqlSyntax) - .OrderBy(x => x.NodeId, SqlSyntax); - var xmlItems = ProcessQuery(SqlSyntax.SelectTop(query, groupSize)) + var sqlFull = translate(baseId, GetBaseQuery(BaseQueryType.Full)); + var sqlIds = translate(baseId, GetBaseQuery(BaseQueryType.Ids)); + + var xmlItems = ProcessQuery(SqlSyntax.SelectTop(sqlFull, groupSize), SqlSyntax.SelectTop(sqlIds, groupSize)) .Select(x => new ContentXmlDto { NodeId = x.Id, Xml = serializer(x).ToString() }) .ToList(); @@ -212,10 +251,16 @@ namespace Umbraco.Core.Persistence.Repositories public override IEnumerable GetAllVersions(int id) { - var sql = GetBaseQuery(false) - .Where(GetBaseWhereClause(), new { Id = id }) - .OrderByDescending(x => x.VersionDate, SqlSyntax); - return ProcessQuery(sql, true); + Func translate = s => + { + return s.Where(GetBaseWhereClause(), new {Id = id}) + .OrderByDescending(x => x.VersionDate, SqlSyntax); + }; + + var sqlFull = translate(GetBaseQuery(BaseQueryType.Full)); + var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids)); + + return ProcessQuery(sqlFull, sqlIds, true); } public override IContent GetByVersion(Guid versionId) @@ -616,19 +661,25 @@ namespace Umbraco.Core.Persistence.Repositories public IEnumerable GetByPublishedVersion(IQuery query) { + Func, Sql> translate = t => + { + return t.Translate() + .Where(x => x.Published, SqlSyntax) + .OrderBy(x => x.Level, SqlSyntax) + .OrderBy(x => x.SortOrder, SqlSyntax); + }; + // we WANT to return contents in top-down order, ie parents should come before children // ideal would be pure xml "document order" which can be achieved with: // ORDER BY substring(path, 1, len(path) - charindex(',', reverse(path))), sortOrder // but that's probably an overkill - sorting by level,sortOrder should be enough - var sqlClause = GetBaseQuery(false); - var translator = new SqlTranslator(sqlClause, query); - var sql = translator.Translate() - .Where(x => x.Published, SqlSyntax) - .OrderBy(x => x.Level, SqlSyntax) - .OrderBy(x => x.SortOrder, SqlSyntax); + var sqlFull = GetBaseQuery(BaseQueryType.Full); + var translatorFull = new SqlTranslator(sqlFull, query); + var sqlIds = GetBaseQuery(BaseQueryType.Ids); + var translatorIds = new SqlTranslator(sqlIds, query); - return ProcessQuery(sql, true); + return ProcessQuery(translate(translatorFull), translate(translatorIds), true); } /// @@ -806,9 +857,9 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", Func> filterCallback = () => new Tuple(filterSql.SQL, filterSql.Arguments); - return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, + return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, new Tuple("cmsDocument", "nodeId"), - sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField, + (sqlFull, sqlIds) => ProcessQuery(sqlFull, sqlIds), orderBy, orderDirection, orderBySystemField, filterCallback); } @@ -839,10 +890,21 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", return base.GetDatabaseFieldNameForOrderBy(orderBy); } - private IEnumerable ProcessQuery(Sql sql, bool withCache = false) + /// + /// This is the underlying method that processes most queries for this repository + /// + /// + /// The full SQL with the outer join to return all data required to create an IContent + /// + /// + /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item + /// + /// + /// + private IEnumerable ProcessQuery(Sql sqlFull, Sql sqlIds, bool withCache = false) { // fetch returns a list so it's ok to iterate it in this method - var dtos = Database.Fetch(sql); + var dtos = Database.Fetch(sqlFull); if (dtos.Count == 0) return Enumerable.Empty(); var content = new IContent[dtos.Count]; @@ -905,7 +967,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", .ToDictionary(x => x.Id, x => x); // load all properties for all documents from database in 1 query - var propertyData = GetPropertyCollection(sql, defs); + var propertyData = GetPropertyCollection(sqlIds, defs); // assign var dtoIndex = 0; diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index bf034bd8ff..8cfb037c6c 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -84,11 +84,11 @@ namespace Umbraco.Core.Persistence.Repositories #endregion #region Overrides of PetaPocoRepositoryBase - - protected override Sql GetBaseQuery(bool isCount) + + protected override Sql GetBaseQuery(BaseQueryType queryType) { var sql = new Sql(); - sql.Select(isCount ? "COUNT(*)" : "*") + sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsContentVersion.contentId" : "*")) .From(SqlSyntax) .InnerJoin(SqlSyntax) .On(SqlSyntax, left => left.NodeId, right => right.NodeId) @@ -98,6 +98,11 @@ namespace Umbraco.Core.Persistence.Repositories return sql; } + protected override Sql GetBaseQuery(bool isCount) + { + return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full); + } + protected override string GetBaseWhereClause() { return "umbracoNode.id = @Id"; @@ -500,9 +505,9 @@ namespace Umbraco.Core.Persistence.Repositories filterCallback = () => new Tuple(sbWhere.ToString().Trim(), args.ToArray()); } - return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, + return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, new Tuple("cmsContentVersion", "contentId"), - sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField, + (sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField, filterCallback); } diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index a92794775f..8b3bf4a471 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -107,24 +107,29 @@ namespace Umbraco.Core.Persistence.Repositories #region Overrides of PetaPocoRepositoryBase - protected override Sql GetBaseQuery(bool isCount) + protected override Sql GetBaseQuery(BaseQueryType queryType) { var sql = new Sql(); - sql.Select(isCount ? "COUNT(*)" : "*") - .From() - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) + sql.Select(queryType == BaseQueryType.Count ? "COUNT(*)" : (queryType == BaseQueryType.Ids ? "cmsMember.nodeId" : "*")) + .From(SqlSyntax) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId) //We're joining the type so we can do a query against the member type - not sure if this adds much overhead or not? // the execution plan says it doesn't so we'll go with that and in that case, it might be worth joining the content // types by default on the document and media repo's so we can query by content type there too. - .InnerJoin().On(left => left.NodeId, right => right.ContentTypeId) - .InnerJoin() - .On(left => left.NodeId, right => right.NodeId) - .Where(x => x.NodeObjectType == NodeObjectTypeId); + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.ContentTypeId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId) + .Where(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax); return sql; + } + protected override Sql GetBaseQuery(bool isCount) + { + return GetBaseQuery(isCount ? BaseQueryType.Count : BaseQueryType.Full); } protected override string GetBaseWhereClause() @@ -617,9 +622,9 @@ namespace Umbraco.Core.Persistence.Repositories filterCallback = () => new Tuple(sbWhere.ToString().Trim(), args.ToArray()); } - return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, + return GetPagedResultsByQuery(query, pageIndex, pageSize, out totalRecords, new Tuple("cmsMember", "nodeId"), - sql => ProcessQuery(sql), orderBy, orderDirection, orderBySystemField, + (sqlFull, sqlIds) => ProcessQuery(sqlFull), orderBy, orderDirection, orderBySystemField, filterCallback); } diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index 16bad74612..31dd75b3d1 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -27,8 +27,6 @@ using Umbraco.Core.IO; namespace Umbraco.Core.Persistence.Repositories { - using SqlSyntax; - internal abstract class VersionableRepositoryBase : PetaPocoRepositoryBase where TEntity : class, IAggregateRoot { @@ -384,12 +382,8 @@ namespace Umbraco.Core.Persistence.Repositories ON CustomPropData.CustomPropValContentId = umbracoNode.id ", sortedInt, sortedDecimal, sortedDate, sortedString, nodeIdSelect.Item2, nodeIdSelect.Item1, versionQuery, sortedSql.Arguments.Length, newestQuery); - //insert this just above the first LEFT OUTER JOIN (for cmsDocument) or the last WHERE (everything else) - string newSql; - if (nodeIdSelect.Item1 == "cmsDocument") - newSql = sortedSql.SQL.Insert(sortedSql.SQL.IndexOf("LEFT OUTER JOIN"), outerJoinTempTable); - else - newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable); + //insert this just above the last WHERE + string newSql = sortedSql.SQL.Insert(sortedSql.SQL.LastIndexOf("WHERE"), outerJoinTempTable); var newArgs = sortedSql.Arguments.ToList(); newArgs.Add(orderBy); @@ -414,7 +408,6 @@ namespace Umbraco.Core.Persistence.Repositories // see: http://issues.umbraco.org/issue/U4-8831 sortedSql.OrderBy("umbracoNode.id"); } - return sortedSql; @@ -424,7 +417,6 @@ namespace Umbraco.Core.Persistence.Repositories /// A helper method for inheritors to get the paged results by query in a way that minimizes queries /// /// The type of the d. - /// The 'true' entity type (i.e. Content, Member, etc...) /// The query. /// Index of the page. /// Size of the page. @@ -437,34 +429,30 @@ namespace Umbraco.Core.Persistence.Repositories /// Flag to indicate when ordering by system field /// /// orderBy - protected IEnumerable GetPagedResultsByQuery(IQuery query, long pageIndex, int pageSize, out long totalRecords, + protected IEnumerable GetPagedResultsByQuery(IQuery query, long pageIndex, int pageSize, out long totalRecords, Tuple nodeIdSelect, - Func> processQuery, + Func> processQuery, string orderBy, Direction orderDirection, bool orderBySystemField, Func> defaultFilter = null) - where TContentBase : class, IAggregateRoot, TEntity { if (orderBy == null) throw new ArgumentNullException("orderBy"); - // Get base query - var sqlBase = GetBaseQuery(false); + // Get base query for returning IDs + var sqlBaseIds = GetBaseQuery(BaseQueryType.Ids); + // Get base query for returning all data + var sqlBaseFull = GetBaseQuery(BaseQueryType.Full); if (query == null) query = new Query(); - var translator = new SqlTranslator(sqlBase, query); - var sqlQuery = translator.Translate(); - - // Note we can't do multi-page for several DTOs like we can multi-fetch and are doing in PerformGetByQuery, - // but actually given we are doing a Get on each one (again as in PerformGetByQuery), we only need the node Id. - // So we'll modify the SQL. - var sqlNodeIds = new Sql( - sqlQuery.SQL.Replace("SELECT *", string.Format("SELECT {0}.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2)), - sqlQuery.Arguments); + var translatorIds = new SqlTranslator(sqlBaseIds, query); + var sqlQueryIds = translatorIds.Translate(); + var translatorFull = new SqlTranslator(sqlBaseFull, query); + var sqlQueryFull = translatorFull.Translate(); //get sorted and filtered sql var sqlNodeIdsWithSort = GetSortedSqlForPagedResults( - GetFilteredSqlForPagedResults(sqlNodeIds, defaultFilter), + GetFilteredSqlForPagedResults(sqlQueryIds, defaultFilter), orderDirection, orderBy, orderBySystemField, nodeIdSelect); // Get page of results and total count @@ -480,31 +468,25 @@ namespace Umbraco.Core.Persistence.Repositories //Crete the inner paged query that was used above to get the paged result, we'll use that as the inner sub query var args = sqlNodeIdsWithSort.Arguments; string sqlStringCount, sqlStringPage; - Database.BuildPageQueries(pageIndex * pageSize, pageSize, sqlNodeIdsWithSort.SQL, ref args, out sqlStringCount, out sqlStringPage); + Database.BuildPageQueries(pageIndex * pageSize, pageSize, sqlNodeIdsWithSort.SQL, ref args, out sqlStringCount, out sqlStringPage); - //if this is for sql server, the sqlPage will start with a SELECT * but we don't want that, we only want to return the nodeId - sqlStringPage = sqlStringPage - .Replace("SELECT *", - //This ensures we only take the field name of the node id select and not the table name - since the resulting select - // will ony work with the field name. - "SELECT " + nodeIdSelect.Item2); - - //We need to make this an inner join on the paged query - var splitQuery = sqlQuery.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None); - var withInnerJoinSql = new Sql(splitQuery[0]) + //We need to make this FULL query an inner join on the paged ID query + var splitQuery = sqlQueryFull.SQL.Split(new[] { "WHERE " }, StringSplitOptions.None); + var fullQueryWithPagedInnerJoin = new Sql(splitQuery[0]) .Append("INNER JOIN (") //join the paged query with the paged query arguments .Append(sqlStringPage, args) .Append(") temp ") .Append(string.Format("ON {0}.{1} = temp.{1}", nodeIdSelect.Item1, nodeIdSelect.Item2)) //add the original where clause back with the original arguments - .Where(splitQuery[1], sqlQuery.Arguments); + .Where(splitQuery[1], sqlQueryIds.Arguments); //get sorted and filtered sql var fullQuery = GetSortedSqlForPagedResults( - GetFilteredSqlForPagedResults(withInnerJoinSql, defaultFilter), + GetFilteredSqlForPagedResults(fullQueryWithPagedInnerJoin, defaultFilter), orderDirection, orderBy, orderBySystemField, nodeIdSelect); - return processQuery(fullQuery); + + return processQuery(fullQuery, sqlNodeIdsWithSort); } else { @@ -529,6 +511,25 @@ namespace Umbraco.Core.Persistence.Repositories parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal)); } + //This retrieves all pre-values for all data types that are referenced for all property types + // that exist in the data set. + //Benchmarks show that eagerly loading these so that we can lazily read the property data + // below (with the use of Query intead of Fetch) go about 30% faster, so we'll eagerly load + // this now since we cannot execute another reader inside of reading the property data. + var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId +FROM cmsDataTypePreValues a +WHERE EXISTS( + SELECT DISTINCT b.id as preValIdInner + FROM cmsDataTypePreValues b + INNER JOIN cmsPropertyType + ON b.datatypeNodeId = cmsPropertyType.dataTypeId + INNER JOIN + (" + string.Format(parsedOriginalSql, "DISTINCT cmsContent.contentType") + @") as docData + ON cmsPropertyType.contentTypeId = docData.contentType + WHERE a.id = b.id)", docSql.Arguments); + + var allPreValues = Database.Fetch(preValsSql); + //It's Important with the sort order here! We require this to be sorted by node id, // this is required because this data set can be huge depending on the page size. Due // to it's size we need to be smart about iterating over the property values to build @@ -548,106 +549,110 @@ ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNode ORDER BY contentNodeId, propertytypeid ", docSql.Arguments); - var allPropertyData = Database.Fetch(propSql); - - //This is a lazy access call to get all prevalue data for the data types that make up all of these properties which we use - // below if any property requires tag support - var allPreValues = new Lazy>(() => - { - var preValsSql = new Sql(@"SELECT a.id, a.value, a.sortorder, a.alias, a.datatypeNodeId -FROM cmsDataTypePreValues a -WHERE EXISTS( - SELECT DISTINCT b.id as preValIdInner - FROM cmsDataTypePreValues b - INNER JOIN cmsPropertyType - ON b.datatypeNodeId = cmsPropertyType.dataTypeId - INNER JOIN - (" + string.Format(parsedOriginalSql, "DISTINCT cmsContent.contentType") + @") as docData - ON cmsPropertyType.contentTypeId = docData.contentType - WHERE a.id = b.id)", docSql.Arguments); - - return Database.Fetch(preValsSql); - }); + //This does NOT fetch all data into memory in a list, this will read + // over the records as a data reader, this is much better for performance and memory, + // but it means that during the reading of this data set, nothing else can be read + // from SQL server otherwise we'll get an exception. + var allPropertyData = Database.Query(propSql); var result = new Dictionary(); var propertiesWithTagSupport = new Dictionary(); //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time var resolvedCompositionProperties = new Dictionary(); - var propertyDataSetIndex = 0; - //This must be sorted by node id because this is how we are sorting the query to lookup property types above, - // which allows us to more efficiently iterate over the large data set of property values - foreach (var def in documentDefs.OrderBy(x => x.Id)) + //keep track of the current property data item being enumerated + var propertyDataSetEnumerator = allPropertyData.GetEnumerator(); + + try { - //get the resolved proeprties from our local cache, or resolve them and put them in cache - PropertyType[] compositionProperties; - if (resolvedCompositionProperties.ContainsKey(def.Composition.Id)) + //This must be sorted by node id because this is how we are sorting the query to lookup property types above, + // which allows us to more efficiently iterate over the large data set of property values + foreach (var def in documentDefs.OrderBy(x => x.Id)) { - compositionProperties = resolvedCompositionProperties[def.Composition.Id]; - } - else - { - compositionProperties = def.Composition.CompositionPropertyTypes.ToArray(); - resolvedCompositionProperties[def.Composition.Id] = compositionProperties; - } - - var propertyDataDtos = new List(); - - for (var i = propertyDataSetIndex; i < allPropertyData.Count; i++) - { - if (allPropertyData[i].NodeId == def.Id) + //get the resolved proeprties from our local cache, or resolve them and put them in cache + PropertyType[] compositionProperties; + if (resolvedCompositionProperties.ContainsKey(def.Composition.Id)) { - propertyDataDtos.Add(allPropertyData[i]); + compositionProperties = resolvedCompositionProperties[def.Composition.Id]; } else { - //the node id has changed so we need to exit the loop and store the index - propertyDataSetIndex = i; - break; + compositionProperties = def.Composition.CompositionPropertyTypes.ToArray(); + resolvedCompositionProperties[def.Composition.Id] = compositionProperties; } - } - - var properties = PropertyFactory.BuildEntity(propertyDataDtos, compositionProperties, def.CreateDate, def.VersionDate).ToArray(); - foreach (var property in properties) - { - //NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck - var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias); + var propertyDataDtos = new List(); - var tagSupport = propertiesWithTagSupport.ContainsKey(property.PropertyType.PropertyEditorAlias) - ? propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] - : TagExtractor.GetAttribute(editor); - - if (tagSupport != null) + //Check if there is a current enumerated item and check if we match and add it + if (propertyDataSetEnumerator.Current != null) { - //add to local cache so we don't need to reflect next time for this property editor alias - propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport; - - //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up - var preValData = allPreValues.Value.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId) - .Distinct() - .ToArray(); - - var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder)); - - var preVals = new PreValueCollection(asDictionary); - - var contentPropData = new ContentPropertyData(property.Value, - preVals, - new Dictionary()); - - TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport); + if (propertyDataSetEnumerator.Current.NodeId == def.Id) + { + propertyDataDtos.Add(propertyDataSetEnumerator.Current); + } + } + //Move to the next position, see if we match and add it, if not exit + while (propertyDataSetEnumerator.MoveNext()) + { + if (propertyDataSetEnumerator.Current.NodeId == def.Id) + { + propertyDataDtos.Add(propertyDataSetEnumerator.Current); + } + else + { + //the node id has changed so we need to exit the loop, + //the enumerator position will be maintained + break; + } } - } - if (result.ContainsKey(def.Id)) - { - Logger.Warn>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name); + var properties = PropertyFactory.BuildEntity(propertyDataDtos, compositionProperties, def.CreateDate, def.VersionDate).ToArray(); + + foreach (var property in properties) + { + //NOTE: The benchmarks run with and without the following code show very little change so this is not a perf bottleneck + var editor = PropertyEditorResolver.Current.GetByAlias(property.PropertyType.PropertyEditorAlias); + + var tagSupport = propertiesWithTagSupport.ContainsKey(property.PropertyType.PropertyEditorAlias) + ? propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] + : TagExtractor.GetAttribute(editor); + + if (tagSupport != null) + { + //add to local cache so we don't need to reflect next time for this property editor alias + propertiesWithTagSupport[property.PropertyType.PropertyEditorAlias] = tagSupport; + + //this property has tags, so we need to extract them and for that we need the prevals which we've already looked up + var preValData = allPreValues.Where(x => x.DataTypeNodeId == property.PropertyType.DataTypeDefinitionId) + .Distinct() + .ToArray(); + + var asDictionary = preValData.ToDictionary(x => x.Alias, x => new PreValue(x.Id, x.Value, x.SortOrder)); + + var preVals = new PreValueCollection(asDictionary); + + var contentPropData = new ContentPropertyData(property.Value, + preVals, + new Dictionary()); + + TagExtractor.SetPropertyTags(property, contentPropData, property.Value, tagSupport); + } + } + + if (result.ContainsKey(def.Id)) + { + Logger.Warn>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name); + } + result[def.Id] = new PropertyCollection(properties); } - result[def.Id] = new PropertyCollection(properties); + } + finally + { + propertyDataSetEnumerator.Dispose(); } return result; + } public class DocumentDefinition @@ -759,5 +764,12 @@ WHERE EXISTS( return allsuccess; } + + /// + /// For Paging, repositories must support returning different query for the query type specified + /// + /// + /// + protected abstract Sql GetBaseQuery(BaseQueryType queryType); } } \ No newline at end of file diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 17be8e62df..8ba327c9c9 100644 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -466,6 +466,7 @@ +