From acf10eeef0d5b6be4ac83f70ee933ef5fd8e24fd Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Thu, 16 Feb 2017 08:11:35 +0100 Subject: [PATCH 01/41] U4-9532 logic in ContentRepository.ClearPublished is inefficient - cherry picked from 7.6.0 --- .../Persistence/Repositories/ContentRepository.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index ff466ea84e..8488bdb65a 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -784,13 +784,8 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", public void ClearPublished(IContent content) { - // race cond! - var documentDtos = Database.Fetch("WHERE nodeId=@id AND published=@published", new { id = content.Id, published = true }); - foreach (var documentDto in documentDtos) - { - documentDto.Published = false; - Database.Update(documentDto); - } + var sql = "UPDATE cmsDocument SET published=0 WHERE nodeId=@id AND published=1"; + Database.Execute(sql, new {id = content.Id}); } /// From af287c387ebd733243d889adbfb3647db88ae23e Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 16 Feb 2017 18:54:44 +1100 Subject: [PATCH 02/41] Adds test to show that the content repo can deal with corrupted data, this fix actually simplifies a bunch of logic too and should consume slightly less memory, have added some TODOs in the code and need to then write more tests and code to deal with this in the EntityRepository. --- .../Models/PropertyTypeCollection.cs | 1 + .../Persistence/Factories/MemberFactory.cs | 12 ++- .../Repositories/ContentRepository.cs | 60 +++++------- .../Repositories/MediaRepository.cs | 47 ++++----- .../Repositories/MemberRepository.cs | 53 ++++------ .../Repositories/VersionableRepositoryBase.cs | 96 +++++++++++++++++-- .../Repositories/ContentRepositoryTest.cs | 52 ++++++++++ 7 files changed, 211 insertions(+), 110 deletions(-) diff --git a/src/Umbraco.Core/Models/PropertyTypeCollection.cs b/src/Umbraco.Core/Models/PropertyTypeCollection.cs index 38abd0c57d..a06e6d737d 100644 --- a/src/Umbraco.Core/Models/PropertyTypeCollection.cs +++ b/src/Umbraco.Core/Models/PropertyTypeCollection.cs @@ -69,6 +69,7 @@ namespace Umbraco.Core.Models OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } + //TODO: Instead of 'new' this should explicitly implement one of the collection interfaces members internal new void Add(PropertyType item) { using (new WriteLock(_addLocker)) diff --git a/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs b/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs index 2901f48539..7b28808429 100644 --- a/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs +++ b/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs @@ -28,17 +28,17 @@ namespace Umbraco.Core.Persistence.Factories #region Implementation of IEntityFactory - public IMember BuildEntity(MemberDto dto) + public static IMember BuildEntity(MemberDto dto, IMemberType contentType) { var member = new Member( dto.ContentVersionDto.ContentDto.NodeDto.Text, - dto.Email, dto.LoginName, dto.Password, _contentType); + dto.Email, dto.LoginName, dto.Password, contentType); try { member.DisableChangeTracking(); - member.Id = _id; + member.Id = dto.NodeId; member.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId; member.Path = dto.ContentVersionDto.ContentDto.NodeDto.Path; member.CreatorId = dto.ContentVersionDto.ContentDto.NodeDto.UserId.Value; @@ -62,6 +62,12 @@ namespace Umbraco.Core.Persistence.Factories } } + [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")] + public IMember BuildEntity(MemberDto dto) + { + return BuildEntity(dto, _contentType); + } + public MemberDto BuildDto(IMember entity) { var dto = new MemberDto diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index ff466ea84e..0e1228960e 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -64,7 +64,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateContentFromDto(dto, dto.ContentVersionDto.VersionId, sql); + var content = CreateContentFromDto(dto, sql); return content; } @@ -287,7 +287,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateContentFromDto(dto, versionId, sql); + var content = CreateContentFromDto(dto, sql); return content; } @@ -933,13 +933,14 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal)); } + //order by update date DESC, if there is corrupted published flags we only want the latest! var publishedSql = new Sql(@"SELECT * FROM cmsDocument AS doc2 INNER JOIN (" + parsedOriginalSql + @") as docData ON doc2.nodeId = docData.nodeId WHERE doc2.published = 1 -ORDER BY doc2.nodeId +ORDER BY doc2.updateDate DESC ", sqlFull.Arguments); //go and get the published version data, we do a Query here and not a Fetch so we are @@ -955,8 +956,8 @@ ORDER BY doc2.nodeId } - var content = new IContent[dtos.Count]; - var defs = new List(); + var content = new List(); + var defs = new DocumentDefinitionCollection(); var templateIds = new List(); //track the looked up content types, even though the content types are cached @@ -964,9 +965,8 @@ ORDER BY doc2.nodeId // the overhead of deep cloning them on every item in this loop var contentTypes = new Dictionary(); - for (var i = 0; i < dtos.Count; i++) + foreach (var dto in dtos) { - var dto = dtos[i]; DocumentPublishedReadOnlyDto publishedDto; publishedDataCollection.TryGetValue(dto.NodeId, out publishedDto); @@ -975,9 +975,10 @@ ORDER BY doc2.nodeId { var cached = RuntimeCache.GetCacheItem(GetCacheIdKey(dto.NodeId)); //only use this cached version if the dto returned is also the publish version, they must match + //TODO: Shouldn't this also match on version!? if (cached != null && cached.Published && dto.Published) { - content[i] = cached; + content.Add(cached); continue; } } @@ -996,20 +997,15 @@ ORDER BY doc2.nodeId contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType; } - content[i] = ContentFactory.BuildEntity(dto, contentType, publishedDto); + // track the definition and if it's successfully added or updated then processed + if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType))) + { + // need template + if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0) + templateIds.Add(dto.TemplateId.Value); - // need template - if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0) - templateIds.Add(dto.TemplateId.Value); - - // need properties - defs.Add(new DocumentDefinition( - dto.NodeId, - dto.VersionId, - dto.ContentVersionDto.VersionDate, - dto.ContentVersionDto.ContentDto.NodeDto.CreateDate, - contentType - )); + content.Add(ContentFactory.BuildEntity(dto, contentType, publishedDto)); + } } // load all required templates in 1 query @@ -1019,26 +1015,21 @@ ORDER BY doc2.nodeId // load all properties for all documents from database in 1 query var propertyData = GetPropertyCollection(pagingSqlQuery, defs); - // assign - var dtoIndex = 0; - foreach (var def in defs) + // assign template and property data + foreach (var cc in content) { - // move to corresponding item (which has to exist) - while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++; + var def = defs[cc.Id]; - // complete the item - var cc = content[dtoIndex]; - var dto = dtos[dtoIndex]; ITemplate template = null; - if (dto.TemplateId.HasValue) - templates.TryGetValue(dto.TemplateId.Value, out template); // else null + if (def.DocumentDto.TemplateId.HasValue) + templates.TryGetValue(def.DocumentDto.TemplateId.Value, out template); // else null cc.Template = template; cc.Properties = propertyData[cc.Id]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 cc.ResetDirtyProperties(false); - } + } return content; } @@ -1047,10 +1038,9 @@ ORDER BY doc2.nodeId /// Private method to create a content object from a DocumentDto, which is used by Get and GetByVersion. /// /// - /// /// /// - private IContent CreateContentFromDto(DocumentDto dto, Guid versionId, Sql docSql) + private IContent CreateContentFromDto(DocumentDto dto, Sql docSql) { var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); @@ -1062,7 +1052,7 @@ ORDER BY doc2.nodeId content.Template = _templateRepository.Get(dto.TemplateId.Value); } - var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType); + var docDef = new DocumentDefinition(dto, contentType); var properties = GetPropertyCollection(docSql, new[] { docDef }); diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index 53b04e1322..3f0d1d2486 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateMediaFromDto(dto, dto.VersionId, sql); + var content = CreateMediaFromDto(dto, sql); return content; } @@ -161,25 +161,24 @@ namespace Umbraco.Core.Persistence.Repositories { // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); - var content = new IMedia[dtos.Count]; - var defs = new List(); + var content = new List(); + var defs = new DocumentDefinitionCollection(); //track the looked up content types, even though the content types are cached // they still need to be deep cloned out of the cache and we don't want to add // the overhead of deep cloning them on every item in this loop var contentTypes = new Dictionary(); - for (var i = 0; i < dtos.Count; i++) + foreach (var dto in dtos) { - var dto = dtos[i]; - // if the cache contains the item, use it if (withCache) { var cached = RuntimeCache.GetCacheItem(GetCacheIdKey(dto.NodeId)); + //TODO: Shouldn't this also match on version!? if (cached != null) { - content[i] = cached; + content.Add(cached); continue; } } @@ -197,37 +196,26 @@ namespace Umbraco.Core.Persistence.Repositories contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); contentTypes[dto.ContentDto.ContentTypeId] = contentType; } - - content[i] = MediaFactory.BuildEntity(dto, contentType); - // need properties - defs.Add(new DocumentDefinition( - dto.NodeId, - dto.VersionId, - dto.VersionDate, - dto.ContentDto.NodeDto.CreateDate, - contentType - )); + // track the definition and if it's successfully added or updated then processed + if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType))) + { + content.Add(MediaFactory.BuildEntity(dto, contentType)); + } } // load all properties for all documents from database in 1 query var propertyData = GetPropertyCollection(pagingSqlQuery, defs); - // assign - var dtoIndex = 0; - foreach (var def in defs) + // assign property data + foreach (var cc in content) { - // move to corresponding item (which has to exist) - while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++; - - // complete the item - var cc = content[dtoIndex]; cc.Properties = propertyData[cc.Id]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 cc.ResetDirtyProperties(false); - } + } return content; } @@ -243,7 +231,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateMediaFromDto(dto, versionId, sql); + var content = CreateMediaFromDto(dto, sql); return content; } @@ -528,16 +516,15 @@ namespace Umbraco.Core.Persistence.Repositories /// Private method to create a media object from a ContentDto /// /// - /// /// /// - private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql) + private IMedia CreateMediaFromDto(ContentVersionDto dto, Sql docSql) { var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); var media = MediaFactory.BuildEntity(dto, contentType); - var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType); + var docDef = new DocumentDefinition(dto, contentType); var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef }); diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index ae8cb08255..23b42108bc 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateMemberFromDto(dto, dto.ContentVersionDto.VersionId, sql); + var content = CreateMemberFromDto(dto, sql); return content; @@ -448,16 +448,16 @@ namespace Umbraco.Core.Persistence.Repositories var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId); - var media = factory.BuildEntity(dto); + var member = factory.BuildEntity(dto); - var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) }); + var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.ContentVersionDto, memberType) }); - media.Properties = properties[dto.NodeId]; + member.Properties = properties[dto.NodeId]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 - ((Entity)media).ResetDirtyProperties(false); - return media; + ((Entity)member).ResetDirtyProperties(false); + return member; } @@ -681,20 +681,19 @@ namespace Umbraco.Core.Persistence.Repositories // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); - var content = new IMember[dtos.Count]; - var defs = new List(); + var content = new List(); + var defs = new DocumentDefinitionCollection(); - for (var i = 0; i < dtos.Count; i++) + foreach (var dto in dtos) { - var dto = dtos[i]; - // if the cache contains the item, use it if (withCache) { var cached = RuntimeCache.GetCacheItem(GetCacheIdKey(dto.NodeId)); + //TODO: Shouldn't this also match on version!? if (cached != null) { - content[i] = cached; + content.Add(cached); continue; } } @@ -702,36 +701,25 @@ namespace Umbraco.Core.Persistence.Repositories // else, need to fetch from the database // content type repository is full-cache so OK to get each one independently var contentType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); - var factory = new MemberFactory(contentType, NodeObjectTypeId, dto.NodeId); - content[i] = factory.BuildEntity(dto); // need properties - defs.Add(new DocumentDefinition( - dto.NodeId, - dto.ContentVersionDto.VersionId, - dto.ContentVersionDto.VersionDate, - dto.ContentVersionDto.ContentDto.NodeDto.CreateDate, - contentType - )); + if (defs.AddOrUpdate(new DocumentDefinition(dto.ContentVersionDto, contentType))) + { + content.Add(MemberFactory.BuildEntity(dto, contentType)); + } } // load all properties for all documents from database in 1 query var propertyData = GetPropertyCollection(pagingSqlQuery, defs); - // assign - var dtoIndex = 0; - foreach (var def in defs) + // assign property data + foreach (var cc in content) { - // move to corresponding item (which has to exist) - while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++; - - // complete the item - var cc = content[dtoIndex]; cc.Properties = propertyData[cc.Id]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 - ((Entity)cc).ResetDirtyProperties(false); + cc.ResetDirtyProperties(false); } return content; @@ -741,17 +729,16 @@ namespace Umbraco.Core.Persistence.Repositories /// Private method to create a member object from a MemberDto /// /// - /// /// /// - private IMember CreateMemberFromDto(MemberDto dto, Guid versionId, Sql docSql) + private IMember CreateMemberFromDto(MemberDto dto, Sql docSql) { var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.ContentVersionDto.NodeId); var member = factory.BuildEntity(dto); - var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType); + var docDef = new DocumentDefinition(dto.ContentVersionDto, memberType); var properties = GetPropertyCollection(docSql, new[] { docDef }); diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index df96112a30..c2c8d3e096 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -773,24 +773,102 @@ ORDER BY contentNodeId, propertytypeid /// protected abstract Sql GetBaseQuery(BaseQueryType queryType); + internal class DocumentDefinitionCollection : KeyedCollection + { + protected override int GetKeyForItem(DocumentDefinition item) + { + return item.Id; + } + + /// + /// if this key already exists if it does then we need to check + /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one + /// + /// + /// + public bool AddOrUpdate(DocumentDefinition item) + { + if (Dictionary == null) + { + base.Add(item); + return true; + } + + var key = GetKeyForItem(item); + DocumentDefinition found; + if (TryGetValue(key, out found)) + { + //it already exists and it's older so we need to replace it + if (item.VersionDate > found.VersionDate) + { + var currIndex = Items.IndexOf(found); + if (currIndex == -1) + throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Version); + + //replace the current one with the newer one + SetItem(currIndex, item); + return true; + } + //could not add or update + return false; + } + + base.Add(item); + return true; + } + + public bool TryGetValue(int key, out DocumentDefinition val) + { + if (Dictionary == null) + { + val = null; + return false; + } + return Dictionary.TryGetValue(key, out val); + } + } + internal class DocumentDefinition { /// /// Initializes a new instance of the class. /// - public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition) + public DocumentDefinition(DocumentDto dto, IContentTypeComposition composition) { - Id = id; - Version = version; - VersionDate = versionDate; - CreateDate = createDate; + DocumentDto = dto; + ContentVersionDto = dto.ContentVersionDto; Composition = composition; } - public int Id { get; set; } - public Guid Version { get; set; } - public DateTime VersionDate { get; set; } - public DateTime CreateDate { get; set; } + public DocumentDefinition(ContentVersionDto dto, IContentTypeComposition composition) + { + ContentVersionDto = dto; + Composition = composition; + } + + public DocumentDto DocumentDto { get; private set; } + public ContentVersionDto ContentVersionDto { get; private set; } + + public int Id + { + get { return ContentVersionDto.NodeId; } + } + + public Guid Version + { + get { return DocumentDto != null ? DocumentDto.VersionId : ContentVersionDto.VersionId; } + } + + public DateTime VersionDate + { + get { return ContentVersionDto.VersionDate; } + } + + public DateTime CreateDate + { + get { return ContentVersionDto.ContentDto.NodeDto.CreateDate; } + } + public IContentTypeComposition Composition { get; set; } } diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs index a561af08a2..c6f9ed21ba 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs @@ -67,6 +67,58 @@ namespace Umbraco.Tests.Persistence.Repositories return repository; } + //TODO: We need to write this same test and fix the EntityRepository! + [Test] + public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags() + { + // Arrange + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + IContent content1; + + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage"); + content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType); + + contentTypeRepository.AddOrUpdate(hasPropertiesContentType); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + } + + //Now manually corrupt the data + for (var index = 0; index < new[] {Guid.NewGuid(), Guid.NewGuid()}.Length; index++) + { + var version = new[] {Guid.NewGuid(), Guid.NewGuid()}[index]; + var versionDate = DateTime.Now.AddMinutes(index); + this.DatabaseContext.Database.Insert(new ContentVersionDto + { + NodeId = content1.Id, + VersionDate = versionDate, + VersionId = version + }); + this.DatabaseContext.Database.Insert(new DocumentDto + { + Newest = true, + NodeId = content1.Id, + Published = true, + Text = content1.Name, + VersionId = version, + WriterUserId = 0, + UpdateDate = versionDate, + TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?) content1.Template.Id + }); + } + + // Assert + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)); + Assert.AreEqual(1, content.Count()); + } + } + /// /// This tests the regression issue of U4-9438 /// From ad9ae867c09a7d2f5cecb0e1fa207e9fbab480fa Mon Sep 17 00:00:00 2001 From: Anders Bjerner Date: Thu, 16 Feb 2017 19:22:26 +0100 Subject: [PATCH 03/41] Added support for ignoring HTTP errors directly from the Angular $http options --- .../src/common/security/securityinterceptor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js b/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js index e47f0663d8..415cf1e812 100644 --- a/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js +++ b/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js @@ -22,7 +22,7 @@ angular.module('umbraco.security.interceptor') //Here we'll check if we should ignore the error, this will be based on an original header set var headers = originalResponse.config ? originalResponse.config.headers : {}; - if (headers["x-umb-ignore-error"] === "ignore") { + if (headers["x-umb-ignore-error"] === "ignore" || originalResponse.config.umbIgnoreErrors === true) { //exit/ignore return promise; } @@ -99,4 +99,4 @@ angular.module('umbraco.security.interceptor') // We have to add the interceptor to the queue as a string because the interceptor depends upon service instances that are not available in the config block. .config(['$httpProvider', function ($httpProvider) { $httpProvider.responseInterceptors.push('securityInterceptor'); - }]); \ No newline at end of file + }]); From 67312d03d0fa5fca45b8cdbc51fcefa1fd80a66c Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 20 Feb 2017 17:24:10 +1100 Subject: [PATCH 04/41] Fixes media query in EntityRepository, it doesn't require outer joins to get the published version, this also enhances the content query to do an inner join on cmsDocument to get the newest instead of an outer join. --- .../Repositories/EntityRepository.cs | 127 +++++++++--------- 1 file changed, 61 insertions(+), 66 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs index ac7e410a8f..772d9a5a18 100644 --- a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs @@ -315,25 +315,30 @@ namespace Umbraco.Core.Persistence.Repositories protected virtual Sql GetBase(bool isContent, bool isMedia, Action customFilter) { var columns = new List - { - "umbracoNode.id", - "umbracoNode.trashed", - "umbracoNode.parentID", - "umbracoNode.nodeUser", - "umbracoNode.level", - "umbracoNode.path", - "umbracoNode.sortOrder", - "umbracoNode.uniqueID", - "umbracoNode.text", - "umbracoNode.nodeObjectType", - "umbracoNode.createDate", - "COUNT(parent.parentID) as children" - }; + { + "umbracoNode.id", + "umbracoNode.trashed", + "umbracoNode.parentID", + "umbracoNode.nodeUser", + "umbracoNode.level", + "umbracoNode.path", + "umbracoNode.sortOrder", + "umbracoNode.uniqueID", + "umbracoNode.text", + "umbracoNode.nodeObjectType", + "umbracoNode.createDate", + "COUNT(parent.parentID) as children" + }; if (isContent || isMedia) { - columns.Add("published.versionId as publishedVersion"); - columns.Add("latest.versionId as newestVersion"); + if (isContent) + { + //only content has this info + columns.Add("published.versionId as publishedVersion"); + columns.Add("document.versionId as newestVersion"); + } + columns.Add("contenttype.alias"); columns.Add("contenttype.icon"); columns.Add("contenttype.thumbnail"); @@ -345,17 +350,21 @@ namespace Umbraco.Core.Persistence.Repositories var entitySql = new Sql() .Select(columns.ToArray()) .From("umbracoNode umbracoNode"); - + if (isContent || isMedia) { - entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id") - .LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType") - .LeftJoin( - "(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1 GROUP BY nodeId, versionId) as published") - .On("umbracoNode.id = published.nodeId") - .LeftJoin( - "(SELECT nodeId, versionId FROM cmsDocument WHERE newest = 1 GROUP BY nodeId, versionId) as latest") - .On("umbracoNode.id = latest.nodeId"); + entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id"); + + if (isContent) + { + //only content has this info + entitySql + .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id") + .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published") + .On("umbracoNode.id = published.nodeId"); + } + + entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType"); } entitySql.LeftJoin("umbracoNode parent").On("parent.parentID = umbracoNode.id"); @@ -372,22 +381,33 @@ namespace Umbraco.Core.Persistence.Repositories { var sql = baseQuery(isContent, isMedia, filter) .Where("umbracoNode.nodeObjectType = @NodeObjectType", new { NodeObjectType = nodeObjectType }); + + if (isContent) { sql.Where("document.newest = 1"); } + return sql; } protected virtual Sql GetBaseWhere(Func, Sql> baseQuery, bool isContent, bool isMedia, int id) { var sql = baseQuery(isContent, isMedia, null) - .Where("umbracoNode.id = @Id", new { Id = id }) - .Append(GetGroupBy(isContent, isMedia)); + .Where("umbracoNode.id = @Id", new { Id = id }); + + if (isContent) { sql.Where("document.newest = 1"); } + + sql.Append(GetGroupBy(isContent, isMedia)); + return sql; } protected virtual Sql GetBaseWhere(Func, Sql> baseQuery, bool isContent, bool isMedia, Guid key) { var sql = baseQuery(isContent, isMedia, null) - .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key }) - .Append(GetGroupBy(isContent, isMedia)); + .Where("umbracoNode.uniqueID = @UniqueID", new {UniqueID = key}); + + if (isContent) { sql.Where("document.newest = 1"); } + + sql.Append(GetGroupBy(isContent, isMedia)); + return sql; } @@ -396,6 +416,9 @@ namespace Umbraco.Core.Persistence.Repositories var sql = baseQuery(isContent, isMedia, null) .Where("umbracoNode.id = @Id AND umbracoNode.nodeObjectType = @NodeObjectType", new {Id = id, NodeObjectType = nodeObjectType}); + + if (isContent) { sql.Where("document.newest = 1"); } + return sql; } @@ -404,47 +427,19 @@ namespace Umbraco.Core.Persistence.Repositories var sql = baseQuery(isContent, isMedia, null) .Where("umbracoNode.uniqueID = @UniqueID AND umbracoNode.nodeObjectType = @NodeObjectType", new { UniqueID = key, NodeObjectType = nodeObjectType }); - return sql; - } - protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true) - { - var columns = new List - { - "umbracoNode.id", - "umbracoNode.trashed", - "umbracoNode.parentID", - "umbracoNode.nodeUser", - "umbracoNode.level", - "umbracoNode.path", - "umbracoNode.sortOrder", - "umbracoNode.uniqueID", - "umbracoNode.text", - "umbracoNode.nodeObjectType", - "umbracoNode.createDate" - }; - - if (isContent || isMedia) - { - columns.Add("published.versionId"); - columns.Add("latest.versionId"); - columns.Add("contenttype.alias"); - columns.Add("contenttype.icon"); - columns.Add("contenttype.thumbnail"); - columns.Add("contenttype.isContainer"); - } - - var sql = new Sql() - .GroupBy(columns.ToArray()); - - if (includeSort) - { - sql = sql.OrderBy("umbracoNode.sortOrder"); - } + if (isContent) { sql.Where("document.newest = 1"); } return sql; } - + + protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true) { var columns = new List { "umbracoNode.id", "umbracoNode.trashed", "umbracoNode.parentID", "umbracoNode.nodeUser", "umbracoNode.level", "umbracoNode.path", "umbracoNode.sortOrder", "umbracoNode.uniqueID", "umbracoNode.text", "umbracoNode.nodeObjectType", "umbracoNode.createDate" }; + if (isContent || isMedia) { if (isContent) { columns.Add("published.versionId"); columns.Add("document.versionId"); } columns.Add("contenttype.alias"); columns.Add("contenttype.icon"); columns.Add("contenttype.thumbnail"); columns.Add("contenttype.isContainer"); } + + var sql = new Sql() .GroupBy(columns.ToArray()); + if (includeSort) { sql = sql.OrderBy("umbracoNode.sortOrder"); } + return sql; } + #endregion /// From 27164093eaf223cf1a66a810e0fb5d2fe19ff135 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 20 Feb 2017 18:08:55 +1100 Subject: [PATCH 05/41] FIxes: U4-9547, Fixes: U4-9546 even though GetChildFolders isn't used anymore i made it work slightly faster --- .../Repositories/ContentTypeRepository.cs | 2 +- .../src/common/resources/media.resource.js | 2 +- .../services/mediatypehelper.service.js | 13 +++++++++ .../listview/listview.controller.js | 29 ++++++++++--------- src/Umbraco.Web/Editors/MediaController.cs | 14 ++++++--- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs index b7b4ddd583..e56f06e0e0 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs @@ -118,7 +118,7 @@ namespace Umbraco.Core.Persistence.Repositories if (objectTypes.Any()) { - sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", objectTypes); + sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", new {objectTypes = objectTypes}); } return Database.Fetch(sql); diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js index 4b56e55801..c0aee7280d 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js @@ -438,7 +438,7 @@ function mediaResource($q, $http, umbDataFormatter, umbRequestHelper) { * * @param {int} parentId Id of the media item to query for child folders * @returns {Promise} resourcePromise object. - * + * @deprecated This method is no longer used and shouldn't be because it performs poorly when there are a lot of media items */ getChildFolders: function (parentId) { if (!parentId) { diff --git a/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js b/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js index 20e5e3799b..b06a5ca8e3 100644 --- a/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js +++ b/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js @@ -7,6 +7,19 @@ function mediaTypeHelper(mediaTypeResource, $q) { var mediaTypeHelperService = { + isFolderType: function(mediaEntity) { + if (!mediaEntity) { + throw "mediaEntity is null"; + } + if (!mediaEntity.contentTypeAlias) { + throw "mediaEntity.contentTypeAlias is null"; + } + + //if you create a media type, which has an alias that ends with ...Folder then its a folder: ex: "secureFolder", "bannerFolder", "Folder" + //this is the exact same logic that is performed in MediaController.GetChildFolders + return mediaEntity.contentTypeAlias.endsWith("Folder"); + }, + getAllowedImagetypes: function (mediaId){ // Get All allowedTypes diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js index 0a1a14fc44..40ab5ce35f 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js @@ -1,4 +1,4 @@ -function listViewController($rootScope, $scope, $routeParams, $injector, $cookieStore, notificationsService, iconHelper, dialogService, editorState, localizationService, $location, appState, $timeout, $q, mediaResource, listViewHelper, userService, navigationService, treeService) { +function listViewController($rootScope, $scope, $routeParams, $injector, $cookieStore, mediaTypeHelper, notificationsService, iconHelper, dialogService, editorState, localizationService, $location, appState, $timeout, $q, mediaResource, listViewHelper, userService, navigationService, treeService) { //this is a quick check to see if we're in create mode, if so just exit - we cannot show children for content // that isn't created yet, if we continue this will use the parent id in the route params which isn't what @@ -261,24 +261,25 @@ function listViewController($rootScope, $scope, $routeParams, $injector, $cookie $scope.actionInProgress = false; $scope.listViewResultSet = data; - //update all values for display + //reset + $scope.folders = []; + + //update all values for display if ($scope.listViewResultSet.items) { _.each($scope.listViewResultSet.items, function (e, index) { - setPropertyValues(e); + setPropertyValues(e); + + //special case, we need to check if any of these types are folder types + //and add them to the folders collection + if ($scope.entityType === 'media') { + if (mediaTypeHelper.isFolderType(e)) { + $scope.folders.push(e); + } + } }); } - if ($scope.entityType === 'media') { - - mediaResource.getChildFolders($scope.contentId) - .then(function (folders) { - $scope.folders = folders; - $scope.viewLoaded = true; - }); - - } else { - $scope.viewLoaded = true; - } + $scope.viewLoaded = true; //NOTE: This might occur if we are requesting a higher page number than what is actually available, for example // if you have more than one page and you delete all items on the last page. In this case, we need to reset to the last diff --git a/src/Umbraco.Web/Editors/MediaController.cs b/src/Umbraco.Web/Editors/MediaController.cs index 77ab9ff6e5..d00a26333e 100644 --- a/src/Umbraco.Web/Editors/MediaController.cs +++ b/src/Umbraco.Web/Editors/MediaController.cs @@ -160,19 +160,25 @@ namespace Umbraco.Web.Editors } /// - /// Returns media items known to be a container of other media items + /// Returns media items known to be of a "Folder" type /// /// /// + [Obsolete("This is no longer used and shouldn't be because it performs poorly when there are a lot of media items")] [FilterAllowedOutgoingMedia(typeof(IEnumerable>))] public IEnumerable> GetChildFolders(int id = -1) { //Suggested convention for folder mediatypes - we can make this more or less complicated as long as we document it... //if you create a media type, which has an alias that ends with ...Folder then its a folder: ex: "secureFolder", "bannerFolder", "Folder" - var folderTypes = Services.ContentTypeService.GetAllMediaTypes().ToArray().Where(x => x.Alias.EndsWith("Folder")).Select(x => x.Id); + var folderTypes = Services.ContentTypeService + .GetAllContentTypeAliases(Constants.ObjectTypes.MediaTypeGuid) + .Where(x => x.EndsWith("Folder")); + + var children = (id < 0) + ? Services.MediaService.GetRootMedia() + : Services.MediaService.GetChildren(id); - var children = (id < 0) ? Services.MediaService.GetRootMedia() : Services.MediaService.GetById(id).Children(); - return children.Where(x => folderTypes.Contains(x.ContentTypeId)).Select(Mapper.Map>); + return children.Where(x => folderTypes.Contains(x.ContentType.Alias)).Select(Mapper.Map>); } /// From 9142314d3918ececdd96f841b6525ba6b0a2fbb7 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 21 Feb 2017 14:18:49 +1100 Subject: [PATCH 06/41] U4-9522 Media recycle bin does not show folders --- .../propertyeditors/listview/layouts/grid/grid.html | 2 +- .../layouts/grid/grid.listviewlayout.controller.js | 3 ++- src/Umbraco.Web/Editors/EntityController.cs | 11 +++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html index 16c5efe799..d09b04f97a 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html @@ -51,7 +51,7 @@ diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js index be1c5856ae..b8ba4f880b 100644 --- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js +++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js @@ -37,7 +37,8 @@ function activate() { vm.itemsWithoutFolders = filterOutFolders($scope.items); - if($scope.entityType === 'media') { + //no need to make another REST/DB call if this data is not used when we are browsing the bin + if ($scope.entityType === 'media' && !vm.isRecycleBin) { mediaTypeHelper.getAllowedImagetypes(vm.nodeId).then(function (types) { vm.acceptedMediatypes = types; }); diff --git a/src/Umbraco.Web/Editors/EntityController.cs b/src/Umbraco.Web/Editors/EntityController.cs index fa3e21a1eb..dfe0c87d10 100644 --- a/src/Umbraco.Web/Editors/EntityController.cs +++ b/src/Umbraco.Web/Editors/EntityController.cs @@ -222,6 +222,17 @@ namespace Umbraco.Web.Editors return GetResultForChildren(id, type); } + /// + /// Get paged descendant entities by id + /// + /// + /// + /// + /// + /// + /// + /// + /// public IEnumerable GetAncestors(int id, UmbracoEntityTypes type) { return GetResultForAncestors(id, type); From 52dce6659802167a79fabbfc3b68b085e0e4ad26 Mon Sep 17 00:00:00 2001 From: Sebastiaan Janssen Date: Tue, 21 Feb 2017 08:49:34 +0100 Subject: [PATCH 07/41] Bump version --- build/UmbracoVersion.txt | 2 +- src/SolutionInfo.cs | 4 ++-- src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +- src/Umbraco.Web.UI/Umbraco.Web.UI.csproj | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/UmbracoVersion.txt b/build/UmbracoVersion.txt index 2f6fa0dce1..cd5058ee6d 100644 --- a/build/UmbracoVersion.txt +++ b/build/UmbracoVersion.txt @@ -1,2 +1,2 @@ # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta) -7.5.9 \ No newline at end of file +7.5.10 \ No newline at end of file diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs index a52db48d23..f4d4ffd6cf 100644 --- a/src/SolutionInfo.cs +++ b/src/SolutionInfo.cs @@ -11,5 +11,5 @@ using System.Resources; [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyFileVersion("7.5.9")] -[assembly: AssemblyInformationalVersion("7.5.9")] \ No newline at end of file +[assembly: AssemblyFileVersion("7.5.10")] +[assembly: AssemblyInformationalVersion("7.5.10")] \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs index d952231be9..69fcd07f46 100644 --- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs +++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs @@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration { public class UmbracoVersion { - private static readonly Version Version = new Version("7.5.9"); + private static readonly Version Version = new Version("7.5.10"); /// /// Gets the current version of Umbraco. diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj index 019feff513..ad668013c2 100644 --- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj +++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj @@ -2423,9 +2423,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\" True True - 7590 + 7510 / - http://localhost:7590 + http://localhost:7510 False False From 6815731da1367da19aa0a4d0bc107c1be66a70bb Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 22 Feb 2017 13:31:04 +1100 Subject: [PATCH 08/41] adds failing test that i'll need to fix --- .../Repositories/VersionableRepositoryBase.cs | 1 + .../Repositories/ContentRepositoryTest.cs | 86 ++++++++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index c2c8d3e096..64930b16c9 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -799,6 +799,7 @@ ORDER BY contentNodeId, propertytypeid if (TryGetValue(key, out found)) { //it already exists and it's older so we need to replace it + //TODO: Also check integer ID? if (item.VersionDate > found.VersionDate) { var currIndex = Items.IndexOf(found); diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs index c6f9ed21ba..e098884d2a 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs @@ -26,6 +26,89 @@ using Umbraco.Core.Persistence.DatabaseModelDefinitions; namespace Umbraco.Tests.Persistence.Repositories { + [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] + [TestFixture] + public class EntityRepositoryTest : BaseDatabaseFactoryTest + { + [SetUp] + public override void Initialize() + { + base.Initialize(); + } + + [TearDown] + public override void TearDown() + { + base.TearDown(); + } + + private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository) + { + TemplateRepository tr; + return CreateContentRepository(unitOfWork, out contentTypeRepository, out tr); + } + + private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository, out TemplateRepository templateRepository) + { + templateRepository = new TemplateRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, Mock.Of(), Mock.Of(), Mock.Of()); + var tagRepository = new TagRepository(unitOfWork, CacheHelper, Logger, SqlSyntax); + contentTypeRepository = new ContentTypeRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, templateRepository); + var repository = new ContentRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, contentTypeRepository, templateRepository, tagRepository, Mock.Of()); + return repository; + } + + [Test] + public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags() + { + // Arrange + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + IContent content1; + + using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository)) + { + var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage"); + content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType); + + contentTypeRepository.AddOrUpdate(hasPropertiesContentType); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + } + + //Now manually corrupt the data + for (var index = 0; index < new[] { Guid.NewGuid(), Guid.NewGuid() }.Length; index++) + { + var version = new[] { Guid.NewGuid(), Guid.NewGuid() }[index]; + var versionDate = DateTime.Now.AddMinutes(index); + this.DatabaseContext.Database.Insert(new ContentVersionDto + { + NodeId = content1.Id, + VersionDate = versionDate, + VersionId = version + }); + this.DatabaseContext.Database.Insert(new DocumentDto + { + Newest = true, + NodeId = content1.Id, + Published = true, + Text = content1.Name, + VersionId = version, + WriterUserId = 0, + UpdateDate = versionDate, + TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id + }); + } + + // Assert + using (var repository = new EntityRepository(unitOfWork)) + { + var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id), Constants.ObjectTypes.DocumentGuid); + Assert.AreEqual(1, content.Count()); + } + } + } + [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] [TestFixture] public class ContentRepositoryTest : BaseDatabaseFactoryTest @@ -66,8 +149,7 @@ namespace Umbraco.Tests.Persistence.Repositories var repository = new ContentRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, contentTypeRepository, templateRepository, tagRepository, Mock.Of()); return repository; } - - //TODO: We need to write this same test and fix the EntityRepository! + [Test] public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags() { From 90ba9a1a3a546bb356ffecbc0b9130a0ad27653a Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 22 Feb 2017 17:22:22 +1100 Subject: [PATCH 09/41] Adds new overload to IMediaService.GetPagedChildren to filter on media type ids which turns out to be much easier than filtering on media type aliases, however i did all of the work to make that happen including unit tests and then it turned out to not be required but we now have the code if necessary, i've left comments about it. I've backported some updates from 7.6 for the SqlIn stuff for the ExpressionVisitor which we still use in the media type id filtering. I updated the JS to query for the folders which all works now. --- .../Persistence/Mappers/MappingResolver.cs | 4 +- .../Querying/ExpressionVisitorBase.cs | 57 ++++++++++-------- .../Querying/ModelToSqlExpressionVisitor.cs | 40 +++++++++++-- ...tensions.cs => SqlExpressionExtensions.cs} | 9 ++- .../Repositories/ContentRepository.cs | 3 + .../Repositories/MediaRepository.cs | 3 + src/Umbraco.Core/Services/IMediaService.cs | 16 +++++ src/Umbraco.Core/Services/MediaService.cs | 25 ++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 2 +- .../ModelToSqlExpressionHelperBenchmarks.cs | 16 +++-- .../Umbraco.Tests.Benchmarks.csproj | 6 ++ src/Umbraco.Tests.Benchmarks/packages.config | 2 + .../Persistence/Querying/ExpressionTests.cs | 39 +++++++++---- .../Repositories/MediaRepositoryTest.cs | 58 ++++++++++++++++++- .../Services/MediaServiceTests.cs | 28 +++++++++ .../src/common/resources/media.resource.js | 13 +++-- .../listview/listview.controller.js | 34 ++++++----- src/Umbraco.Web/Editors/MediaController.cs | 37 +++++++++--- 18 files changed, 313 insertions(+), 79 deletions(-) rename src/Umbraco.Core/Persistence/Querying/{SqlStringExtensions.cs => SqlExpressionExtensions.cs} (83%) diff --git a/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs b/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs index 6909c77744..a4dee60ee9 100644 --- a/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs +++ b/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs @@ -31,7 +31,7 @@ namespace Umbraco.Core.Persistence.Mappers /// /// /// - internal BaseMapper ResolveMapperByType(Type type) + public virtual BaseMapper ResolveMapperByType(Type type) { return _mapperCache.GetOrAdd(type, type1 => { @@ -67,7 +67,7 @@ namespace Umbraco.Core.Persistence.Mappers return Attempt.Succeed(mapper); } - internal string GetMapping(Type type, string propertyName) + public virtual string GetMapping(Type type, string propertyName) { var mapper = ResolveMapperByType(type); var result = mapper.Map(propertyName); diff --git a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs index 678ceb1d8e..0dc421045d 100644 --- a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs +++ b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs @@ -581,6 +581,18 @@ namespace Umbraco.Core.Persistence.Querying case "InvariantContains": case "InvariantEquals": + //special case, if it is 'Contains' and the argumet that Contains is being called on is + //Enumerable and the methodArgs is the actual member access, then it's an SQL IN claus + if (m.Object == null + && m.Arguments[0].Type != typeof(string) + && m.Arguments.Count == 2 + && methodArgs.Length == 1 + && methodArgs[0].NodeType == ExpressionType.MemberAccess + && TypeHelper.IsTypeAssignableFrom(m.Arguments[0].Type)) + { + goto case "SqlIn"; + } + string compareValue; if (methodArgs[0].NodeType != ExpressionType.Constant) @@ -597,13 +609,6 @@ namespace Umbraco.Core.Persistence.Querying compareValue = methodArgs[0].ToString(); } - //special case, if it is 'Contains' and the member that Contains is being called on is not a string, then - // we should be doing an 'In' clause - but we currently do not support this - if (methodArgs[0].Type != typeof(string) && TypeHelper.IsTypeAssignableFrom(methodArgs[0].Type)) - { - throw new NotSupportedException("An array Contains method is not supported"); - } - //default column type var colType = TextColumnType.NVarchar; @@ -705,29 +710,33 @@ namespace Umbraco.Core.Persistence.Querying // } // return string.Format("{0}{1}", r, s); - //case "In": + case "SqlIn": - // var member = Expression.Convert(m.Arguments[0], typeof(object)); - // var lambda = Expression.Lambda>(member); - // var getter = lambda.Compile(); + if (m.Object == null && methodArgs.Length == 1 && methodArgs[0].NodeType == ExpressionType.MemberAccess) + { + var memberAccess = VisitMemberAccess((MemberExpression) methodArgs[0]); + + var member = Expression.Convert(m.Arguments[0], typeof(object)); + var lambda = Expression.Lambda>(member); + var getter = lambda.Compile(); - // var inArgs = (object[])getter(); + var inArgs = (IEnumerable)getter(); - // var sIn = new StringBuilder(); - // foreach (var e in inArgs) - // { - // SqlParameters.Add(e); + var sIn = new StringBuilder(); + foreach (var e in inArgs) + { + SqlParameters.Add(e); - // sIn.AppendFormat("{0}{1}", - // sIn.Length > 0 ? "," : "", - // string.Format("@{0}", SqlParameters.Count - 1)); + sIn.AppendFormat("{0}{1}", + sIn.Length > 0 ? "," : "", + string.Format("@{0}", SqlParameters.Count - 1)); + } - // //sIn.AppendFormat("{0}{1}", - // // sIn.Length > 0 ? "," : "", - // // GetQuotedValue(e, e.GetType())); - // } + return string.Format("{0} IN ({1})", memberAccess, sIn); + } + + throw new NotSupportedException("SqlIn must contain the member being accessed"); - // return string.Format("{0} {1} ({2})", r, m.Method.Name, sIn.ToString()); //case "Desc": // return string.Format("{0} DESC", r); //case "Alias": diff --git a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs index 7f5e479af6..c4029516a3 100644 --- a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs +++ b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs @@ -1,5 +1,7 @@ using System; +using System.Linq; using System.Linq.Expressions; +using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Persistence.Mappers; using Umbraco.Core.Persistence.SqlSyntax; @@ -12,17 +14,19 @@ namespace Umbraco.Core.Persistence.Querying /// This object is stateful and cannot be re-used to parse an expression. internal class ModelToSqlExpressionVisitor : ExpressionVisitorBase { + private readonly MappingResolver _mappingResolver; private readonly BaseMapper _mapper; - public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, BaseMapper mapper) + public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, MappingResolver mappingResolver) : base(sqlSyntax) { - _mapper = mapper; + _mapper = mappingResolver.ResolveMapperByType(typeof(T)); + _mappingResolver = mappingResolver; } [Obsolete("Use the overload the specifies a SqlSyntaxProvider")] public ModelToSqlExpressionVisitor() - : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T))) + : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current) { } protected override string VisitMemberAccess(MemberExpression m) @@ -36,7 +40,7 @@ namespace Umbraco.Core.Persistence.Querying { var field = _mapper.Map(m.Member.Name, true); if (field.IsNullOrWhiteSpace()) - throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name); + throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type)); return field; } //already compiled, return @@ -50,13 +54,39 @@ namespace Umbraco.Core.Persistence.Querying { var field = _mapper.Map(m.Member.Name, true); if (field.IsNullOrWhiteSpace()) - throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name); + throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type)); return field; } //already compiled, return return string.Empty; } + if (m.Expression != null && m.Expression.Type != typeof(T) && TypeHelper.IsTypeAssignableFrom(m.Expression.Type)) + { + //if this is the case, it means we have a sub expression / nested property access, such as: x.ContentType.Alias == "Test"; + //and since the sub type (x.ContentType) is not the same as x, we need to resolve a mapper for x.ContentType to get it's mapped SQL column + + //don't execute if compiled + if (Visited == false) + { + var subMapper = _mappingResolver.ResolveMapperByType(m.Expression.Type); + if (subMapper == null) + throw new NullReferenceException("No mapper found for type " + m.Expression.Type); + var field = subMapper.Map(m.Member.Name, true); + if (field.IsNullOrWhiteSpace()) + throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type)); + return field; + } + //already compiled, return + return string.Empty; + } + + //TODO: When m.Expression.NodeType == ExpressionType.Constant and it's an expression like: content => aliases.Contains(content.ContentType.Alias); + // then an SQL parameter will be added for aliases as an array, however in SqlIn on the subclass it will manually add these SqlParameters anyways, + // however the query will still execute because the SQL that is written will only contain the correct indexes of SQL parameters, this would be ignored, + // I'm just unsure right now due to time constraints how to make it correct. It won't matter right now and has been working already with this bug but I've + // only just discovered what it is actually doing. + var member = Expression.Convert(m, typeof(object)); var lambda = Expression.Lambda>(member); var getter = lambda.Compile(); diff --git a/src/Umbraco.Core/Persistence/Querying/SqlStringExtensions.cs b/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs similarity index 83% rename from src/Umbraco.Core/Persistence/Querying/SqlStringExtensions.cs rename to src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs index cbecc0a591..5a0452c3aa 100644 --- a/src/Umbraco.Core/Persistence/Querying/SqlStringExtensions.cs +++ b/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; namespace Umbraco.Core.Persistence.Querying @@ -6,8 +8,13 @@ namespace Umbraco.Core.Persistence.Querying /// /// String extension methods used specifically to translate into SQL /// - internal static class SqlStringExtensions + internal static class SqlExpressionExtensions { + public static bool SqlIn(this IEnumerable collection, T item) + { + return collection.Contains(item); + } + public static bool SqlWildcard(this string str, string txt, TextColumnType columnType) { var wildcardmatch = new Regex("^" + Regex.Escape(txt). diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 8488bdb65a..83c6095775 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -134,6 +134,9 @@ namespace Umbraco.Core.Persistence.Repositories .On(SqlSyntax, left => left.NodeId, right => right.NodeId) .InnerJoin(SqlSyntax) .On(SqlSyntax, left => left.NodeId, right => right.NodeId); + //TODO: IF we want to enable querying on content type information this will need to be joined + //.InnerJoin(SqlSyntax) + //.On(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax); if (queryType == BaseQueryType.FullSingle) { diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index 53b04e1322..3279571518 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -94,6 +94,9 @@ namespace Umbraco.Core.Persistence.Repositories .On(SqlSyntax, left => left.NodeId, right => right.NodeId) .InnerJoin(SqlSyntax) .On(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax) + //TODO: IF we want to enable querying on content type information this will need to be joined + //.InnerJoin(SqlSyntax) + //.On(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax); .Where(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax); return sql; } diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs index d4218764b7..ac0ff94d3c 100644 --- a/src/Umbraco.Core/Services/IMediaService.cs +++ b/src/Umbraco.Core/Services/IMediaService.cs @@ -166,6 +166,22 @@ namespace Umbraco.Core.Services IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords, string orderBy, Direction orderDirection, bool orderBySystemField, string filter); + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Children from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Flag to indicate when ordering by system field + /// Search text filter + /// A list of content type Ids to filter the list by + /// An Enumerable list of objects + IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords, + string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter); + [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index a67ee8285d..d236dccc5a 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -20,6 +20,7 @@ using Umbraco.Core.Persistence.Repositories; using Umbraco.Core.Persistence.SqlSyntax; using Umbraco.Core.Persistence.UnitOfWork; using Umbraco.Core.Publishing; +using ContentType = System.Net.Mime.ContentType; namespace Umbraco.Core.Services { @@ -435,6 +436,25 @@ namespace Umbraco.Core.Services /// An Enumerable list of objects public IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy, Direction orderDirection, bool orderBySystemField, string filter) + { + return GetPagedChildren(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter, null); + } + + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Children from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Flag to indicate when ordering by system field + /// Search text filter + /// A list of content type Ids to filter the list by + /// An Enumerable list of objects + public IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren, + string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter) { Mandate.ParameterCondition(pageIndex >= 0, "pageIndex"); Mandate.ParameterCondition(pageSize > 0, "pageSize"); @@ -443,6 +463,11 @@ namespace Umbraco.Core.Services var query = Query.Builder; query.Where(x => x.ParentId == id); + if (contentTypeFilter != null && contentTypeFilter.Length > 0) + { + query.Where(x => contentTypeFilter.Contains(x.ContentTypeId)); + } + var medias = repository.GetPagedResultsByQuery(query, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, orderBySystemField, filter); return medias; diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 2e8fc0fd39..4aca6bdf02 100644 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -464,6 +464,7 @@ + @@ -649,7 +650,6 @@ - diff --git a/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs b/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs index 03aa3206ee..5fa395d2f1 100644 --- a/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs +++ b/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs @@ -16,6 +16,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; +using Moq; using Umbraco.Core; using Umbraco.Core.Logging; using Umbraco.Core.Models; @@ -43,15 +44,18 @@ namespace Umbraco.Tests.Benchmarks public ModelToSqlExpressionHelperBenchmarks() { - _contentMapper = new ContentMapper(_syntaxProvider); - _contentMapper.BuildMap(); + var contentMapper = new ContentMapper(_syntaxProvider); + contentMapper.BuildMap(); _cachedExpression = new CachedExpression(); + var mappingResolver = new Mock(); + mappingResolver.Setup(resolver => resolver.ResolveMapperByType(It.IsAny())).Returns(contentMapper); + _mappingResolver = mappingResolver.Object; } private readonly ISqlSyntaxProvider _syntaxProvider = new SqlCeSyntaxProvider(); - private readonly BaseMapper _contentMapper; private readonly CachedExpression _cachedExpression; - + private readonly MappingResolver _mappingResolver; + [Benchmark(Baseline = true)] public void WithNonCached() { @@ -62,7 +66,7 @@ namespace Umbraco.Tests.Benchmarks Expression> predicate = content => content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b); - var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _contentMapper); + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _mappingResolver); var result = modelToSqlExpressionHelper.Visit(predicate); } @@ -80,7 +84,7 @@ namespace Umbraco.Tests.Benchmarks Expression> predicate = content => content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b); - var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _contentMapper); + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _mappingResolver); //wrap it! _cachedExpression.Wrap(predicate); diff --git a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj index 50328b2ecf..46f9dfde10 100644 --- a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj +++ b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj @@ -49,6 +49,9 @@ ..\packages\BenchmarkDotNet.Toolchains.Roslyn.0.9.9\lib\net45\BenchmarkDotNet.Toolchains.Roslyn.dll + + ..\packages\Castle.Core.4.0.0\lib\net45\Castle.Core.dll + ..\packages\Microsoft.CodeAnalysis.Common.1.3.2\lib\net45\Microsoft.CodeAnalysis.dll @@ -58,6 +61,9 @@ ..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.41\lib\net40\Microsoft.Diagnostics.Tracing.TraceEvent.dll + + ..\packages\Moq.4.7.0\lib\net45\Moq.dll + ..\packages\System.Collections.Immutable.1.1.37\lib\dotnet\System.Collections.Immutable.dll diff --git a/src/Umbraco.Tests.Benchmarks/packages.config b/src/Umbraco.Tests.Benchmarks/packages.config index c4d2ba1df2..ec400d2ae8 100644 --- a/src/Umbraco.Tests.Benchmarks/packages.config +++ b/src/Umbraco.Tests.Benchmarks/packages.config @@ -4,11 +4,13 @@ + + diff --git a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs index 65a660146e..b9869005f7 100644 --- a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs +++ b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.Linq; using System.Linq.Expressions; using Moq; using NUnit.Framework; @@ -16,19 +17,35 @@ namespace Umbraco.Tests.Persistence.Querying [TestFixture] public class ExpressionTests : BaseUsingSqlCeSyntax { - // [Test] - // public void Can_Query_With_Content_Type_Alias() - // { - // //Arrange - // Expression> predicate = content => content.ContentType.Alias == "Test"; - // var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(); - // var result = modelToSqlExpressionHelper.Visit(predicate); + [Test] + public void Can_Query_With_Content_Type_Alias() + { + //Arrange + Expression> predicate = content => content.ContentType.Alias == "Test"; + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(); + var result = modelToSqlExpressionHelper.Visit(predicate); - // Debug.Print("Model to Sql ExpressionHelper: \n" + result); + Debug.Print("Model to Sql ExpressionHelper: \n" + result); - // Assert.AreEqual("[cmsContentType].[alias] = @0", result); - // Assert.AreEqual("Test", modelToSqlExpressionHelper.GetSqlParameters()[0]); - // } + Assert.AreEqual("([cmsContentType].[alias] = @0)", result); + Assert.AreEqual("Test", modelToSqlExpressionHelper.GetSqlParameters()[0]); + } + + [Test] + public void Can_Query_With_Content_Type_Aliases() + { + //Arrange + var aliases = new[] {"Test1", "Test2"}; + Expression> predicate = content => aliases.Contains(content.ContentType.Alias); + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(); + var result = modelToSqlExpressionHelper.Visit(predicate); + + Debug.Print("Model to Sql ExpressionHelper: \n" + result); + + Assert.AreEqual("[cmsContentType].[alias] IN (@1,@2)", result); + Assert.AreEqual("Test1", modelToSqlExpressionHelper.GetSqlParameters()[1]); + Assert.AreEqual("Test2", modelToSqlExpressionHelper.GetSqlParameters()[2]); + } [Test] public void CachedExpression_Can_Verify_Path_StartsWith_Predicate_In_Same_Result() diff --git a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs index 1566441d5a..70e72eca9b 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Xml.Linq; using Moq; using NUnit.Framework; +using Umbraco.Core; using Umbraco.Core.Configuration.UmbracoSettings; using Umbraco.Core.Logging; using Umbraco.Core.Models; @@ -345,6 +346,61 @@ namespace Umbraco.Tests.Persistence.Repositories } } + [Test] + public void Can_Perform_GetByQuery_On_MediaRepository_With_ContentType_Id_Filter() + { + // Arrange + var folderMediaType = ServiceContext.ContentTypeService.GetMediaType(1031); + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + MediaTypeRepository mediaTypeRepository; + using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository)) + { + // Act + for (int i = 0; i < 10; i++) + { + var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1); + repository.AddOrUpdate(folder); + } + unitOfWork.Commit(); + + var types = new[] { 1031 }; + var query = Query.Builder.Where(x => types.Contains(x.ContentTypeId)); + var result = repository.GetByQuery(query); + + // Assert + Assert.That(result.Count(), Is.GreaterThanOrEqualTo(11)); + } + } + + [Ignore("We could allow this to work but it requires an extra join on the query used which currently we don't absolutely need so leaving this out for now")] + [Test] + public void Can_Perform_GetByQuery_On_MediaRepository_With_ContentType_Alias_Filter() + { + // Arrange + var folderMediaType = ServiceContext.ContentTypeService.GetMediaType(1031); + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + MediaTypeRepository mediaTypeRepository; + using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository)) + { + // Act + for (int i = 0; i < 10; i++) + { + var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1); + repository.AddOrUpdate(folder); + } + unitOfWork.Commit(); + + var types = new[] { "Folder" }; + var query = Query.Builder.Where(x => types.Contains(x.ContentType.Alias)); + var result = repository.GetByQuery(query); + + // Assert + Assert.That(result.Count(), Is.GreaterThanOrEqualTo(11)); + } + } + [Test] public void Can_Perform_GetPagedResultsByQuery_ForFirstPage_On_MediaRepository() { @@ -470,7 +526,7 @@ namespace Umbraco.Tests.Persistence.Repositories Assert.That(result.First().Name, Is.EqualTo("Test File")); } } - + [Test] public void Can_Perform_GetPagedResultsByQuery_WithFilterMatchingAll_On_MediaRepository() { diff --git a/src/Umbraco.Tests/Services/MediaServiceTests.cs b/src/Umbraco.Tests/Services/MediaServiceTests.cs index 8193df911c..82738a26f4 100644 --- a/src/Umbraco.Tests/Services/MediaServiceTests.cs +++ b/src/Umbraco.Tests/Services/MediaServiceTests.cs @@ -7,6 +7,7 @@ using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Repositories; using Umbraco.Core.Persistence.UnitOfWork; using Umbraco.Tests.TestHelpers; @@ -30,6 +31,33 @@ namespace Umbraco.Tests.Services base.TearDown(); } + [Test] + public void Get_Paged_Children_With_Media_Type_Filter() + { + var mediaService = ServiceContext.MediaService; + var mediaType1 = MockedContentTypes.CreateImageMediaType("Image2"); + ServiceContext.ContentTypeService.Save(mediaType1); + var mediaType2 = MockedContentTypes.CreateImageMediaType("Image3"); + ServiceContext.ContentTypeService.Save(mediaType2); + + for (int i = 0; i < 10; i++) + { + var m1 = MockedMedia.CreateMediaImage(mediaType1, -1); + mediaService.Save(m1); + var m2 = MockedMedia.CreateMediaImage(mediaType2, -1); + mediaService.Save(m2); + } + + long total; + var result = ServiceContext.MediaService.GetPagedChildren(-1, 0, 11, out total, "SortOrder", Direction.Ascending, true, null, new[] {mediaType1.Id, mediaType2.Id}); + Assert.AreEqual(11, result.Count()); + Assert.AreEqual(20, total); + + result = ServiceContext.MediaService.GetPagedChildren(-1, 1, 11, out total, "SortOrder", Direction.Ascending, true, null, new[] { mediaType1.Id, mediaType2.Id }); + Assert.AreEqual(9, result.Count()); + Assert.AreEqual(20, total); + } + [Test] public void Can_Move_Media() { diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js index c0aee7280d..c41b7e91c7 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js @@ -427,7 +427,9 @@ function mediaResource($q, $http, umbDataFormatter, umbRequestHelper) { * Retrieves all media children with types used as folders. * Uses the convention of looking for media items with mediaTypes ending in * *Folder so will match "Folder", "bannerFolder", "secureFolder" etc, - * + * + * NOTE: This will return a max of 500 folders, if more is required it needs to be paged + * * ##usage * 
           * mediaResource.getChildFolders(1234)
@@ -438,21 +440,22 @@ function mediaResource($q, $http, umbDataFormatter, umbRequestHelper) {
           *
           * @param {int} parentId Id of the media item to query for child folders    
           * @returns {Promise} resourcePromise object.
-          * @deprecated This method is no longer used and shouldn't be because it performs poorly when there are a lot of media items
+          *
           */
         getChildFolders: function (parentId) {
             if (!parentId) {
                 parentId = -1;
             }
 
+            //NOTE: This will return a max of 500 folders, if more is required it needs to be paged
             return umbRequestHelper.resourcePromise(
                   $http.get(
                         umbRequestHelper.getApiUrl(
                               "mediaApiBaseUrl",
                               "GetChildFolders",
-                              [
-                                    { id: parentId }
-                              ])),
+                            {
+                                id: parentId
+                            })),
                   'Failed to retrieve child folders for media item ' + parentId);
         },
 
diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js
index 40ab5ce35f..f8887c6c44 100644
--- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js
+++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js
@@ -1,4 +1,4 @@
-function listViewController($rootScope, $scope, $routeParams, $injector, $cookieStore, mediaTypeHelper, notificationsService, iconHelper, dialogService, editorState, localizationService, $location, appState, $timeout, $q, mediaResource, listViewHelper, userService, navigationService, treeService) {
+function listViewController($rootScope, $scope, $routeParams, $injector, $cookieStore, notificationsService, iconHelper, dialogService, editorState, localizationService, $location, appState, $timeout, $q, mediaResource, listViewHelper, userService, navigationService, treeService) {
 
    //this is a quick check to see if we're in create mode, if so just exit - we cannot show children for content
    // that isn't created yet, if we continue this will use the parent id in the route params which isn't what
@@ -53,7 +53,9 @@ function listViewController($rootScope, $scope, $routeParams, $injector, $cookie
    $scope.isNew = false;
    $scope.actionInProgress = false;
    $scope.selection = [];
-   $scope.folders = [];
+   $scope.folders = [];   
+   //tracks if we've already loaded the folders for the current node
+   var foldersLoaded = false;
    $scope.listViewResultSet = {
       totalPages: 0,
       items: []
@@ -261,25 +263,25 @@ function listViewController($rootScope, $scope, $routeParams, $injector, $cookie
          $scope.actionInProgress = false;
          $scope.listViewResultSet = data;
 
-          //reset
-         $scope.folders = [];
-
-          //update all values for display
+         //update all values for display
          if ($scope.listViewResultSet.items) {
             _.each($scope.listViewResultSet.items, function (e, index) {
-                setPropertyValues(e);
-
-                //special case, we need to check if any of these types are folder types 
-                //and add them to the folders collection
-                if ($scope.entityType === 'media') {
-                    if (mediaTypeHelper.isFolderType(e)) {
-                        $scope.folders.push(e);
-                    }
-                }
+               setPropertyValues(e);
             });
          }
 
-         $scope.viewLoaded = true;
+         if (!foldersLoaded && $scope.entityType === 'media') {
+            //The folders aren't loaded - we only need to do this once since we're never changing node ids
+            mediaResource.getChildFolders($scope.contentId)
+                    .then(function (folders) {
+                       $scope.folders = folders;
+                       $scope.viewLoaded = true;
+                       foldersLoaded = true;
+                    });
+
+         } else {
+            $scope.viewLoaded = true;
+         }
 
          //NOTE: This might occur if we are requesting a higher page number than what is actually available, for example
          // if you have more than one page and you delete all items on the last page. In this case, we need to reset to the last
diff --git a/src/Umbraco.Web/Editors/MediaController.cs b/src/Umbraco.Web/Editors/MediaController.cs
index d00a26333e..2f3fe5bd94 100644
--- a/src/Umbraco.Web/Editors/MediaController.cs
+++ b/src/Umbraco.Web/Editors/MediaController.cs
@@ -167,18 +167,41 @@ namespace Umbraco.Web.Editors
         [Obsolete("This is no longer used and shouldn't be because it performs poorly when there are a lot of media items")]
         [FilterAllowedOutgoingMedia(typeof(IEnumerable>))]
         public IEnumerable> GetChildFolders(int id = -1)
+        {
+            //we are only allowing a max of 500 to be returned here, if more is required it needs to be paged
+            var result = GetChildFolders(id, 1, 500);
+            return result.Items;
+        }
+
+        /// 
+        /// Returns a paged result of media items known to be of a "Folder" type
+        /// 
+        /// 
+        /// 
+        /// 
+        /// 
+        public PagedResult> GetChildFolders(int id, int pageNumber, int pageSize)
         {
             //Suggested convention for folder mediatypes - we can make this more or less complicated as long as we document it...
             //if you create a media type, which has an alias that ends with ...Folder then its a folder: ex: "secureFolder", "bannerFolder", "Folder"
             var folderTypes = Services.ContentTypeService
-                .GetAllContentTypeAliases(Constants.ObjectTypes.MediaTypeGuid)
-                .Where(x => x.EndsWith("Folder"));
-            
-            var children = (id < 0) 
-                ? Services.MediaService.GetRootMedia()
-                : Services.MediaService.GetChildren(id);
+                .GetAllMediaTypes()
+                .Where(x => x.Alias.EndsWith("Folder"))
+                .Select(x => x.Id)
+                .ToArray();
 
-            return children.Where(x => folderTypes.Contains(x.ContentType.Alias)).Select(Mapper.Map>);
+            if (folderTypes.Length == 0)
+            {
+                return new PagedResult>(0, pageNumber, pageSize);
+            }
+
+            long total;
+            var children = Services.MediaService.GetPagedChildren(id, pageNumber - 1, pageSize, out total, "Name", Direction.Ascending, true, null, folderTypes.ToArray());
+            
+            return new PagedResult>(total, pageNumber, pageSize)
+            {
+                Items = children.Select(Mapper.Map>)
+            };
         }
 
         /// 

From f8d6c7e7d09c407cd800b106bdb342724cb39a27 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Wed, 22 Feb 2017 18:34:55 +1100
Subject: [PATCH 10/41] Fixes expression tree changes

---
 .../Querying/ModelToSqlExpressionVisitor.cs   | 24 ++++++++++++++++++-
 .../Persistence/Querying/ExpressionTests.cs   | 17 +++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs
index c4029516a3..bd6a2020ab 100644
--- a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs
+++ b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs
@@ -61,7 +61,10 @@ namespace Umbraco.Core.Persistence.Querying
                 return string.Empty;
             }
 
-            if (m.Expression != null && m.Expression.Type != typeof(T) && TypeHelper.IsTypeAssignableFrom(m.Expression.Type))
+            if (m.Expression != null 
+                && m.Expression.Type != typeof(T) 
+                && TypeHelper.IsTypeAssignableFrom(m.Expression.Type)
+                && EndsWithConstant(m) == false)
             {
                 //if this is the case, it means we have a sub expression / nested property access, such as: x.ContentType.Alias == "Test";
                 //and since the sub type (x.ContentType) is not the same as x, we need to resolve a mapper for x.ContentType to get it's mapped SQL column
@@ -101,5 +104,24 @@ namespace Umbraco.Core.Persistence.Querying
             return string.Empty;
 
         }
+
+        /// 
+        /// Determines if the MemberExpression ends in a Constant value
+        /// 
+        /// 
+        /// 
+        private bool EndsWithConstant(MemberExpression m)
+        {
+            Expression expr = m;
+            
+            while (expr is MemberExpression)
+            {
+                var memberExpr = expr as MemberExpression;
+                expr = memberExpr.Expression;
+            }
+            
+            var constExpr = expr as ConstantExpression;
+            return constExpr != null;
+        }
     }
 }
\ No newline at end of file
diff --git a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs
index b9869005f7..b51ad01c4d 100644
--- a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs
+++ b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs
@@ -17,6 +17,23 @@ namespace Umbraco.Tests.Persistence.Querying
     [TestFixture]
     public class ExpressionTests : BaseUsingSqlCeSyntax
     {
+        [Test]
+        public void Equals_Claus_With_Two_Entity_Values()
+        {
+            var dataType = new DataTypeDefinition(-1, "Test")
+            {
+                Id = 12345
+            };
+            Expression> predicate = p => p.DataTypeDefinitionId == dataType.Id;
+            var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor();
+            var result = modelToSqlExpressionHelper.Visit(predicate);
+
+            Debug.Print("Model to Sql ExpressionHelper: \n" + result);
+
+            Assert.AreEqual("([cmsPropertyType].[dataTypeId] = @0)", result);
+            Assert.AreEqual(12345, modelToSqlExpressionHelper.GetSqlParameters()[0]);
+        }
+
         [Test]
         public void Can_Query_With_Content_Type_Alias()
         {

From 0b9a3ef68a1e4dda93e156d6e55862a88bbe407f Mon Sep 17 00:00:00 2001
From: Sebastiaan Janssen 
Date: Wed, 22 Feb 2017 10:26:37 +0100
Subject: [PATCH 11/41] Bump version

---
 build/UmbracoVersion.txt                         | 2 +-
 src/SolutionInfo.cs                              | 4 ++--
 src/Umbraco.Core/Configuration/UmbracoVersion.cs | 2 +-
 src/Umbraco.Web.UI/Umbraco.Web.UI.csproj         | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/build/UmbracoVersion.txt b/build/UmbracoVersion.txt
index cd5058ee6d..2ff5e18029 100644
--- a/build/UmbracoVersion.txt
+++ b/build/UmbracoVersion.txt
@@ -1,2 +1,2 @@
 # Usage: on line 2 put the release version, on line 3 put the version comment (example: beta)
-7.5.10
\ No newline at end of file
+7.5.11
\ No newline at end of file
diff --git a/src/SolutionInfo.cs b/src/SolutionInfo.cs
index f4d4ffd6cf..6b17f79fd3 100644
--- a/src/SolutionInfo.cs
+++ b/src/SolutionInfo.cs
@@ -11,5 +11,5 @@ using System.Resources;
 
 [assembly: AssemblyVersion("1.0.*")]
 
-[assembly: AssemblyFileVersion("7.5.10")]
-[assembly: AssemblyInformationalVersion("7.5.10")]
\ No newline at end of file
+[assembly: AssemblyFileVersion("7.5.11")]
+[assembly: AssemblyInformationalVersion("7.5.11")]
\ No newline at end of file
diff --git a/src/Umbraco.Core/Configuration/UmbracoVersion.cs b/src/Umbraco.Core/Configuration/UmbracoVersion.cs
index 69fcd07f46..2fb1d9ebda 100644
--- a/src/Umbraco.Core/Configuration/UmbracoVersion.cs
+++ b/src/Umbraco.Core/Configuration/UmbracoVersion.cs
@@ -6,7 +6,7 @@ namespace Umbraco.Core.Configuration
 {
     public class UmbracoVersion
     {
-        private static readonly Version Version = new Version("7.5.10");
+        private static readonly Version Version = new Version("7.5.11");
 
         /// 
         /// Gets the current version of Umbraco.
diff --git a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj
index ad668013c2..15ca616975 100644
--- a/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj
+++ b/src/Umbraco.Web.UI/Umbraco.Web.UI.csproj
@@ -2423,9 +2423,9 @@ xcopy "$(ProjectDir)"..\packages\SqlServerCE.4.0.0.1\x86\*.* "$(TargetDir)x86\"
         
           True
           True
-          7510
+          7511
           /
-          http://localhost:7510
+          http://localhost:7511
           False
           False
           

From bf22399abae711669ddff45e60e99d6d8845e9be Mon Sep 17 00:00:00 2001
From: Emil Wangaa 
Date: Wed, 22 Feb 2017 10:34:46 +0100
Subject: [PATCH 12/41] Fixup linendings for EntityRepository

---
 .../Repositories/EntityRepository.cs          | 69 ++++++++++++++++---
 1 file changed, 59 insertions(+), 10 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
index 772d9a5a18..87f4a7811c 100644
--- a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
@@ -382,7 +382,10 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, filter)
                 .Where("umbracoNode.nodeObjectType = @NodeObjectType", new { NodeObjectType = nodeObjectType });
 
-            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
 
             return sql;
         }
@@ -392,7 +395,10 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.id = @Id", new { Id = id });
 
-            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
 
             sql.Append(GetGroupBy(isContent, isMedia));
 
@@ -404,7 +410,10 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.uniqueID = @UniqueID", new {UniqueID = key});
 
-            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
 
             sql.Append(GetGroupBy(isContent, isMedia));
 
@@ -417,7 +426,10 @@ namespace Umbraco.Core.Persistence.Repositories
                 .Where("umbracoNode.id = @Id AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new {Id = id, NodeObjectType = nodeObjectType});
 
-            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
 
             return sql;
         }
@@ -428,17 +440,54 @@ namespace Umbraco.Core.Persistence.Repositories
                 .Where("umbracoNode.uniqueID = @UniqueID AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new { UniqueID = key, NodeObjectType = nodeObjectType });
 
-            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+            if (isContent)
+            {
+                sql.Where("document.newest = 1");
+            }
 
             return sql;
         }
 
-        protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
        {
            var columns = new List
            {
                "umbracoNode.id",
                "umbracoNode.trashed",
                "umbracoNode.parentID",
                "umbracoNode.nodeUser",
                "umbracoNode.level",
                "umbracoNode.path",
                "umbracoNode.sortOrder",
                "umbracoNode.uniqueID",
                "umbracoNode.text",
                "umbracoNode.nodeObjectType",
                "umbracoNode.createDate"
            };

-            if (isContent || isMedia)
            {
                if (isContent)
                {
                    columns.Add("published.versionId");
                    columns.Add("document.versionId");
                }
                columns.Add("contenttype.alias");
                columns.Add("contenttype.icon");
                columns.Add("contenttype.thumbnail");
                columns.Add("contenttype.isContainer");
            }
+        protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
+        {
+            var columns = new List
+            {
+                "umbracoNode.id",
+                "umbracoNode.trashed",
+                "umbracoNode.parentID",
+                "umbracoNode.nodeUser",
+                "umbracoNode.level",
+                "umbracoNode.path",
+                "umbracoNode.sortOrder",
+                "umbracoNode.uniqueID",
+                "umbracoNode.text",
+                "umbracoNode.nodeObjectType",
+                "umbracoNode.createDate"
+            };
 
-            var sql = new Sql()
                .GroupBy(columns.ToArray());

-            if (includeSort)
            {
                sql = sql.OrderBy("umbracoNode.sortOrder");
            }

-            return sql;
        }
+            if (isContent || isMedia)
+            {
+                if (isContent)
+                {
+                    columns.Add("published.versionId");
+                    columns.Add("document.versionId");
+                }
+                columns.Add("contenttype.alias");
+                columns.Add("contenttype.icon");
+                columns.Add("contenttype.thumbnail");
+                columns.Add("contenttype.isContainer");
+            }
+
+            var sql = new Sql()
+                .GroupBy(columns.ToArray());
+
+            if (includeSort)
+            {
+                sql = sql.OrderBy("umbracoNode.sortOrder");
+            }
+
+            return sql;
+        }
 
         #endregion
 

From 7cadbb6398e19dad1826c85eb6082658fc1530d8 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Wed, 22 Feb 2017 21:09:25 +1100
Subject: [PATCH 13/41] Backports some required EntityRepository changes,
 starts adding the custom collection to remove duplicate (throw err currently)

---
 .../Factories/UmbracoEntityFactory.cs         |  64 +----
 .../Repositories/EntityRepository.cs          | 261 +++++++++++-------
 .../Repositories/ContentRepositoryTest.cs     |  84 ------
 .../Repositories/EntityRepositoryTest.cs      | 101 +++++++
 src/Umbraco.Tests/Umbraco.Tests.csproj        |   1 +
 5 files changed, 271 insertions(+), 240 deletions(-)
 create mode 100644 src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs

diff --git a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs
index 18073c088e..e8eb7b6342 100644
--- a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs
+++ b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs
@@ -6,6 +6,7 @@ using System.Reflection;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Repositories;
+using Umbraco.Core.Strings;
 
 namespace Umbraco.Core.Persistence.Factories
 {
@@ -17,7 +18,7 @@ namespace Umbraco.Core.Persistence.Factories
             
             //figure out what extra properties we have that are not on the IUmbracoEntity and add them to additional data
             foreach (var k in originalEntityProperties.Keys
-                .Select(x => new { orig = x, title = x.ConvertCase(StringAliasCaseType.PascalCase) })
+                .Select(x => new { orig = x, title = x.ToCleanString(CleanStringType.PascalCase) })
                 .Where(x => entityProps.InvariantContains(x.title) == false))
             {
                 entity.AdditionalData[k.title] = originalEntityProperties[k.orig];
@@ -75,65 +76,6 @@ namespace Umbraco.Core.Persistence.Factories
                 entity.EnableChangeTracking();
             }
         }
-
-        public UmbracoEntity BuildEntity(EntityRepository.UmbracoEntityDto dto)
-        {
-            var entity = new UmbracoEntity(dto.Trashed)
-                             {
-                                 CreateDate = dto.CreateDate,
-                                 CreatorId = dto.UserId.Value,
-                                 Id = dto.NodeId,
-                                 Key = dto.UniqueId,
-                                 Level = dto.Level,
-                                 Name = dto.Text,
-                                 NodeObjectTypeId = dto.NodeObjectType.Value,
-                                 ParentId = dto.ParentId,
-                                 Path = dto.Path,
-                                 SortOrder = dto.SortOrder,
-                                 HasChildren = dto.Children > 0,
-                                 ContentTypeAlias = dto.Alias ?? string.Empty,
-                                 ContentTypeIcon = dto.Icon ?? string.Empty,
-                                 ContentTypeThumbnail = dto.Thumbnail ?? string.Empty,
-                             };
-
-            entity.IsPublished = dto.PublishedVersion != default(Guid) || (dto.NewestVersion != default(Guid) && dto.PublishedVersion == dto.NewestVersion);
-            entity.IsDraft = dto.NewestVersion != default(Guid) && (dto.PublishedVersion == default(Guid) || dto.PublishedVersion != dto.NewestVersion);
-            entity.HasPendingChanges = (dto.PublishedVersion != default(Guid) && dto.NewestVersion != default(Guid)) && dto.PublishedVersion != dto.NewestVersion;
-
-            if (dto.UmbracoPropertyDtos != null)
-            {
-                foreach (var propertyDto in dto.UmbracoPropertyDtos)
-                {
-                    entity.AdditionalData[propertyDto.PropertyAlias] = new UmbracoEntity.EntityProperty
-                    {
-                        PropertyEditorAlias = propertyDto.PropertyEditorAlias,
-                        Value = propertyDto.NTextValue.IsNullOrWhiteSpace()
-                            ? propertyDto.NVarcharValue
-                            : propertyDto.NTextValue.ConvertToJsonIfPossible()
-                    };
-                }
-            }
-
-            return entity;
-        }
-
-        public EntityRepository.UmbracoEntityDto BuildDto(UmbracoEntity entity)
-        {
-            var node = new EntityRepository.UmbracoEntityDto
-                           {
-                               CreateDate = entity.CreateDate,
-                               Level = short.Parse(entity.Level.ToString(CultureInfo.InvariantCulture)),
-                               NodeId = entity.Id,
-                               NodeObjectType = entity.NodeObjectTypeId,
-                               ParentId = entity.ParentId,
-                               Path = entity.Path,
-                               SortOrder = entity.SortOrder,
-                               Text = entity.Name,
-                               Trashed = entity.Trashed,
-                               UniqueId = entity.Key,
-                               UserId = entity.CreatorId
-                           };
-            return node;
-        }
+        
     }
 }
\ No newline at end of file
diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
index ac7e410a8f..7a450a919b 100644
--- a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Dynamic;
 using System.Globalization;
 using System.Linq;
@@ -241,8 +242,15 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 //use dynamic so that we can get ALL properties from the SQL so we can chuck that data into our AdditionalData
                 var finalSql = entitySql.Append(GetGroupBy(isContent, false));
-                var dtos = _work.Database.Fetch(finalSql);
-                return dtos.Select(factory.BuildEntityFromDynamic).Cast().ToList();
+
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query(finalSql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();
             }
         }
 
@@ -315,25 +323,30 @@ namespace Umbraco.Core.Persistence.Repositories
         protected virtual Sql GetBase(bool isContent, bool isMedia, Action customFilter)
         {
             var columns = new List
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate",
-                                  "COUNT(parent.parentID) as children"
-                              };
+            {
+                "umbracoNode.id",
+                "umbracoNode.trashed",
+                "umbracoNode.parentID",
+                "umbracoNode.nodeUser",
+                "umbracoNode.level",
+                "umbracoNode.path",
+                "umbracoNode.sortOrder",
+                "umbracoNode.uniqueID",
+                "umbracoNode.text",
+                "umbracoNode.nodeObjectType",
+                "umbracoNode.createDate",
+                "COUNT(parent.parentID) as children"
+            };
 
             if (isContent || isMedia)
             {
-                columns.Add("published.versionId as publishedVersion");
-                columns.Add("latest.versionId as newestVersion");
+                if (isContent)
+                {
+                    //only content has this info
+                    columns.Add("published.versionId as publishedVersion");
+                    columns.Add("document.versionId as newestVersion");
+                }
+                
                 columns.Add("contenttype.alias");
                 columns.Add("contenttype.icon");
                 columns.Add("contenttype.thumbnail");
@@ -345,17 +358,21 @@ namespace Umbraco.Core.Persistence.Repositories
             var entitySql = new Sql()
                 .Select(columns.ToArray())
                 .From("umbracoNode umbracoNode");
-                
+
             if (isContent || isMedia)
             {
-                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id")
-                   .LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1 GROUP BY nodeId, versionId) as published")
-                   .On("umbracoNode.id = published.nodeId")
-                   .LeftJoin(
-                       "(SELECT nodeId, versionId FROM cmsDocument WHERE newest = 1 GROUP BY nodeId, versionId) as latest")
-                   .On("umbracoNode.id = latest.nodeId");
+                entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id");
+
+                if (isContent)
+                {
+                    //only content has this info
+                    entitySql
+                        .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id")
+                        .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published")
+                        .On("umbracoNode.id = published.nodeId");
+                }
+
+                entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType");
             }
 
             entitySql.LeftJoin("umbracoNode parent").On("parent.parentID = umbracoNode.id");
@@ -372,22 +389,33 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             var sql = baseQuery(isContent, isMedia, filter)
                 .Where("umbracoNode.nodeObjectType = @NodeObjectType", new { NodeObjectType = nodeObjectType });
+
+            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+
             return sql;
         }
 
         protected virtual Sql GetBaseWhere(Func, Sql> baseQuery, bool isContent, bool isMedia, int id)
         {
             var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.id = @Id", new { Id = id })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.id = @Id", new { Id = id });
+
+            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
             return sql;
         }
 
         protected virtual Sql GetBaseWhere(Func, Sql> baseQuery, bool isContent, bool isMedia, Guid key)
         {
             var sql = baseQuery(isContent, isMedia, null)
-                .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key })
-                .Append(GetGroupBy(isContent, isMedia));
+                .Where("umbracoNode.uniqueID = @UniqueID", new {UniqueID = key});
+
+            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+
+            sql.Append(GetGroupBy(isContent, isMedia));
+
             return sql;
         }
 
@@ -396,6 +424,9 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.id = @Id AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new {Id = id, NodeObjectType = nodeObjectType});
+
+            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
+
             return sql;
         }
 
@@ -404,47 +435,19 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = baseQuery(isContent, isMedia, null)
                 .Where("umbracoNode.uniqueID = @UniqueID AND umbracoNode.nodeObjectType = @NodeObjectType",
                        new { UniqueID = key, NodeObjectType = nodeObjectType });
-            return sql;
-        }
 
-        protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
-        {
-            var columns = new List
-                              {
-                                  "umbracoNode.id",
-                                  "umbracoNode.trashed",
-                                  "umbracoNode.parentID",
-                                  "umbracoNode.nodeUser",
-                                  "umbracoNode.level",
-                                  "umbracoNode.path",
-                                  "umbracoNode.sortOrder",
-                                  "umbracoNode.uniqueID",
-                                  "umbracoNode.text",
-                                  "umbracoNode.nodeObjectType",
-                                  "umbracoNode.createDate"
-                              };
-
-            if (isContent || isMedia)
-            {
-                columns.Add("published.versionId");
-                columns.Add("latest.versionId");
-                columns.Add("contenttype.alias");
-                columns.Add("contenttype.icon");
-                columns.Add("contenttype.thumbnail");
-                columns.Add("contenttype.isContainer");
-            }
-            
-            var sql = new Sql()
-                .GroupBy(columns.ToArray());
-
-            if (includeSort)
-            {
-                sql = sql.OrderBy("umbracoNode.sortOrder");
-            }
+            if (isContent)
            {
                sql.Where("document.newest = 1");
            }
 
             return sql;
         }
-        
+
+        protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
        {
            var columns = new List
            {
                "umbracoNode.id",
                "umbracoNode.trashed",
                "umbracoNode.parentID",
                "umbracoNode.nodeUser",
                "umbracoNode.level",
                "umbracoNode.path",
                "umbracoNode.sortOrder",
                "umbracoNode.uniqueID",
                "umbracoNode.text",
                "umbracoNode.nodeObjectType",
                "umbracoNode.createDate"
            };

+            if (isContent || isMedia)
            {
                if (isContent)
                {
                    columns.Add("published.versionId");
                    columns.Add("document.versionId");
                }
                columns.Add("contenttype.alias");
                columns.Add("contenttype.icon");
                columns.Add("contenttype.thumbnail");
                columns.Add("contenttype.isContainer");
            }
+
+            var sql = new Sql()
                .GroupBy(columns.ToArray());

+            if (includeSort)
            {
                sql = sql.OrderBy("umbracoNode.sortOrder");
            }

+            return sql;
        }
+
         #endregion
 
         /// 
@@ -458,33 +461,7 @@ namespace Umbraco.Core.Persistence.Repositories
             UnitOfWork.DisposeIfDisposable();
         }
 
-        #region umbracoNode POCO - Extends NodeDto
-        [TableName("umbracoNode")]
-        [PrimaryKey("id")]
-        [ExplicitColumns]
-        internal class UmbracoEntityDto : NodeDto
-        {
-            [Column("children")]
-            public int Children { get; set; }
-
-            [Column("publishedVersion")]
-            public Guid PublishedVersion { get; set; }
-
-            [Column("newestVersion")]
-            public Guid NewestVersion { get; set; }
-
-            [Column("alias")]
-            public string Alias { get; set; }
-
-            [Column("icon")]
-            public string Icon { get; set; }
-
-            [Column("thumbnail")]
-            public string Thumbnail { get; set; }
-
-            [ResultColumn]
-            public List UmbracoPropertyDtos { get; set; }
-        }
+        #region private classes
         
         [ExplicitColumns]
         internal class UmbracoPropertyDto
@@ -568,6 +545,100 @@ namespace Umbraco.Core.Persistence.Repositories
                 return prev;
             }
         }
+
+        private class EntityDefinitionCollection : KeyedCollection
+        {
+            protected override int GetKeyForItem(EntityDefinition item)
+            {
+                return item.Id;
+            }
+
+            /// 
+            /// if this key already exists if it does then we need to check
+            /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one
+            /// 
+            /// 
+            /// 
+            public bool AddOrUpdate(EntityDefinition item)
+            {
+                if (Dictionary == null)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
+                var key = GetKeyForItem(item);
+                EntityDefinition found;
+                if (TryGetValue(key, out found))
+                {
+                    //it already exists and it's older so we need to replace it
+                    //TODO: Also check integer ID?
+                    if (item.EntityDate > found.EntityDate)
+                    {
+                        var currIndex = Items.IndexOf(found);
+                        if (currIndex == -1)
+                            throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Id);
+
+                        //replace the current one with the newer one
+                        SetItem(currIndex, item);
+                        return true;
+                    }
+                    //could not add or update
+                    return false;
+                }
+
+                base.Add(item);
+                return true;
+            }
+
+            public bool TryGetValue(int key, out EntityDefinition val)
+            {
+                if (Dictionary == null)
+                {
+                    val = null;
+                    return false;
+                }
+                return Dictionary.TryGetValue(key, out val);
+            }
+        }
+
+        private class EntityDefinition
+        {
+            private readonly UmbracoEntityFactory _factory;
+            private readonly dynamic _entity;
+            private readonly bool _isContent;
+            private readonly bool _isMedia;
+
+            public EntityDefinition(UmbracoEntityFactory factory, dynamic entity, bool isContent, bool isMedia)
+            {
+                _factory = factory;
+                _entity = entity;
+                _isContent = isContent;
+                _isMedia = isMedia;
+            }
+
+            public IUmbracoEntity BuildFromDynamic()
+            {
+                return _factory.BuildEntityFromDynamic(_entity);
+            }
+
+            public int Id
+            {
+                get { return _entity.id; }
+            }
+
+            public DateTime EntityDate
+            {
+                get
+                {
+                    if (_isContent || _isMedia)
+                    {
+                        return _entity.VersionDate;
+                    }
+                    return _entity.createDate;
+                }
+            }
+        }
         #endregion
     }
 }
\ No newline at end of file
diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
index e098884d2a..7de83809b2 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
@@ -21,94 +21,10 @@ using Umbraco.Tests.TestHelpers;
 using Umbraco.Tests.TestHelpers.Entities;
 using umbraco.editorControls.tinyMCE3;
 using umbraco.interfaces;
-using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 
 namespace Umbraco.Tests.Persistence.Repositories
 {
-    [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
-    [TestFixture]
-    public class EntityRepositoryTest : BaseDatabaseFactoryTest
-    {
-        [SetUp]
-        public override void Initialize()
-        {
-            base.Initialize();            
-        }
-
-        [TearDown]
-        public override void TearDown()
-        {
-            base.TearDown();
-        }
-        
-        private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository)
-        {
-            TemplateRepository tr;
-            return CreateContentRepository(unitOfWork, out contentTypeRepository, out tr);
-        }
-
-        private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository, out TemplateRepository templateRepository)
-        {
-            templateRepository = new TemplateRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, Mock.Of(), Mock.Of(), Mock.Of());
-            var tagRepository = new TagRepository(unitOfWork, CacheHelper, Logger, SqlSyntax);
-            contentTypeRepository = new ContentTypeRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, templateRepository);
-            var repository = new ContentRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, contentTypeRepository, templateRepository, tagRepository, Mock.Of());
-            return repository;
-        }
-
-        [Test]
-        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags()
-        {
-            // Arrange
-            var provider = new PetaPocoUnitOfWorkProvider(Logger);
-            var unitOfWork = provider.GetUnitOfWork();
-            ContentTypeRepository contentTypeRepository;
-            IContent content1;            
-
-            using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository))
-            {
-                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
-                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
-
-                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
-                repository.AddOrUpdate(content1);
-                unitOfWork.Commit();
-            }
-
-            //Now manually corrupt the data
-            for (var index = 0; index < new[] { Guid.NewGuid(), Guid.NewGuid() }.Length; index++)
-            {
-                var version = new[] { Guid.NewGuid(), Guid.NewGuid() }[index];
-                var versionDate = DateTime.Now.AddMinutes(index);
-                this.DatabaseContext.Database.Insert(new ContentVersionDto
-                {
-                    NodeId = content1.Id,
-                    VersionDate = versionDate,
-                    VersionId = version
-                });
-                this.DatabaseContext.Database.Insert(new DocumentDto
-                {
-                    Newest = true,
-                    NodeId = content1.Id,
-                    Published = true,
-                    Text = content1.Name,
-                    VersionId = version,
-                    WriterUserId = 0,
-                    UpdateDate = versionDate,
-                    TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id
-                });
-            }
-
-            // Assert
-            using (var repository = new EntityRepository(unitOfWork))
-            {
-                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id), Constants.ObjectTypes.DocumentGuid);
-                Assert.AreEqual(1, content.Count());
-            }
-        }
-    }
-
     [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
     [TestFixture]
     public class ContentRepositoryTest : BaseDatabaseFactoryTest
diff --git a/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs
new file mode 100644
index 0000000000..6de75f2046
--- /dev/null
+++ b/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs
@@ -0,0 +1,101 @@
+using System;
+using System.Linq;
+using Moq;
+using NUnit.Framework;
+using Umbraco.Core;
+using Umbraco.Core.Configuration.UmbracoSettings;
+using Umbraco.Core.IO;
+using Umbraco.Core.Models;
+using Umbraco.Core.Models.EntityBase;
+using Umbraco.Core.Models.Rdbms;
+using Umbraco.Core.Persistence.Querying;
+using Umbraco.Core.Persistence.Repositories;
+using Umbraco.Core.Persistence.UnitOfWork;
+using Umbraco.Tests.TestHelpers;
+using Umbraco.Tests.TestHelpers.Entities;
+
+namespace Umbraco.Tests.Persistence.Repositories
+{
+    [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
+    [TestFixture]
+    public class EntityRepositoryTest : BaseDatabaseFactoryTest
+    {
+        [SetUp]
+        public override void Initialize()
+        {
+            base.Initialize();            
+        }
+
+        [TearDown]
+        public override void TearDown()
+        {
+            base.TearDown();
+        }
+        
+        private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository)
+        {
+            TemplateRepository tr;
+            return CreateContentRepository(unitOfWork, out contentTypeRepository, out tr);
+        }
+
+        private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository, out TemplateRepository templateRepository)
+        {
+            templateRepository = new TemplateRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, Mock.Of(), Mock.Of(), Mock.Of());
+            var tagRepository = new TagRepository(unitOfWork, CacheHelper, Logger, SqlSyntax);
+            contentTypeRepository = new ContentTypeRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, templateRepository);
+            var repository = new ContentRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, contentTypeRepository, templateRepository, tagRepository, Mock.Of());
+            return repository;
+        }
+
+        [Test]
+        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            IContent content1;            
+
+            using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository))
+            {
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+            }
+
+            //Now manually corrupt the data
+            for (var index = 0; index < new[] { Guid.NewGuid(), Guid.NewGuid() }.Length; index++)
+            {
+                var version = new[] { Guid.NewGuid(), Guid.NewGuid() }[index];
+                var versionDate = DateTime.Now.AddMinutes(index);
+                this.DatabaseContext.Database.Insert(new ContentVersionDto
+                {
+                    NodeId = content1.Id,
+                    VersionDate = versionDate,
+                    VersionId = version
+                });
+                this.DatabaseContext.Database.Insert(new DocumentDto
+                {
+                    Newest = true,
+                    NodeId = content1.Id,
+                    Published = true,
+                    Text = content1.Name,
+                    VersionId = version,
+                    WriterUserId = 0,
+                    UpdateDate = versionDate,
+                    TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id
+                });
+            }
+
+            // Assert
+            using (var repository = new EntityRepository(unitOfWork))
+            {
+                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id), Constants.ObjectTypes.DocumentGuid);
+                Assert.AreEqual(1, content.Count());
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj
index 1334451890..de3423bb4a 100644
--- a/src/Umbraco.Tests/Umbraco.Tests.csproj
+++ b/src/Umbraco.Tests/Umbraco.Tests.csproj
@@ -160,6 +160,7 @@
     
     
     
+    
     
     
     

From 76b696e3bf4cbff6ac372aa390ea32b6a64bff5e Mon Sep 17 00:00:00 2001
From: Sebastiaan Janssen 
Date: Wed, 22 Feb 2017 11:12:24 +0100
Subject: [PATCH 14/41] U4-9559 Security: files of type xhtml should not be
 allowed to be uploaded to the media section

---
 src/Umbraco.Web.UI/config/umbracoSettings.Release.config | 2 +-
 src/Umbraco.Web.UI/config/umbracoSettings.config         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Umbraco.Web.UI/config/umbracoSettings.Release.config b/src/Umbraco.Web.UI/config/umbracoSettings.Release.config
index 4d229c7dcc..053411c5bb 100644
--- a/src/Umbraco.Web.UI/config/umbracoSettings.Release.config
+++ b/src/Umbraco.Web.UI/config/umbracoSettings.Release.config
@@ -51,7 +51,7 @@
     throw
 
     
-    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,html,htm,svg,php,htaccess
+    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,xhtml,html,htm,svg,php,htaccess
 
     
     Textstring
diff --git a/src/Umbraco.Web.UI/config/umbracoSettings.config b/src/Umbraco.Web.UI/config/umbracoSettings.config
index 9eb3f135e9..ff3b87607d 100644
--- a/src/Umbraco.Web.UI/config/umbracoSettings.config
+++ b/src/Umbraco.Web.UI/config/umbracoSettings.config
@@ -100,7 +100,7 @@
     throw
     
     
-    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,html,htm,svg,php,htaccess
+    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,xhtml,html,htm,svg,php,htaccess
 
     
     Textstring

From 16f8f6036ca53d5e358a42e9c140e5ca99fddf43 Mon Sep 17 00:00:00 2001
From: Stephan 
Date: Wed, 22 Feb 2017 15:52:02 +0100
Subject: [PATCH 15/41] U4-9560 - fix db issue

---
 src/Umbraco.Tests/Issues/U9560.cs             | 92 +++++++++++++++++++
 .../Persistence/DatabaseContextTests.cs       | 11 ++-
 src/Umbraco.Tests/Umbraco.Tests.csproj        |  1 +
 .../propertytype/propertytype.cs              |  9 +-
 4 files changed, 106 insertions(+), 7 deletions(-)
 create mode 100644 src/Umbraco.Tests/Issues/U9560.cs

diff --git a/src/Umbraco.Tests/Issues/U9560.cs b/src/Umbraco.Tests/Issues/U9560.cs
new file mode 100644
index 0000000000..f9174498ff
--- /dev/null
+++ b/src/Umbraco.Tests/Issues/U9560.cs
@@ -0,0 +1,92 @@
+using System;
+using NUnit.Framework;
+using Umbraco.Core.Models;
+using Umbraco.Tests.TestHelpers;
+
+namespace Umbraco.Tests.Issues
+{
+    [TestFixture]
+    [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)]
+    public class U9560 : BaseDatabaseFactoryTest
+    {
+        [Test]
+        public void Test()
+        {
+            // create a content type and some properties
+            var contentType = new ContentType(-1);
+            contentType.Alias = "test";
+            contentType.Name = "test";
+            var propertyType = new PropertyType("test", DataTypeDatabaseType.Ntext, "prop") { Name = "Prop", Description = "", Mandatory = false, SortOrder = 1, DataTypeDefinitionId = -88 };
+            contentType.PropertyTypeCollection.Add(propertyType);
+            ServiceContext.ContentTypeService.Save(contentType);
+
+            var aliasName = string.Empty;
+
+            // read fields, same as what we do with PetaPoco Fetch
+            var db = DatabaseContext.Database;
+            db.OpenSharedConnection();
+            try
+            {
+                var conn = db.Connection;
+                var cmd = conn.CreateCommand();
+                cmd.CommandText = "SELECT mandatory, dataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias, name, validationRegExp, description from cmsPropertyType where id=" + propertyType.Id;
+                using (var reader = cmd.ExecuteReader())
+                {
+                    while (reader.Read())
+                    {
+                        for (var i = 0; i < reader.FieldCount; i++)
+                            Console.WriteLine(reader.GetName(i));
+                        aliasName = reader.GetName(5);
+                    }
+                }
+            }
+            finally
+            {
+                db.CloseSharedConnection();
+            }
+
+            // note that although the query is for 'alias' the field is named 'Alias'
+            Assert.AreEqual("Alias", aliasName);
+
+            // try differently
+            db.OpenSharedConnection();
+            try
+            {
+                var conn = db.Connection;
+                var cmd = conn.CreateCommand();
+                cmd.CommandText = "SELECT mandatory, dataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias as alias, name, validationRegExp, description from cmsPropertyType where id=" + propertyType.Id;
+                using (var reader = cmd.ExecuteReader())
+                {
+                    while (reader.Read())
+                    {
+                        for (var i = 0; i < reader.FieldCount; i++)
+                            Console.WriteLine(reader.GetName(i));
+                        aliasName = reader.GetName(5);
+                    }
+                }
+            }
+            finally
+            {
+                db.CloseSharedConnection();
+            }
+
+            // and now it is OK
+            Assert.AreEqual("alias", aliasName);
+
+            // get the legacy content type
+            var legacyContentType = new umbraco.cms.businesslogic.ContentType(contentType.Id);
+            Assert.AreEqual("test", legacyContentType.Alias);
+
+            // get the legacy properties
+            var legacyProperties = legacyContentType.PropertyTypes;
+
+            // without the fix, due to some (swallowed) inner exception, we have no properties
+            //Assert.IsNull(legacyProperties);
+
+            // thanks to the fix, it works
+            Assert.IsNotNull(legacyProperties);
+            Assert.AreEqual(1, legacyProperties.Count);
+            Assert.AreEqual("prop", legacyProperties[0].Alias);
+        }
+    }
+}
diff --git a/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs b/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs
index 94f0010201..eea011bc20 100644
--- a/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs
+++ b/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs
@@ -34,7 +34,7 @@ namespace Umbraco.Tests.Persistence
 				{
 					DatabaseContext = _dbContext,
 					IsReady = true
-				};			
+				};
 		}
 
 		[TearDown]
@@ -44,6 +44,13 @@ namespace Umbraco.Tests.Persistence
 			ApplicationContext.Current = null;
 		}
 
+        [Test]
+        public void Database_Connection()
+        {
+            var db = _dbContext.Database;
+            Assert.IsNull(db.Connection);
+        }
+
         [Test]
         public void Can_Verify_Single_Database_Instance()
         {
@@ -99,7 +106,7 @@ namespace Umbraco.Tests.Persistence
 
             var appCtx = new ApplicationContext(
                 new DatabaseContext(Mock.Of(), Mock.Of(), Mock.Of(), "test"),
-                new ServiceContext(migrationEntryService: Mock.Of()), 
+                new ServiceContext(migrationEntryService: Mock.Of()),
                 CacheHelper.CreateDisabledCacheHelper(),
                 new ProfilingLogger(Mock.Of(), Mock.Of()));
 
diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj
index 1334451890..d5f83d4cb2 100644
--- a/src/Umbraco.Tests/Umbraco.Tests.csproj
+++ b/src/Umbraco.Tests/Umbraco.Tests.csproj
@@ -155,6 +155,7 @@
   
   
     
+    
     
     
     
diff --git a/src/umbraco.cms/businesslogic/propertytype/propertytype.cs b/src/umbraco.cms/businesslogic/propertytype/propertytype.cs
index 8d5f538fd9..63a4d80e91 100644
--- a/src/umbraco.cms/businesslogic/propertytype/propertytype.cs
+++ b/src/umbraco.cms/businesslogic/propertytype/propertytype.cs
@@ -56,7 +56,7 @@ namespace umbraco.cms.businesslogic.propertytype
         {
             var found = ApplicationContext.Current.DatabaseContext.Database
                 .SingleOrDefault(
-                    "Select mandatory, DataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias, name, validationRegExp, description from cmsPropertyType where id=@id",
+                    "Select mandatory as mandatory, dataTypeId as dataTypeId, propertyTypeGroupId as propertyTypeGroupId, contentTypeId as contentTypeId, sortOrder as sortOrder, alias as alias, name as name, validationRegExp as validationRegExp, description as description from cmsPropertyType where id=@id",
                     new {id = id});
 
             if (found == null)
@@ -72,14 +72,13 @@ namespace umbraco.cms.businesslogic.propertytype
                 _tabId = _propertyTypeGroup;
             }
 
-            //Fixed issue U4-9493 Case issues
             _sortOrder = found.sortOrder;
-            _alias = found.Alias;
-            _name = found.Name;
+            _alias = found.alias;
+            _name = found.name;
             _validationRegExp = found.validationRegExp;
             _DataTypeId = found.dataTypeId;
             _contenttypeid = found.contentTypeId;
-            _description = found.Description;
+            _description = found.description;
         }
 
         #endregion

From 6e347dd32c00e94f40e7a6db625374a9bdac6512 Mon Sep 17 00:00:00 2001
From: Anders Bjerner 
Date: Wed, 22 Feb 2017 17:28:40 +0100
Subject: [PATCH 16/41] Added some extra validation

---
 .../src/common/security/securityinterceptor.js      | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js b/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js
index 415cf1e812..0c18656c83 100644
--- a/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js
+++ b/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js
@@ -19,15 +19,20 @@ angular.module('umbraco.security.interceptor')
                     return promise;
                 }, function(originalResponse) {
                     // Intercept failed requests
+                    
+                    // Make sure we have the configuration of the request (don't we always?)
+                    var config = originalResponse.config ? originalResponse.config : {};
+                    
+                    // Make sure we have an object for the headers of the request
+                    var headers = config.headers ? config.headers : {};
 
-                    //Here we'll check if we should ignore the error, this will be based on an original header set
-                    var headers = originalResponse.config ? originalResponse.config.headers : {};
-                    if (headers["x-umb-ignore-error"] === "ignore" || originalResponse.config.umbIgnoreErrors === true) {
+                    //Here we'll check if we should ignore the error (either based on the original header set or the request configuration)
+                    if (headers["x-umb-ignore-error"] === "ignore" || config.umbIgnoreErrors === true) {
                         //exit/ignore
                         return promise;
                     }
                     var filtered = _.find(requestInterceptorFilter(), function(val) {
-                        return originalResponse.config.url.indexOf(val) > 0;
+                        return config.url.indexOf(val) > 0;
                     });
                     if (filtered) {
                         return promise;

From 813a8956d3719b5bb2f09e25deb897fe86bd056a Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Fri, 24 Feb 2017 12:27:36 +1100
Subject: [PATCH 17/41] Gets EntityRepository to deal with corrupted
 newest/published flags, adds tests and lots of assertions for each method

---
 .../Factories/UmbracoEntityFactory.cs         |  2 +-
 .../Repositories/EntityRepository.cs          | 92 +++++++++---------
 .../Repositories/VersionableRepositoryBase.cs | 11 ++-
 .../Repositories/ContentRepositoryTest.cs     | 19 ++--
 .../Repositories/EntityRepositoryTest.cs      | 96 +++++++++++++++++--
 5 files changed, 159 insertions(+), 61 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs
index e8eb7b6342..d8543c2ff6 100644
--- a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs
+++ b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs
@@ -18,7 +18,7 @@ namespace Umbraco.Core.Persistence.Factories
             
             //figure out what extra properties we have that are not on the IUmbracoEntity and add them to additional data
             foreach (var k in originalEntityProperties.Keys
-                .Select(x => new { orig = x, title = x.ToCleanString(CleanStringType.PascalCase) })
+                .Select(x => new { orig = x, title = x.ToCleanString(CleanStringType.PascalCase | CleanStringType.Ascii | CleanStringType.ConvertCase) })
                 .Where(x => entityProps.InvariantContains(x.title) == false))
             {
                 entity.AdditionalData[k.title] = originalEntityProperties[k.orig];
diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
index 7a450a919b..9680ea0689 100644
--- a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
@@ -69,11 +69,12 @@ namespace Umbraco.Core.Persistence.Repositories
             bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);
 
             var sql = GetFullSqlForEntityType(key, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                 var entities = _work.Database.Fetch(
                     new UmbracoEntityRelator().Map, sql);
 
@@ -81,14 +82,16 @@ namespace Umbraco.Core.Persistence.Repositories
             }
             else
             {
-                var nodeDto = _work.Database.FirstOrDefault(sql);
-                if (nodeDto == null)
-                    return null;
 
-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;                
             }
             
             
@@ -113,29 +116,29 @@ namespace Umbraco.Core.Persistence.Repositories
             bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media);
 
             var sql = GetFullSqlForEntityType(id, isContent, isMedia, objectTypeId);
-            
+
+            var factory = new UmbracoEntityFactory();
+
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too  
                 var entities = _work.Database.Fetch(
                     new UmbracoEntityRelator().Map, sql);
 
                 return entities.FirstOrDefault();
             }
             else
-            {
-                var nodeDto = _work.Database.FirstOrDefault(sql);
-                if (nodeDto == null)
-                    return null;
-
-                var factory = new UmbracoEntityFactory();
-                var entity = factory.BuildEntityFromDynamic(nodeDto);
-
-                return entity;
+            {                
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
+                {
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
+                }
+                var found = collection.FirstOrDefault();
+                return found != null ? found.BuildFromDynamic() : null;
             }
-
-            
         }
 
         public virtual IEnumerable GetAll(Guid objectTypeId, params int[] ids)
@@ -172,22 +175,21 @@ namespace Umbraco.Core.Persistence.Repositories
 
             if (isMedia)
             {
-                //for now treat media differently
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too                
                 var entities = _work.Database.Fetch(
                     new UmbracoEntityRelator().Map, sql);
-                foreach (var entity in entities)
-                {
-                    yield return entity;
-                }
+                return entities;
             }
             else
             {
-                var dtos = _work.Database.Fetch(sql);
-                foreach (var entity in dtos.Select(dto => factory.BuildEntityFromDynamic(dto)))
+                //query = read forward data reader, do not load everything into mem
+                var dtos = _work.Database.Query(sql);
+                var collection = new EntityDefinitionCollection();
+                foreach (var dto in dtos)
                 {
-                    yield return entity;
+                    collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false));
                 }
+                return collection.Select(x => x.BuildFromDynamic()).ToList();                
             }
         }
 
@@ -232,8 +234,7 @@ namespace Umbraco.Core.Persistence.Repositories
                     }
                 });
 
-                //treat media differently for now 
-                //TODO: We should really use this methodology for Content/Members too!! since it includes properties and ALL of the dynamic db fields
+                //for now treat media differently and include all property data too    
                 var entities = _work.Database.Fetch(
                     new UmbracoEntityRelator().Map, mediaSql);
                 return entities;
@@ -342,9 +343,10 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 if (isContent)
                 {
-                    //only content has this info
+                    //only content has/needs this info
                     columns.Add("published.versionId as publishedVersion");
                     columns.Add("document.versionId as newestVersion");
+                    columns.Add("contentversion.id as versionId");
                 }
                 
                 columns.Add("contenttype.alias");
@@ -365,9 +367,10 @@ namespace Umbraco.Core.Persistence.Repositories
 
                 if (isContent)
                 {
-                    //only content has this info
-                    entitySql
+                    //only content has/needs this info
+                    entitySql                        
                         .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id")
+                        .InnerJoin("cmsContentVersion contentversion").On("contentversion.VersionId = document.versionId")
                         .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published")
                         .On("umbracoNode.id = published.nodeId");
                 }
@@ -442,7 +445,7 @@ namespace Umbraco.Core.Persistence.Repositories
         }
 
         protected virtual Sql GetGroupBy(bool isContent, bool isMedia, bool includeSort = true)
        {
            var columns = new List
            {
                "umbracoNode.id",
                "umbracoNode.trashed",
                "umbracoNode.parentID",
                "umbracoNode.nodeUser",
                "umbracoNode.level",
                "umbracoNode.path",
                "umbracoNode.sortOrder",
                "umbracoNode.uniqueID",
                "umbracoNode.text",
                "umbracoNode.nodeObjectType",
                "umbracoNode.createDate"
            };

-            if (isContent || isMedia)
            {
                if (isContent)
                {
                    columns.Add("published.versionId");
                    columns.Add("document.versionId");
                }
                columns.Add("contenttype.alias");
                columns.Add("contenttype.icon");
                columns.Add("contenttype.thumbnail");
                columns.Add("contenttype.isContainer");
            }
+            if (isContent || isMedia)
            {
                if (isContent)
                {
                    columns.Add("published.versionId");
                    columns.Add("document.versionId");
                    columns.Add("contentversion.id");
                }
                columns.Add("contenttype.alias");
                columns.Add("contenttype.icon");
                columns.Add("contenttype.thumbnail");
                columns.Add("contenttype.isContainer");                
            }
 
             var sql = new Sql()
                .GroupBy(columns.ToArray());

             if (includeSort)
            {
                sql = sql.OrderBy("umbracoNode.sortOrder");
            }

@@ -572,8 +575,7 @@ namespace Umbraco.Core.Persistence.Repositories
                 if (TryGetValue(key, out found))
                 {
                     //it already exists and it's older so we need to replace it
-                    //TODO: Also check integer ID?
-                    if (item.EntityDate > found.EntityDate)
+                    if (item.VersionId > found.VersionId)
                     {
                         var currIndex = Items.IndexOf(found);
                         if (currIndex == -1)
@@ -591,7 +593,7 @@ namespace Umbraco.Core.Persistence.Repositories
                 return true;
             }
 
-            public bool TryGetValue(int key, out EntityDefinition val)
+            private bool TryGetValue(int key, out EntityDefinition val)
             {
                 if (Dictionary == null)
                 {
@@ -627,15 +629,15 @@ namespace Umbraco.Core.Persistence.Repositories
                 get { return _entity.id; }
             }
 
-            public DateTime EntityDate
+            public int VersionId
             {
                 get
                 {
                     if (_isContent || _isMedia)
                     {
-                        return _entity.VersionDate;
+                        return _entity.versionId;
                     }
-                    return _entity.createDate;
+                    return _entity.id;
                 }
             }
         }
diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
index 64930b16c9..0d9d0bf3d2 100644
--- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
@@ -799,8 +799,7 @@ ORDER BY contentNodeId, propertytypeid
                 if (TryGetValue(key, out found))
                 {
                     //it already exists and it's older so we need to replace it
-                    //TODO: Also check integer ID?
-                    if (item.VersionDate > found.VersionDate)
+                    if (item.VersionId > found.VersionId)
                     {
                         var currIndex = Items.IndexOf(found);
                         if (currIndex == -1)
@@ -860,6 +859,14 @@ ORDER BY contentNodeId, propertytypeid
                 get { return DocumentDto != null ? DocumentDto.VersionId : ContentVersionDto.VersionId; }
             }
 
+            /// 
+            /// This is used to determien which version is the most recent
+            /// 
+            public int VersionId
+            {
+                get { return ContentVersionDto.Id; }
+            }
+
             public DateTime VersionDate
             {
                 get { return ContentVersionDto.VersionDate; }
diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
index 7de83809b2..cf2347c9e9 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Xml.Linq;
@@ -85,17 +86,22 @@ namespace Umbraco.Tests.Persistence.Repositories
                 unitOfWork.Commit();                
             }
 
+            var versionDtos = new List();
+
             //Now manually corrupt the data
-            for (var index = 0; index < new[] {Guid.NewGuid(), Guid.NewGuid()}.Length; index++)
+            var versions = new[] { Guid.NewGuid(), Guid.NewGuid() };
+            for (var index = 0; index < versions.Length; index++)
             {
-                var version = new[] {Guid.NewGuid(), Guid.NewGuid()}[index];
+                var version = versions[index];
                 var versionDate = DateTime.Now.AddMinutes(index);
-                this.DatabaseContext.Database.Insert(new ContentVersionDto
+                var versionDto = new ContentVersionDto
                 {
                     NodeId = content1.Id,
                     VersionDate = versionDate,
                     VersionId = version
-                });
+                };
+                this.DatabaseContext.Database.Insert(versionDto);
+                versionDtos.Add(versionDto);
                 this.DatabaseContext.Database.Insert(new DocumentDto
                 {
                     Newest = true,
@@ -112,8 +118,9 @@ namespace Umbraco.Tests.Persistence.Repositories
             // Assert
             using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
             {
-                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id));
-                Assert.AreEqual(1, content.Count());
+                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(content[0].UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture));
             }
         }
 
diff --git a/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs
index 6de75f2046..baf574e091 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Moq;
 using NUnit.Framework;
@@ -48,7 +49,7 @@ namespace Umbraco.Tests.Persistence.Repositories
         }
 
         [Test]
-        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags()
+        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags_Full_Model()
         {
             // Arrange
             var provider = new PetaPocoUnitOfWorkProvider(Logger);
@@ -66,17 +67,22 @@ namespace Umbraco.Tests.Persistence.Repositories
                 unitOfWork.Commit();
             }
 
+            var versionDtos = new List();
+
             //Now manually corrupt the data
-            for (var index = 0; index < new[] { Guid.NewGuid(), Guid.NewGuid() }.Length; index++)
+            var versions = new[] {Guid.NewGuid(), Guid.NewGuid()};
+            for (var index = 0; index < versions.Length; index++)
             {
-                var version = new[] { Guid.NewGuid(), Guid.NewGuid() }[index];
+                var version = versions[index];
                 var versionDate = DateTime.Now.AddMinutes(index);
-                this.DatabaseContext.Database.Insert(new ContentVersionDto
+                var versionDto = new ContentVersionDto
                 {
                     NodeId = content1.Id,
                     VersionDate = versionDate,
                     VersionId = version
-                });
+                };
+                this.DatabaseContext.Database.Insert(versionDto);
+                versionDtos.Add(versionDto);
                 this.DatabaseContext.Database.Insert(new DocumentDto
                 {
                     Newest = true,
@@ -93,8 +99,84 @@ namespace Umbraco.Tests.Persistence.Repositories
             // Assert
             using (var repository = new EntityRepository(unitOfWork))
             {
-                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id), Constants.ObjectTypes.DocumentGuid);
-                Assert.AreEqual(1, content.Count());
+                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id), Constants.ObjectTypes.DocumentGuid).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]);
+
+                content = repository.GetAll(Constants.ObjectTypes.DocumentGuid, content[0].Key).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]);
+
+                content = repository.GetAll(Constants.ObjectTypes.DocumentGuid, content[0].Id).ToArray();
+                Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]);
+
+                var contentItem = repository.Get(content[0].Id, Constants.ObjectTypes.DocumentGuid);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), contentItem.AdditionalData["VersionId"]);
+
+                contentItem = repository.GetByKey(content[0].Key, Constants.ObjectTypes.DocumentGuid);
+                Assert.AreEqual(versionDtos.Max(x => x.Id), contentItem.AdditionalData["VersionId"]);
+            }
+        }
+
+        /// 
+        /// The Slim model will test the EntityRepository when it doesn't know the object type so it will 
+        /// make the simplest (slim) query
+        /// 
+        [Test]
+        public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags_Slim_Model()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            IContent content1;
+
+            using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository))
+            {
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+            }
+
+            //Now manually corrupt the data
+            var versions = new[] { Guid.NewGuid(), Guid.NewGuid() };
+            for (var index = 0; index < versions.Length; index++)
+            {
+                var version = versions[index];
+                var versionDate = DateTime.Now.AddMinutes(index);
+                var versionDto = new ContentVersionDto
+                {
+                    NodeId = content1.Id,
+                    VersionDate = versionDate,
+                    VersionId = version
+                };
+                this.DatabaseContext.Database.Insert(versionDto);
+                this.DatabaseContext.Database.Insert(new DocumentDto
+                {
+                    Newest = true,
+                    NodeId = content1.Id,
+                    Published = true,
+                    Text = content1.Name,
+                    VersionId = version,
+                    WriterUserId = 0,
+                    UpdateDate = versionDate,
+                    TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id
+                });
+            }
+
+            // Assert
+            using (var repository = new EntityRepository(unitOfWork))
+            {
+                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray();
+                Assert.AreEqual(1, content.Length);
+                var contentItem = repository.Get(content[0].Id);
+                Assert.IsNotNull(contentItem);
+                contentItem = repository.GetByKey(content[0].Key);
+                Assert.IsNotNull(contentItem);
             }
         }
     }

From 266f82b6c506e55e01f3fd7b216614459c120e61 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Fri, 24 Feb 2017 13:23:27 +1100
Subject: [PATCH 18/41] Fixes issue with returning all versions, adds more
 tests/assertions, updates content and media repository to double check that
 the versions match before returning from cache.

---
 .../Repositories/ContentRepository.cs         | 48 +++++++++++++------
 .../Repositories/MediaRepository.cs           |  5 +-
 .../Repositories/MemberRepository.cs          |  1 -
 .../Repositories/VersionableRepositoryBase.cs | 19 ++++++--
 .../Repositories/ContentRepositoryTest.cs     |  8 ++++
 5 files changed, 60 insertions(+), 21 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index 33dce9be3b..cd1793f4d5 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -273,12 +273,13 @@ namespace Umbraco.Core.Persistence.Repositories
             var sqlFull = translate(GetBaseQuery(BaseQueryType.FullMultiple));
             var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids));
             
-            return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true);
+            return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true,  includeAllVersions:true);
         }
 
         public override IContent GetByVersion(Guid versionId)
         {
             var sql = GetBaseQuery(BaseQueryType.FullSingle);
+            //TODO: cmsContentVersion.VersionId has a Unique Index constraint applied, seems silly then to also add OrderByDescending since it would be impossible to return more than one.
             sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId });
             sql.OrderByDescending(x => x.VersionDate, SqlSyntax);
 
@@ -297,7 +298,8 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = new Sql()
                 .Select("*")
                 .From(SqlSyntax)
-                .InnerJoin(SqlSyntax).On(SqlSyntax, left => left.VersionId, right => right.VersionId)
+                .InnerJoin(SqlSyntax)
+                .On(SqlSyntax, left => left.VersionId, right => right.VersionId)
                 .Where(x => x.VersionId == versionId, SqlSyntax)
                 .Where(x => x.Newest != true, SqlSyntax);
             var dto = Database.Fetch(sql).FirstOrDefault();
@@ -317,7 +319,8 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = new Sql()
                 .Select("*")
                 .From()
-                .InnerJoin().On(left => left.VersionId, right => right.VersionId)
+                .InnerJoin()
+                .On(left => left.VersionId, right => right.VersionId)
                 .Where(x => x.NodeId == id)
                 .Where(x => x.VersionDate < versionDate)
                 .Where(x => x.Newest != true);
@@ -897,7 +900,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
 
             return base.GetDatabaseFieldNameForOrderBy(orderBy);
         }
-        
+
         /// 
         /// This is the underlying method that processes most queries for this repository
         /// 
@@ -908,8 +911,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
         /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item
         /// 
         /// 
+        /// 
+        /// Generally when querying for content we only want to return the most recent version of the content item, however in some cases like when 
+        /// we want to return all versions of a content item, we can't simply return the latest
+        /// 
         /// 
-        private IEnumerable ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false)
+        private IEnumerable ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false, bool includeAllVersions = false)
         {
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch(sqlFull);
@@ -952,7 +959,7 @@ ORDER BY doc2.updateDate DESC
 
 
             var content = new List();
-            var defs = new DocumentDefinitionCollection();
+            var defs = new DocumentDefinitionCollection(includeAllVersions);
             var templateIds = new List();
             
             //track the looked up content types, even though the content types are cached
@@ -969,9 +976,8 @@ ORDER BY doc2.updateDate DESC
                 if (withCache)
                 {
                     var cached = RuntimeCache.GetCacheItem(GetCacheIdKey(dto.NodeId));
-                    //only use this cached version if the dto returned is also the publish version, they must match
-                    //TODO: Shouldn't this also match on version!?
-                    if (cached != null && cached.Published && dto.Published)
+                    //only use this cached version if the dto returned is also the publish version, they must match and be teh same version
+                    if (cached != null && cached.Version == dto.VersionId && cached.Published && dto.Published)
                     {
                         content.Add(cached);
                         continue;
@@ -991,16 +997,30 @@ ORDER BY doc2.updateDate DESC
                     contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId);
                     contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
                 }
-                
-                // track the definition and if it's successfully added or updated then processed
-                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+
+                if (includeAllVersions)
                 {
-                    // need template
+                    // track the definition
+                    defs.Add(new DocumentDefinition(dto, contentType));
+
+                    // assign template
                     if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
                         templateIds.Add(dto.TemplateId.Value);
 
                     content.Add(ContentFactory.BuildEntity(dto, contentType, publishedDto));
                 }
+                else
+                {
+                    // track the definition and if it's successfully added or updated then processed
+                    if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
+                    {
+                        // assign template
+                        if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
+                            templateIds.Add(dto.TemplateId.Value);
+
+                        content.Add(ContentFactory.BuildEntity(dto, contentType, publishedDto));
+                    }
+                }
             }
 
             // load all required templates in 1 query
@@ -1013,7 +1033,7 @@ ORDER BY doc2.updateDate DESC
             // assign template and property data
             foreach (var cc in content)
             {
-                var def = defs[cc.Id];
+                var def = defs[includeAllVersions ? (ValueType)cc.Version : cc.Id];
 
                 ITemplate template = null;
                 if (def.DocumentDto.TemplateId.HasValue)
diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
index 3f0d1d2486..272ea4334e 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
@@ -175,8 +175,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 if (withCache)
                 {
                     var cached = RuntimeCache.GetCacheItem(GetCacheIdKey(dto.NodeId));
-                    //TODO: Shouldn't this also match on version!?
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, media doesn't 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.VersionId)
                     {
                         content.Add(cached);
                         continue;
diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
index 23b42108bc..f81424c99d 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
@@ -690,7 +690,6 @@ namespace Umbraco.Core.Persistence.Repositories
                 if (withCache)
                 {
                     var cached = RuntimeCache.GetCacheItem(GetCacheIdKey(dto.NodeId));
-                    //TODO: Shouldn't this also match on version!?
                     if (cached != null)
                     {
                         content.Add(cached);
diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
index 0d9d0bf3d2..f96b49c1db 100644
--- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
@@ -773,11 +773,22 @@ ORDER BY contentNodeId, propertytypeid
         /// 
         protected abstract Sql GetBaseQuery(BaseQueryType queryType);
 
-        internal class DocumentDefinitionCollection : KeyedCollection
+        internal class DocumentDefinitionCollection : KeyedCollection
         {
-            protected override int GetKeyForItem(DocumentDefinition item)
+            private readonly bool _includeAllVersions;
+
+            /// 
+            /// Constructor specifying if all versions should be allowed, in that case the key for the collection becomes the versionId (GUID)
+            /// 
+            /// 
+            public DocumentDefinitionCollection(bool includeAllVersions = false)
             {
-                return item.Id;
+                _includeAllVersions = includeAllVersions;
+            }
+
+            protected override ValueType GetKeyForItem(DocumentDefinition item)
+            {
+                return _includeAllVersions ? (ValueType)item.Version : item.Id;
             }
 
             /// 
@@ -817,7 +828,7 @@ ORDER BY contentNodeId, propertytypeid
                 return true;
             }
           
-            public bool TryGetValue(int key, out DocumentDefinition val)
+            public bool TryGetValue(ValueType key, out DocumentDefinition val)
             {
                 if (Dictionary == null)
                 {
diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
index cf2347c9e9..04f8944232 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
@@ -121,6 +121,14 @@ namespace Umbraco.Tests.Persistence.Repositories
                 var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray();
                 Assert.AreEqual(1, content.Length);
                 Assert.AreEqual(content[0].UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture));
+
+                var contentItem = repository.GetByVersion(content1.Version);
+                Assert.IsNotNull(contentItem);
+
+                var allVersions = repository.GetAllVersions(content[0].Id);
+                var allKnownVersions = versionDtos.Select(x => x.VersionId).Union(new[]{ content1.Version }).ToArray();
+                Assert.IsTrue(allKnownVersions.ContainsAll(allVersions.Select(x => x.Version)));
+                Assert.IsTrue(allVersions.Select(x => x.Version).ContainsAll(allKnownVersions));
             }
         }
 

From 28f0ab1001e0b2f7b4d5ee0f1fe390cadf7498c0 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Fri, 24 Feb 2017 13:51:14 +1100
Subject: [PATCH 19/41] Fixes U4-9456, ensures that property sets are mapped to
 a content version instead of an Id, adds more unit tests, ensures that if
 this warning ever occurs in a test that the test will fail.

---
 .../Repositories/ContentRepository.cs         |  4 +-
 .../Repositories/MediaRepository.cs           |  4 +-
 .../Repositories/MemberRepository.cs          | 10 +--
 .../Repositories/VersionableRepositoryBase.cs | 27 ++++++--
 .../Repositories/ContentRepositoryTest.cs     | 66 ++++++++++++++++++-
 .../Services/ContentServiceTests.cs           |  4 ++
 6 files changed, 99 insertions(+), 16 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index cd1793f4d5..e156e1c112 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -1039,7 +1039,7 @@ ORDER BY doc2.updateDate DESC
                 if (def.DocumentDto.TemplateId.HasValue)
                     templates.TryGetValue(def.DocumentDto.TemplateId.Value, out template); // else null
                 cc.Template = template;
-                cc.Properties = propertyData[cc.Id];
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
@@ -1071,7 +1071,7 @@ ORDER BY doc2.updateDate DESC
 
             var properties = GetPropertyCollection(docSql, new[] { docDef });
 
-            content.Properties = properties[dto.NodeId];
+            content.Properties = properties[dto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946
diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
index 272ea4334e..03435c5781 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
@@ -211,7 +211,7 @@ namespace Umbraco.Core.Persistence.Repositories
             // assign property data
             foreach (var cc in content)
             {
-                cc.Properties = propertyData[cc.Id];
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
@@ -529,7 +529,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
             var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef });
 
-            media.Properties = properties[dto.NodeId];
+            media.Properties = properties[dto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946
diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
index f81424c99d..9d01e195c7 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
@@ -452,7 +452,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
             var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.ContentVersionDto, memberType) });
 
-            member.Properties = properties[dto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946
@@ -690,7 +690,9 @@ namespace Umbraco.Core.Persistence.Repositories
                 if (withCache)
                 {
                     var cached = RuntimeCache.GetCacheItem(GetCacheIdKey(dto.NodeId));
-                    if (cached != null)
+                    //only use this cached version if the dto returned is the same version - this is just a safety check, members dont 
+                    //store different versions, but just in case someone corrupts some data we'll double check to be sure.
+                    if (cached != null && cached.Version == dto.ContentVersionDto.VersionId)
                     {
                         content.Add(cached);
                         continue;
@@ -714,7 +716,7 @@ namespace Umbraco.Core.Persistence.Repositories
             // assign property data
             foreach (var cc in content)
             {
-                cc.Properties = propertyData[cc.Id];
+                cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
@@ -741,7 +743,7 @@ namespace Umbraco.Core.Persistence.Repositories
 
             var properties = GetPropertyCollection(docSql, new[] { docDef });
 
-            member.Properties = properties[dto.ContentVersionDto.NodeId];
+            member.Properties = properties[dto.ContentVersionDto.VersionId];
 
             //on initial construction we don't want to have dirty properties tracked
             // http://issues.umbraco.org/issue/U4-1946
diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
index f96b49c1db..412d3eea78 100644
--- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
@@ -32,6 +32,11 @@ namespace Umbraco.Core.Persistence.Repositories
     {
         private readonly IContentSection _contentSection;
 
+        /// 
+        /// This is used for unit tests ONLY
+        /// 
+        internal static bool ThrowOnWarning = false;
+
         protected VersionableRepositoryBase(IDatabaseUnitOfWork work, CacheHelper cache, ILogger logger, ISqlSyntaxProvider sqlSyntax, IContentSection contentSection)
             : base(work, cache, logger, sqlSyntax)
         {
@@ -502,7 +507,7 @@ namespace Umbraco.Core.Persistence.Repositories
         /// 
         /// 
         /// 
-        protected IDictionary GetPropertyCollection(
+        protected IDictionary GetPropertyCollection(
             Sql sql,
             IReadOnlyCollection documentDefs)
         {
@@ -515,11 +520,11 @@ namespace Umbraco.Core.Persistence.Repositories
         /// 
         /// 
         /// 
-        protected IDictionary GetPropertyCollection(
+        protected IDictionary GetPropertyCollection(
             PagingSqlQuery pagingSqlQuery,
             IReadOnlyCollection documentDefs)
         {
-            if (documentDefs.Count == 0) return new Dictionary();
+            if (documentDefs.Count == 0) return new Dictionary();
 
             //initialize to the query passed in
             var docSql = pagingSqlQuery.PrePagedSql;
@@ -584,7 +589,7 @@ ORDER BY contentNodeId, propertytypeid
             // from SQL server otherwise we'll get an exception.
             var allPropertyData = Database.Query(propSql);
 
-            var result = new Dictionary();
+            var result = new Dictionary();
             var propertiesWithTagSupport = new Dictionary();
             //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time
             var resolvedCompositionProperties = new Dictionary();
@@ -661,11 +666,19 @@ ORDER BY contentNodeId, propertytypeid
                         }
                     }
 
-                    if (result.ContainsKey(def.Id))
+                    if (result.ContainsKey(def.Version))
                     {
-                        Logger.Warn>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name);
+                        var msg = string.Format("The query returned multiple property sets for document definition {0}, {1}, {2}", def.Id, def.Version, def.Composition.Name);
+                        if (ThrowOnWarning)
+                        {
+                            throw new InvalidOperationException(msg);
+                        }
+                        else
+                        {
+                            Logger.Warn>(msg);
+                        }
                     }
-                    result[def.Id] = new PropertyCollection(properties);
+                    result[def.Version] = new PropertyCollection(properties);
                 }
             }
             finally
diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
index 04f8944232..eaa3aa1835 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
@@ -36,11 +36,15 @@ namespace Umbraco.Tests.Persistence.Repositories
             base.Initialize();
 
             CreateTestData();
+
+            VersionableRepositoryBase.ThrowOnWarning = true;
         }
 
         [TearDown]
         public override void TearDown()
         {
+            VersionableRepositoryBase.ThrowOnWarning = false;
+
             base.TearDown();
         }
 
@@ -66,7 +70,61 @@ namespace Umbraco.Tests.Persistence.Repositories
             var repository = new ContentRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, contentTypeRepository, templateRepository, tagRepository, Mock.Of());
             return repository;
         }
-        
+
+        [Test]
+        public void Get_Always_Returns_Latest_Version()
+        {
+            // Arrange
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            IContent content1;
+
+            var versions = new List();
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage");
+                content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType);
+
+                //save version
+                contentTypeRepository.AddOrUpdate(hasPropertiesContentType);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+                versions.Add(content1.Version);
+
+                //publish version
+                content1.ChangePublishedState(PublishedState.Published);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+                versions.Add(content1.Version);
+
+                //change something and make a pending version
+                content1.Name = "new name";
+                content1.ChangePublishedState(PublishedState.Saved);
+                repository.AddOrUpdate(content1);
+                unitOfWork.Commit();
+                versions.Add(content1.Version);
+            }
+
+            // Assert
+            Assert.AreEqual(3, versions.Distinct().Count());
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray()[0];
+                Assert.AreEqual(versions[2], content.Version);
+
+                content = repository.Get(content1.Id);
+                Assert.AreEqual(versions[2], content.Version);
+
+                foreach (var version in versions)
+                {
+                    content = repository.GetByVersion(version);
+                    Assert.IsNotNull(content);
+                    Assert.AreEqual(version, content.Version);
+                }
+            }
+        }
+
         [Test]
         public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags()
         {
@@ -120,11 +178,17 @@ namespace Umbraco.Tests.Persistence.Repositories
             {
                 var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray();
                 Assert.AreEqual(1, content.Length);
+                Assert.AreEqual(content[0].Version, versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionId);
                 Assert.AreEqual(content[0].UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture));
 
                 var contentItem = repository.GetByVersion(content1.Version);
                 Assert.IsNotNull(contentItem);
 
+                contentItem = repository.Get(content1.Id);
+                Assert.IsNotNull(contentItem);
+                Assert.AreEqual(contentItem.UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture));
+                Assert.AreEqual(contentItem.Version, versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionId);
+
                 var allVersions = repository.GetAllVersions(content[0].Id);
                 var allKnownVersions = versionDtos.Select(x => x.VersionId).Union(new[]{ content1.Version }).ToArray();
                 Assert.IsTrue(allKnownVersions.ContainsAll(allVersions.Select(x => x.Version)));
diff --git a/src/Umbraco.Tests/Services/ContentServiceTests.cs b/src/Umbraco.Tests/Services/ContentServiceTests.cs
index e89d520f0f..c5c4e3dc32 100644
--- a/src/Umbraco.Tests/Services/ContentServiceTests.cs
+++ b/src/Umbraco.Tests/Services/ContentServiceTests.cs
@@ -36,11 +36,15 @@ namespace Umbraco.Tests.Services
         public override void Initialize()
         {
             base.Initialize();
+
+            VersionableRepositoryBase.ThrowOnWarning = true;
         }
 
         [TearDown]
         public override void TearDown()
         {
+            VersionableRepositoryBase.ThrowOnWarning = false;
+
             base.TearDown();
         }
 

From 8122c0b3523e08d33544673ae561b395d4606d38 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Fri, 24 Feb 2017 14:00:38 +1100
Subject: [PATCH 20/41] Simplifies the includeAllVersions check

---
 .../Repositories/ContentRepository.cs          | 18 ++----------------
 .../Repositories/VersionableRepositoryBase.cs  |  7 +++++++
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index e156e1c112..30630fd223 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -998,29 +998,15 @@ ORDER BY doc2.updateDate DESC
                     contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType;
                 }
 
-                if (includeAllVersions)
+                // track the definition and if it's successfully added or updated then processed
+                if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
                 {
-                    // track the definition
-                    defs.Add(new DocumentDefinition(dto, contentType));
-
                     // assign template
                     if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
                         templateIds.Add(dto.TemplateId.Value);
 
                     content.Add(ContentFactory.BuildEntity(dto, contentType, publishedDto));
                 }
-                else
-                {
-                    // track the definition and if it's successfully added or updated then processed
-                    if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
-                    {
-                        // assign template
-                        if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
-                            templateIds.Add(dto.TemplateId.Value);
-
-                        content.Add(ContentFactory.BuildEntity(dto, contentType, publishedDto));
-                    }
-                }
             }
 
             // load all required templates in 1 query
diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
index 412d3eea78..50c204cdba 100644
--- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
@@ -812,6 +812,13 @@ ORDER BY contentNodeId, propertytypeid
             /// 
             public bool AddOrUpdate(DocumentDefinition item)
             {
+                //if we are including all versions then just add, we aren't checking for latest
+                if (_includeAllVersions)
+                {
+                    base.Add(item);
+                    return true;
+                }
+
                 if (Dictionary == null)
                 {
                     base.Add(item);

From 09492fa7790a8be5770a069457f357194785c6ae Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Fri, 24 Feb 2017 14:14:28 +1100
Subject: [PATCH 21/41] Ensures that the published data retrieved uses the
 latest content version PK instead of by date in case dates are exactly the
 same.

---
 .../Persistence/Repositories/ContentRepository.cs             | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index 30630fd223..393e9eadee 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -928,7 +928,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
 
             //we need to parse the original SQL statement and reduce the columns to just cmsDocument.nodeId so that we can use
             // the statement to go get the published data for all of the items by using an inner join
-            var parsedOriginalSql = "SELECT cmsDocument.nodeId " + sqlFull.SQL.Substring(sqlFull.SQL.IndexOf("FROM", StringComparison.Ordinal));
+            var parsedOriginalSql = "SELECT cmsDocument.nodeId, cmsContentVersion.id as contentVersionPk " + sqlFull.SQL.Substring(sqlFull.SQL.IndexOf("FROM", StringComparison.Ordinal));
             //now remove everything from an Orderby clause and beyond
             if (parsedOriginalSql.InvariantContains("ORDER BY "))
             {
@@ -942,7 +942,7 @@ INNER JOIN
 	(" + parsedOriginalSql + @") as docData
 ON doc2.nodeId = docData.nodeId
 WHERE doc2.published = 1
-ORDER BY doc2.updateDate DESC
+ORDER BY docData.contentVersionPk DESC
 ", sqlFull.Arguments);
 
             //go and get the published version data, we do a Query here and not a Fetch so we are

From ed87789abe2dd64aafd0e28533f729c295e0b2cc Mon Sep 17 00:00:00 2001
From: Daniel Sune Andreasen 
Date: Mon, 27 Feb 2017 11:14:37 +0100
Subject: [PATCH 22/41] collides with .legacy-custom-file

---
 src/Umbraco.Web.UI.Client/src/less/tree.less | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Umbraco.Web.UI.Client/src/less/tree.less b/src/Umbraco.Web.UI.Client/src/less/tree.less
index 4e2e3e77e8..013b60ce19 100644
--- a/src/Umbraco.Web.UI.Client/src/less/tree.less
+++ b/src/Umbraco.Web.UI.Client/src/less/tree.less
@@ -40,7 +40,7 @@
 .umb-tree li.current > div i.icon,
 .umb-tree li.current > div ins {
 	color: white !important;
-	background: @blue;
+	background-color: @blue;
 	border-color: @blue;
 }
 .umb-tree li.root > div {

From 259879c9641dede9f6a8d11efc8b50b3a0eed9dc Mon Sep 17 00:00:00 2001
From: Claus 
Date: Mon, 27 Feb 2017 11:59:19 +0100
Subject: [PATCH 23/41] just cleaning some using statements and fixing a few
 typos.

---
 .../Persistence/Querying/ExpressionVisitorBase.cs   |  6 +++---
 .../Querying/ModelToSqlExpressionVisitor.cs         |  1 -
 .../Persistence/Querying/SqlExpressionExtensions.cs |  3 +--
 .../Repositories/ContentTypeRepository.cs           |  6 ------
 .../Persistence/Repositories/EntityRepository.cs    |  6 ------
 src/Umbraco.Core/Services/MediaService.cs           |  4 ----
 src/Umbraco.Web/Editors/EntityController.cs         | 13 -------------
 src/Umbraco.Web/Editors/MediaController.cs          |  9 ---------
 8 files changed, 4 insertions(+), 44 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs
index 0dc421045d..47d5ee6bd8 100644
--- a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs
+++ b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs
@@ -581,15 +581,15 @@ namespace Umbraco.Core.Persistence.Querying
                 case "InvariantContains":
                 case "InvariantEquals":
 
-                    //special case, if it is 'Contains' and the argumet that Contains is being called on is 
-                    //Enumerable and the methodArgs is the actual member access, then it's an SQL IN claus
+                    //special case, if it is 'Contains' and the argument that Contains is being called on is 
+                    //Enumerable and the methodArgs is the actual member access, then it's an SQL IN clause
                     if (m.Object == null 
                         && m.Arguments[0].Type != typeof(string)
                         && m.Arguments.Count == 2
                         && methodArgs.Length == 1 
                         && methodArgs[0].NodeType == ExpressionType.MemberAccess
                         && TypeHelper.IsTypeAssignableFrom(m.Arguments[0].Type))
-                    {                        
+                    {
                         goto case "SqlIn";
                     }
 
diff --git a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs
index bd6a2020ab..da1f93a032 100644
--- a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs
+++ b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs
@@ -1,5 +1,4 @@
 using System;
-using System.Linq;
 using System.Linq.Expressions;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Persistence.Mappers;
diff --git a/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs b/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs
index 5a0452c3aa..593955734e 100644
--- a/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs
+++ b/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs
@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
 
diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs
index e56f06e0e0..55af8fc60b 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs
@@ -2,19 +2,13 @@
 using System.Collections.Generic;
 using System.Linq;
 using Umbraco.Core.Cache;
-using Umbraco.Core.Events;
-using Umbraco.Core.Exceptions;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
-
-using Umbraco.Core.Persistence.Factories;
 using Umbraco.Core.Persistence.Querying;
-using Umbraco.Core.Persistence.Relators;
 using Umbraco.Core.Persistence.SqlSyntax;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Services;
 
 namespace Umbraco.Core.Persistence.Repositories
 {
diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
index 87f4a7811c..ec60ae605b 100644
--- a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs
@@ -1,18 +1,12 @@
 using System;
 using System.Collections.Generic;
-using System.Dynamic;
-using System.Globalization;
 using System.Linq;
-using System.Reflection;
-using System.Text;
 using Umbraco.Core.Models;
-using Umbraco.Core;
 using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models.Rdbms;
 using Umbraco.Core.Persistence.Factories;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Strings;
 
 namespace Umbraco.Core.Persistence.Repositories
 {
diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs
index d236dccc5a..51f8ed5148 100644
--- a/src/Umbraco.Core/Services/MediaService.cs
+++ b/src/Umbraco.Core/Services/MediaService.cs
@@ -7,7 +7,6 @@ using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Xml.Linq;
-using Umbraco.Core.Auditing;
 using Umbraco.Core.Configuration;
 using Umbraco.Core.Events;
 using Umbraco.Core.Logging;
@@ -17,10 +16,7 @@ using Umbraco.Core.Persistence;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Persistence.Querying;
 using Umbraco.Core.Persistence.Repositories;
-using Umbraco.Core.Persistence.SqlSyntax;
 using Umbraco.Core.Persistence.UnitOfWork;
-using Umbraco.Core.Publishing;
-using ContentType = System.Net.Mime.ContentType;
 
 namespace Umbraco.Core.Services
 {
diff --git a/src/Umbraco.Web/Editors/EntityController.cs b/src/Umbraco.Web/Editors/EntityController.cs
index dfe0c87d10..cd8e91274f 100644
--- a/src/Umbraco.Web/Editors/EntityController.cs
+++ b/src/Umbraco.Web/Editors/EntityController.cs
@@ -4,29 +4,16 @@ using System.Globalization;
 using System.Net;
 using System.Text;
 using System.Web.Http;
-using System.Web.Http.ModelBinding;
 using AutoMapper;
-using ClientDependency.Core;
-using Examine.LuceneEngine;
-using Examine.LuceneEngine.Providers;
-using Newtonsoft.Json;
 using Umbraco.Core;
-using Umbraco.Core.Logging;
 using Umbraco.Core.Models.Membership;
-using Umbraco.Core.Services;
 using Umbraco.Web.Models.ContentEditing;
 using Umbraco.Web.Mvc;
 using System.Linq;
-using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models;
-using Umbraco.Web.WebApi.Filters;
-using umbraco.cms.businesslogic.packager;
 using Constants = Umbraco.Core.Constants;
 using Examine;
-using Examine.LuceneEngine.SearchCriteria;
-using Examine.SearchCriteria;
 using Umbraco.Web.Dynamics;
-using umbraco;
 using System.Text.RegularExpressions;
 using Umbraco.Core.Xml;
 
diff --git a/src/Umbraco.Web/Editors/MediaController.cs b/src/Umbraco.Web/Editors/MediaController.cs
index 2f3fe5bd94..82ddfe3fa9 100644
--- a/src/Umbraco.Web/Editors/MediaController.cs
+++ b/src/Umbraco.Web/Editors/MediaController.cs
@@ -5,37 +5,28 @@ using System.IO;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Formatting;
-using System.Security.AccessControl;
 using System.Text;
 using System.Threading.Tasks;
-using System.Web;
 using System.Web.Http;
 using System.Web.Http.ModelBinding;
 using AutoMapper;
 using Umbraco.Core;
-using Umbraco.Core.Dynamics;
 using Umbraco.Core.IO;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
-using Umbraco.Core.Models.Editors;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Services;
-using Umbraco.Web.Models;
 using Umbraco.Web.Models.ContentEditing;
 using Umbraco.Web.Models.Mapping;
 using Umbraco.Web.Mvc;
 using Umbraco.Web.WebApi;
 using System.Linq;
-using System.Runtime.Serialization;
 using System.Web.Http.Controllers;
 using Umbraco.Web.WebApi.Binders;
 using Umbraco.Web.WebApi.Filters;
-using umbraco;
-using umbraco.BusinessLogic.Actions;
 using Constants = Umbraco.Core.Constants;
 using Umbraco.Core.Configuration;
-using Umbraco.Core.Persistence.FaultHandling;
 using Umbraco.Web.UI;
 using Notification = Umbraco.Web.Models.ContentEditing.Notification;
 using Umbraco.Core.Persistence;

From 65c39a3117abf60acb54baf03c0c455d585c6c24 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Mon, 27 Feb 2017 15:19:59 +0100
Subject: [PATCH 24/41] Fixes issue with returning multiple versions and
 mapping their property values, adds more assertions

---
 .../Repositories/ContentRepository.cs         | 16 +++---
 .../Repositories/VersionableRepositoryBase.cs | 53 +++++++++++++++----
 .../Services/ContentServiceTests.cs           | 10 ++++
 3 files changed, 62 insertions(+), 17 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index 393e9eadee..e6543ba1d1 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -560,6 +560,7 @@ namespace Umbraco.Core.Persistence.Repositories
             //if (((ICanBeDirty)entity).IsPropertyDirty("Published") && (entity.Published || publishedState == PublishedState.Unpublished))
             if (entity.ShouldClearPublishedFlagForPreviousVersions(publishedState, shouldCreateNewVersion))
             {
+                //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
                 var publishedDocs = Database.Fetch("WHERE nodeId = @Id AND published = @IsPublished", new { Id = entity.Id, IsPublished = true });
                 foreach (var doc in publishedDocs)
                 {
@@ -573,6 +574,7 @@ namespace Umbraco.Core.Persistence.Repositories
             }
 
             //Look up (newest) entries by id in cmsDocument table to set newest = false
+            //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many)
             var documentDtos = Database.Fetch("WHERE nodeId = @Id AND newest = @IsNewest", new { Id = entity.Id, IsNewest = true });
             foreach (var documentDto in documentDtos)
             {
@@ -928,7 +930,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
 
             //we need to parse the original SQL statement and reduce the columns to just cmsDocument.nodeId so that we can use
             // the statement to go get the published data for all of the items by using an inner join
-            var parsedOriginalSql = "SELECT cmsDocument.nodeId, cmsContentVersion.id as contentVersionPk " + sqlFull.SQL.Substring(sqlFull.SQL.IndexOf("FROM", StringComparison.Ordinal));
+            var parsedOriginalSql = "SELECT cmsDocument.nodeId " + sqlFull.SQL.Substring(sqlFull.SQL.IndexOf("FROM", StringComparison.Ordinal));
             //now remove everything from an Orderby clause and beyond
             if (parsedOriginalSql.InvariantContains("ORDER BY "))
             {
@@ -936,13 +938,11 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder",
             }            
 
             //order by update date DESC, if there is corrupted published flags we only want the latest!
-            var publishedSql = new Sql(@"SELECT *
-FROM cmsDocument AS doc2
-INNER JOIN 
-	(" + parsedOriginalSql + @") as docData
-ON doc2.nodeId = docData.nodeId
-WHERE doc2.published = 1
-ORDER BY docData.contentVersionPk DESC
+            var publishedSql = new Sql(@"SELECT cmsDocument.nodeId, cmsDocument.published, cmsDocument.versionId, cmsDocument.newest
+FROM cmsDocument INNER JOIN cmsContentVersion ON cmsContentVersion.VersionId = cmsDocument.versionId
+WHERE cmsDocument.published = 1 AND cmsDocument.nodeId IN 
+(" + parsedOriginalSql + @")
+ORDER BY cmsContentVersion.id DESC
 ", sqlFull.Arguments);
 
             //go and get the published version data, we do a Query here and not a Fetch so we are
diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
index 50c204cdba..faa197007b 100644
--- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Data.SqlTypes;
 using System.Diagnostics;
 using System.Linq;
 using System.Text;
@@ -580,9 +581,9 @@ ON cmsPropertyData.propertytypeid = cmsPropertyType.id
 INNER JOIN 
 	(" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData
 ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId
-ORDER BY contentNodeId, propertytypeid
+ORDER BY contentNodeId, versionId, propertytypeid
 ", docSql.Arguments);
-
+            
             //This does NOT fetch all data into memory in a list, this will read
             // over the records as a data reader, this is much better for performance and memory,
             // but it means that during the reading of this data set, nothing else can be read
@@ -598,11 +599,13 @@ ORDER BY contentNodeId, propertytypeid
             var propertyDataSetEnumerator = allPropertyData.GetEnumerator();
             var hasCurrent = false; // initially there is no enumerator.Current
 
+            var comparer = new DocumentDefinitionComparer(SqlSyntax);
+
             try
             {
                 //This must be sorted by node id because this is how we are sorting the query to lookup property types above,
                 // which allows us to more efficiently iterate over the large data set of property values
-                foreach (var def in documentDefs.OrderBy(x => x.Id))
+                foreach (var def in documentDefs.OrderBy(x => x.Id).ThenBy(x => x.Version, comparer))
                 {
                     // get the resolved properties from our local cache, or resolve them and put them in cache
                     PropertyType[] compositionProperties;
@@ -621,15 +624,17 @@ ORDER BY contentNodeId, propertytypeid
                     var propertyDataDtos = new List();
                     while (hasCurrent || propertyDataSetEnumerator.MoveNext())
                     {
-                        if (propertyDataSetEnumerator.Current.NodeId == def.Id)
+                        //Not checking null on VersionId because it can never be null - no idea why it's set to nullable
+                        // ReSharper disable once PossibleInvalidOperationException
+                        if (propertyDataSetEnumerator.Current.VersionId.Value == def.Version)
                         {
                             hasCurrent = false; // enumerator.Current is not available
                             propertyDataDtos.Add(propertyDataSetEnumerator.Current);
                         }
                         else
                         {
-                            hasCurrent = true; // enumerator.Current is available for another def
-                            break; // no more propertyDataDto for this def
+                            hasCurrent = true;  // enumerator.Current is available for another def
+                            break;              // no more propertyDataDto for this def
                         }
                     }
 
@@ -701,8 +706,8 @@ ORDER BY contentNodeId, propertytypeid
                 case "NAME":
                     return "umbracoNode.text";
                 case "PUBLISHED":
-                    return "cmsDocument.published";
                 case "OWNER":
+                    return "cmsDocument.published";
                     //TODO: This isn't going to work very nicely because it's going to order by ID, not by letter
                     return "umbracoNode.nodeUser";
                 // Members only
@@ -859,6 +864,34 @@ ORDER BY contentNodeId, propertytypeid
             }
         }
 
+        /// 
+        /// A custom comparer required for sorting entities by GUIDs to match how the sorting of GUIDs works on SQL server
+        /// 
+        /// 
+        /// MySql sorts GUIDs as a string, MSSQL sorts based on byte sections, this comparer will allow sorting GUIDs to be the same as how SQL server does
+        /// 
+        private class DocumentDefinitionComparer : IComparer
+        {
+            private readonly ISqlSyntaxProvider _sqlSyntax;
+
+            public DocumentDefinitionComparer(ISqlSyntaxProvider sqlSyntax)
+            {
+                _sqlSyntax = sqlSyntax;
+            }
+
+            public int Compare(Guid x, Guid y)
+            {
+                //MySql sorts on GUIDs as strings (i.e. normal)
+                if (_sqlSyntax is MySqlSyntaxProvider)
+                {
+                    return x.CompareTo(y);
+                }
+
+                //MSSQL doesn't it sorts them on byte sections!
+                return new SqlGuid(x).CompareTo(new SqlGuid(y));
+            }
+        }
+
         internal class DocumentDefinition
         {
             /// 
@@ -884,7 +917,7 @@ ORDER BY contentNodeId, propertytypeid
             {
                 get { return ContentVersionDto.NodeId; }
             }
-
+            
             public Guid Version
             {
                 get { return DocumentDto != null ? DocumentDto.VersionId : ContentVersionDto.VersionId; }
@@ -908,7 +941,9 @@ ORDER BY contentNodeId, propertytypeid
                 get { return ContentVersionDto.ContentDto.NodeDto.CreateDate; }
             }
 
-            public IContentTypeComposition Composition { get; set; }
+            public IContentTypeComposition Composition { get; set; }            
+
+            
         }
 
         /// 
diff --git a/src/Umbraco.Tests/Services/ContentServiceTests.cs b/src/Umbraco.Tests/Services/ContentServiceTests.cs
index c5c4e3dc32..f4701a951f 100644
--- a/src/Umbraco.Tests/Services/ContentServiceTests.cs
+++ b/src/Umbraco.Tests/Services/ContentServiceTests.cs
@@ -766,16 +766,26 @@ namespace Umbraco.Tests.Services
             var parent = ServiceContext.ContentService.GetById(NodeDto.NodeIdSeed + 1);
             ServiceContext.ContentService.Publish(parent);//Publishing root, so Text Page 2 can be updated.
             var subpage2 = contentService.GetById(NodeDto.NodeIdSeed + 3);
+
             subpage2.Name = "Text Page 2 Updated";
             subpage2.SetValue("author", "Jane Doe");
             contentService.SaveAndPublishWithStatus(subpage2, 0);//NOTE New versions are only added between publish-state-changed, so publishing to ensure addition version.
 
+            subpage2.Name = "Text Page 2 Updated again";
+            subpage2.SetValue("author", "Bob Hope");
+            contentService.SaveAndPublishWithStatus(subpage2, 0);//NOTE New versions are only added between publish-state-changed, so publishing to ensure addition version.
+
             // Act
             var versions = contentService.GetVersions(NodeDto.NodeIdSeed + 3).ToList();
 
             // Assert
             Assert.That(versions.Any(), Is.True);
             Assert.That(versions.Count(), Is.GreaterThanOrEqualTo(2));
+
+            //ensure each version contains the correct property values
+            Assert.AreEqual("John Doe", versions[2].GetValue("author"));
+            Assert.AreEqual("Jane Doe", versions[1].GetValue("author"));
+            Assert.AreEqual("Bob Hope", versions[0].GetValue("author"));
         }
 
         [Test]

From 0f32c2d6809daef554c31ce07c76ff745865e4bb Mon Sep 17 00:00:00 2001
From: Colletz 
Date: Tue, 28 Feb 2017 12:43:49 +0100
Subject: [PATCH 25/41] PublicAccess - Ensure correct ordering on join

Ensured the correct order on JOIN for the subsequent mapping on AccessDto and AccessRuleDto.
---
 .../Persistence/Repositories/PublicAccessRepository.cs       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs b/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs
index 37200b2172..4ed29abd13 100644
--- a/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs
@@ -69,7 +69,8 @@ namespace Umbraco.Core.Persistence.Repositories
             sql.Select("*")
                 .From(SqlSyntax)
                 .LeftJoin(SqlSyntax)
-                .On(SqlSyntax, left => left.Id, right => right.AccessId);
+                .On(SqlSyntax, left => left.Id, right => right.AccessId)
+                .OrderBy(new string[] { "umbracoAccess.id" });
 
             return sql;
         }
@@ -164,4 +165,4 @@ namespace Umbraco.Core.Persistence.Repositories
             return entity.Key;
         }
     }
-}
\ No newline at end of file
+}

From 337ba4ef349f9d39f856efcdfbb2a877a611afc9 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Tue, 28 Feb 2017 23:53:12 +0100
Subject: [PATCH 26/41] U4-9577 Ability to store the xml content file in diff
 storage locations: Default, Environment Temp or ASP.NET temp location

---
 .../Configuration/ContentXmlStorage.cs        |  9 ++++++++
 .../Configuration/GlobalSettings.cs           | 19 +++++++++++++---
 src/Umbraco.Core/IO/SystemFiles.cs            | 22 +++++++++++++++----
 src/Umbraco.Core/Umbraco.Core.csproj          |  1 +
 src/umbraco.businesslogic/IO/SystemFiles.cs   |  5 ++++-
 5 files changed, 48 insertions(+), 8 deletions(-)
 create mode 100644 src/Umbraco.Core/Configuration/ContentXmlStorage.cs

diff --git a/src/Umbraco.Core/Configuration/ContentXmlStorage.cs b/src/Umbraco.Core/Configuration/ContentXmlStorage.cs
new file mode 100644
index 0000000000..7cbbc70675
--- /dev/null
+++ b/src/Umbraco.Core/Configuration/ContentXmlStorage.cs
@@ -0,0 +1,9 @@
+namespace Umbraco.Core.Configuration
+{
+    internal enum ContentXmlStorage
+    {
+        Default,
+        AspNetTemp,
+        EnvironmentTemp
+    }
+}
\ No newline at end of file
diff --git a/src/Umbraco.Core/Configuration/GlobalSettings.cs b/src/Umbraco.Core/Configuration/GlobalSettings.cs
index 525bff2999..fc2534633f 100644
--- a/src/Umbraco.Core/Configuration/GlobalSettings.cs
+++ b/src/Umbraco.Core/Configuration/GlobalSettings.cs
@@ -520,12 +520,25 @@ namespace Umbraco.Core.Configuration
         }
 
         internal static bool ContentCacheXmlStoredInCodeGen
+        {
+            get { return ContentCacheXmlStorageLocation == ContentXmlStorage.AspNetTemp; }
+        }
+
+        internal static ContentXmlStorage ContentCacheXmlStorageLocation
         {
             get
             {
-                //defaults to false
-                return ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp") 
-                    && bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]); //default to false
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLStorage"))
+                {
+                    return Enum.Parse(ConfigurationManager.AppSettings["umbracoContentXMLStorage"]);
+                }
+                if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp"))
+                {
+                    return bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]) 
+                        ? ContentXmlStorage.AspNetTemp 
+                        : ContentXmlStorage.Default;
+                }
+                return ContentXmlStorage.Default;
             }
         }
 
diff --git a/src/Umbraco.Core/IO/SystemFiles.cs b/src/Umbraco.Core/IO/SystemFiles.cs
index 48bdea2884..437ddd3ef7 100644
--- a/src/Umbraco.Core/IO/SystemFiles.cs
+++ b/src/Umbraco.Core/IO/SystemFiles.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Configuration;
 using System.IO;
 using System.Linq;
@@ -72,15 +73,28 @@ namespace Umbraco.Core.IO
         {
             get
             {
-                if (GlobalSettings.ContentCacheXmlStoredInCodeGen && SystemUtilities.GetCurrentTrustLevel() == AspNetHostingPermissionLevel.Unrestricted)
-                {
-                    return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                switch (GlobalSettings.ContentCacheXmlStorageLocation)
+                {                    
+                    case ContentXmlStorage.AspNetTemp:
+                        return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config");
+                    case ContentXmlStorage.EnvironmentTemp:
+                        var appDomainHash = HttpRuntime.AppDomainAppId.ToSHA1();
+                        var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "UmbracoXml",
+                            //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back
+                            // to worker A again, in theory the %temp%  folder should already be empty but we really want to make sure that its not
+                            // utilizing an old path
+                            appDomainHash);
+                        return Path.Combine(cachePath, "umbraco.config");
+                    case ContentXmlStorage.Default:
+                        return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
+                    default:
+                        throw new ArgumentOutOfRangeException();
                 }
-                return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config");
             }
         }
 
         [Obsolete("Use GlobalSettings.ContentCacheXmlStoredInCodeGen instead")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
         internal static bool ContentCacheXmlStoredInCodeGen
         {
             get { return GlobalSettings.ContentCacheXmlStoredInCodeGen; }
diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj
index 4aca6bdf02..e03dabfa5f 100644
--- a/src/Umbraco.Core/Umbraco.Core.csproj
+++ b/src/Umbraco.Core/Umbraco.Core.csproj
@@ -179,6 +179,7 @@
     
     
     
+    
     
     
     
diff --git a/src/umbraco.businesslogic/IO/SystemFiles.cs b/src/umbraco.businesslogic/IO/SystemFiles.cs
index 991ec0b29f..3c5841a31b 100644
--- a/src/umbraco.businesslogic/IO/SystemFiles.cs
+++ b/src/umbraco.businesslogic/IO/SystemFiles.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Configuration;
 using System.IO;
 using System.Linq;
@@ -54,7 +55,9 @@ namespace umbraco.IO
 			get { return Umbraco.Core.IO.SystemFiles.ContentCacheXml; }
 		}
 
-		public static bool ContentCacheXmlIsEphemeral
+        [Obsolete("This is not used and will be removed in future versions")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        public static bool ContentCacheXmlIsEphemeral
 		{
 			get { return Umbraco.Core.IO.SystemFiles.ContentCacheXmlStoredInCodeGen; }
 		}

From 2e027e61957dfa7354bb42bb6bba303e1b7700cf Mon Sep 17 00:00:00 2001
From: Niels Hartvig 
Date: Thu, 2 Mar 2017 10:33:00 +0100
Subject: [PATCH 27/41] Removes async from filestream instance

---
 src/Umbraco.Web/umbraco.presentation/content.cs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Umbraco.Web/umbraco.presentation/content.cs b/src/Umbraco.Web/umbraco.presentation/content.cs
index a7c5a75337..a7431ffae9 100644
--- a/src/Umbraco.Web/umbraco.presentation/content.cs
+++ b/src/Umbraco.Web/umbraco.presentation/content.cs
@@ -844,7 +844,7 @@ namespace umbraco
         internal void SaveXmlToFile()
         {
             LogHelper.Info("Save Xml to file...");
-
+            //Thread.Sleep(30000);
             try
             {
                 var xml = _xmlContent; // capture (atomic + volatile), immutable anyway
@@ -861,7 +861,7 @@ namespace umbraco
                     Directory.CreateDirectory(directoryName);
 
                 // save
-                using (var fs = new FileStream(_xmlFileName, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 4096, useAsync: true))
+                using (var fs = new FileStream(_xmlFileName, FileMode.Create, FileAccess.Write, FileShare.Read))
                 {
                     SaveXmlToStream(xml, fs);
                 }

From 7aa00ee6e09958cfb920c575740d8f15e1f848db Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 10:37:04 +0100
Subject: [PATCH 28/41] Removes unused method that also uses the async flag for
 file stream - which we know in some cases causes fcn issues, fixes streams
 not being closed property if there are exceptions

---
 src/Umbraco.Core/XmlExtensions.cs             | 26 -------------------
 .../umbraco.presentation/template.cs          | 24 ++++++++---------
 2 files changed, 12 insertions(+), 38 deletions(-)

diff --git a/src/Umbraco.Core/XmlExtensions.cs b/src/Umbraco.Core/XmlExtensions.cs
index e2518c791a..8707523615 100644
--- a/src/Umbraco.Core/XmlExtensions.cs
+++ b/src/Umbraco.Core/XmlExtensions.cs
@@ -16,32 +16,6 @@ namespace Umbraco.Core
 	/// 
 	internal static class XmlExtensions
 	{
-        /// 
-        /// Saves the xml document async
-        /// 
-        /// 
-        /// 
-        /// 
-	    public static async Task SaveAsync(this XmlDocument xdoc, string filename)
-	    {
-            if (xdoc.DocumentElement == null)
-                throw new XmlException("Cannot save xml document, there is no root element");
-
-            using (var fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 4096, useAsync: true))
-            using (var xmlWriter = XmlWriter.Create(fs, new XmlWriterSettings
-            {
-                Async = true,
-                Encoding = Encoding.UTF8,
-                Indent = true
-            }))
-            {
-                //NOTE: There are no nice methods to write it async, only flushing it async. We
-                // could implement this ourselves but it'd be a very manual process.
-                xdoc.WriteTo(xmlWriter);
-                await xmlWriter.FlushAsync().ConfigureAwait(false);
-            }
-	    }
-
         public static bool HasAttribute(this XmlAttributeCollection attributes, string attributeName)
         {
             return attributes.Cast().Any(x => x.Name == attributeName);
diff --git a/src/Umbraco.Web/umbraco.presentation/template.cs b/src/Umbraco.Web/umbraco.presentation/template.cs
index 4290ce2acf..7b8e60dfb6 100644
--- a/src/Umbraco.Web/umbraco.presentation/template.cs
+++ b/src/Umbraco.Web/umbraco.presentation/template.cs
@@ -88,19 +88,19 @@ namespace umbraco
                 string originalPath = IOHelper.MapPath(VirtualPathUtility.ToAbsolute(MasterPageFile));
                 string copyPath = IOHelper.MapPath(VirtualPathUtility.ToAbsolute(path));
 
-                FileStream fs = new FileStream(originalPath, FileMode.Open, FileAccess.ReadWrite);
-                StreamReader f = new StreamReader(fs);
-                String newfile = f.ReadToEnd();
-                f.Close();
-                fs.Close();
+                string newFile;
+                using (var fs = new FileStream(originalPath, FileMode.Open, FileAccess.ReadWrite))
+                using (var f = new StreamReader(fs))
+                {
+                    newFile = f.ReadToEnd();
+                    newFile = newFile.Replace("MasterPageFile=\"~/masterpages/", "MasterPageFile=\"");
+                }
 
-                newfile = newfile.Replace("MasterPageFile=\"~/masterpages/", "MasterPageFile=\"");
-
-                fs = new FileStream(copyPath, FileMode.Create, FileAccess.Write);
-
-                StreamWriter replacement = new StreamWriter(fs);
-                replacement.Write(newfile);
-                replacement.Close();
+                using (var fs = new FileStream(copyPath, FileMode.Create, FileAccess.Write))
+                using (var replacement = new StreamWriter(fs))
+                {
+                    replacement.Write(newFile);
+                }
             }
 
             return path;

From 7e8f6d391fe19ce8819b4c0c54874866458bb390 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 10:48:11 +0100
Subject: [PATCH 29/41] Just to be sure, moves the text replacement outside of
 the using - though that wouldn't make any diff

---
 src/Umbraco.Web/umbraco.presentation/content.cs  | 1 -
 src/Umbraco.Web/umbraco.presentation/template.cs | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Umbraco.Web/umbraco.presentation/content.cs b/src/Umbraco.Web/umbraco.presentation/content.cs
index a7431ffae9..07f73e88f6 100644
--- a/src/Umbraco.Web/umbraco.presentation/content.cs
+++ b/src/Umbraco.Web/umbraco.presentation/content.cs
@@ -844,7 +844,6 @@ namespace umbraco
         internal void SaveXmlToFile()
         {
             LogHelper.Info("Save Xml to file...");
-            //Thread.Sleep(30000);
             try
             {
                 var xml = _xmlContent; // capture (atomic + volatile), immutable anyway
diff --git a/src/Umbraco.Web/umbraco.presentation/template.cs b/src/Umbraco.Web/umbraco.presentation/template.cs
index 7b8e60dfb6..50d3876f40 100644
--- a/src/Umbraco.Web/umbraco.presentation/template.cs
+++ b/src/Umbraco.Web/umbraco.presentation/template.cs
@@ -92,10 +92,11 @@ namespace umbraco
                 using (var fs = new FileStream(originalPath, FileMode.Open, FileAccess.ReadWrite))
                 using (var f = new StreamReader(fs))
                 {
-                    newFile = f.ReadToEnd();
-                    newFile = newFile.Replace("MasterPageFile=\"~/masterpages/", "MasterPageFile=\"");
+                    newFile = f.ReadToEnd();                    
                 }
 
+                newFile = newFile.Replace("MasterPageFile=\"~/masterpages/", "MasterPageFile=\"");
+
                 using (var fs = new FileStream(copyPath, FileMode.Create, FileAccess.Write))
                 using (var replacement = new StreamWriter(fs))
                 {

From 6c271e7af90c2017db4a1342941e60a13dda7023 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 16:54:25 +0100
Subject: [PATCH 30/41] Fixes issue with U4-9456 when we use the withCache flag
 we do not add that to the defs collection so an error would occur if one was
 resolved from cache.

---
 .../Repositories/ContentRepository.cs         | 20 ++++++++++++-------
 .../Repositories/MediaRepository.cs           | 20 +++++++++++++------
 .../Repositories/MemberRepository.cs          | 17 +++++++++++-----
 src/Umbraco.Core/Services/ContentService.cs   |  1 +
 4 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index 8b9342a3ba..384091e316 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -960,8 +960,8 @@ ORDER BY cmsContentVersion.id DESC
                     publishedDataCollection.Add(publishedDto);
             }
 
-
-            var content = new List();
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List>();
             var defs = new DocumentDefinitionCollection(includeAllVersions);
             var templateIds = new List();
             
@@ -982,7 +982,7 @@ ORDER BY cmsContentVersion.id DESC
                     //only use this cached version if the dto returned is also the publish version, they must match and be teh same version
                     if (cached != null && cached.Version == dto.VersionId && cached.Published && dto.Published)
                     {
-                        content.Add(cached);
+                        content.Add(new Tuple(cached, true));
                         continue;
                     }
                 }
@@ -1008,7 +1008,7 @@ ORDER BY cmsContentVersion.id DESC
                     if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0)
                         templateIds.Add(dto.TemplateId.Value);
 
-                    content.Add(ContentFactory.BuildEntity(dto, contentType, publishedDto));
+                    content.Add(new Tuple(ContentFactory.BuildEntity(dto, contentType, publishedDto), false));
                 }
             }
 
@@ -1020,8 +1020,14 @@ ORDER BY cmsContentVersion.id DESC
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
             // assign template and property data
-            foreach (var cc in content)
+            foreach (var contentItem in content)
             {
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
+
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
                 var def = defs[includeAllVersions ? (ValueType)cc.Version : cc.Id];
 
                 ITemplate template = null;
@@ -1033,9 +1039,9 @@ ORDER BY cmsContentVersion.id DESC
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
                 cc.ResetDirtyProperties(false);
-            }            
+            }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         /// 
diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
index 2278970f26..fc4f8432d1 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
@@ -164,7 +164,9 @@ namespace Umbraco.Core.Persistence.Repositories
         {
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch(sqlFull);
-            var content = new List();
+            
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List>();
             var defs = new DocumentDefinitionCollection();
 
             //track the looked up content types, even though the content types are cached
@@ -182,7 +184,7 @@ namespace Umbraco.Core.Persistence.Repositories
                     //store different versions, but just in case someone corrupts some data we'll double check to be sure.
                     if (cached != null && cached.Version == dto.VersionId)
                     {
-                        content.Add(cached);
+                        content.Add(new Tuple(cached, true));
                         continue;
                     }
                 }
@@ -204,7 +206,7 @@ namespace Umbraco.Core.Persistence.Repositories
                 // track the definition and if it's successfully added or updated then processed
                 if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType)))
                 {
-                    content.Add(MediaFactory.BuildEntity(dto, contentType));
+                    content.Add(new Tuple(MediaFactory.BuildEntity(dto, contentType), false));
                 }
             }
 
@@ -212,16 +214,22 @@ namespace Umbraco.Core.Persistence.Repositories
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
             // assign property data
-            foreach (var cc in content)
+            foreach (var contentItem in content)
             {
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
+
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
                 cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
                 // http://issues.umbraco.org/issue/U4-1946
                 cc.ResetDirtyProperties(false);
-            }            
+            }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         public override IMedia GetByVersion(Guid versionId)
diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
index 9d01e195c7..9d451a3066 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs
@@ -681,7 +681,8 @@ namespace Umbraco.Core.Persistence.Repositories
             // fetch returns a list so it's ok to iterate it in this method
             var dtos = Database.Fetch(sqlFull);
 
-            var content = new List();
+            //This is a tuple list identifying if the content item came from the cache or not
+            var content = new List>();
             var defs = new DocumentDefinitionCollection();
 
             foreach (var dto in dtos)
@@ -694,7 +695,7 @@ namespace Umbraco.Core.Persistence.Repositories
                     //store different versions, but just in case someone corrupts some data we'll double check to be sure.
                     if (cached != null && cached.Version == dto.ContentVersionDto.VersionId)
                     {
-                        content.Add(cached);
+                        content.Add(new Tuple(cached, true));
                         continue;
                     }
                 }
@@ -706,7 +707,7 @@ namespace Umbraco.Core.Persistence.Repositories
                 // need properties
                 if (defs.AddOrUpdate(new DocumentDefinition(dto.ContentVersionDto, contentType)))
                 {
-                    content.Add(MemberFactory.BuildEntity(dto, contentType));
+                    content.Add(new Tuple(MemberFactory.BuildEntity(dto, contentType), false));
                 }
             }
 
@@ -714,8 +715,14 @@ namespace Umbraco.Core.Persistence.Repositories
             var propertyData = GetPropertyCollection(pagingSqlQuery, defs);
 
             // assign property data
-            foreach (var cc in content)
+            foreach (var contentItem in content)
             {
+                var cc = contentItem.Item1;
+                var fromCache = contentItem.Item2;
+
+                //if this has come from cache, we do not need to build up it's structure
+                if (fromCache) continue;
+
                 cc.Properties = propertyData[cc.Version];
 
                 //on initial construction we don't want to have dirty properties tracked
@@ -723,7 +730,7 @@ namespace Umbraco.Core.Persistence.Repositories
                 cc.ResetDirtyProperties(false);
             }
 
-            return content;
+            return content.Select(x => x.Item1).ToArray();
         }
 
         /// 
diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs
index 32665910ad..44e91e2b80 100644
--- a/src/Umbraco.Core/Services/ContentService.cs
+++ b/src/Umbraco.Core/Services/ContentService.cs
@@ -2185,6 +2185,7 @@ namespace Umbraco.Core.Services
                 //We need to check if children and their publish state to ensure that we 'republish' content that was previously published
                 if (published && previouslyPublished == false && HasChildren(content.Id))
                 {
+                    //TODO: Horrible for performance if there are lots of descendents! We should page if anything but this is crazy
                     var descendants = GetPublishedDescendants(content);
 
                     _publishingStrategy.PublishingFinalized(descendants, false);

From 96c28c2611e2589662f5ede05da78123afc58262 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 17:28:34 +0100
Subject: [PATCH 31/41] U4-9587 RebuildXmlStructures doesn't clear out stale
 data so there is unpublished or trashed items remaining in the xml table, xml
 data integrity check is misleading due to the media lookup

---
 .../Repositories/ContentRepository.cs         | 15 +++++
 .../Interfaces/IRepositoryVersionable.cs      |  2 +-
 .../Repositories/MediaRepository.cs           | 24 +++++++-
 src/Umbraco.Core/Services/IMediaService.cs    |  1 +
 src/Umbraco.Core/Services/MediaService.cs     | 14 +++++
 .../Repositories/ContentRepositoryTest.cs     | 56 +++++++++++++++++++
 .../Repositories/MediaRepositoryTest.cs       | 38 +++++++++++++
 .../WebServices/XmlDataIntegrityController.cs |  2 +-
 8 files changed, 147 insertions(+), 5 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index 384091e316..4b31cb6120 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -263,6 +263,21 @@ namespace Umbraco.Core.Persistence.Repositories
                 }
                 baseId = xmlItems[xmlItems.Count - 1].NodeId;
             }
+
+            //now delete the items that shouldn't be there
+            var allContentIds = Database.Fetch(translate(0, GetBaseQuery(BaseQueryType.Ids)));
+
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From(SqlSyntax)
+                .InnerJoin(SqlSyntax)
+                .On(SqlSyntax, left => left.NodeId, right => right.NodeId);
+
+            var allXmlIds = Database.Fetch(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allContentIds).ToArray();
+            if (toRemove.Length > 0)
+                Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = toRemove });
         }
 
         public override IEnumerable GetAllVersions(int id)
diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs
index b318223ca7..6b5fcd43d7 100644
--- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs
@@ -20,7 +20,7 @@ namespace Umbraco.Core.Persistence.Repositories
         /// The serializer to convert TEntity to Xml
         /// Structures will be rebuilt in chunks of this size
         /// 
-        void RebuildXmlStructures(Func serializer, int groupSize = 5000, IEnumerable contentTypeIds = null);
+        void RebuildXmlStructures(Func serializer, int groupSize = 200, IEnumerable contentTypeIds = null);
 
         /// 
         /// Get the total count of entities
diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
index fc4f8432d1..51f61217fa 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
@@ -263,10 +263,12 @@ namespace Umbraco.Core.Persistence.Repositories
                 // get the next group of nodes
                 var query = GetBaseQuery(false);
                 if (contentTypeIdsA.Length > 0)
-                    query = query
-                        .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                {
+                    query = query.WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                }
                 query = query
                     .Where(x => x.NodeId > baseId, SqlSyntax)
+                    .Where(x => x.Trashed == false, SqlSyntax)
                     .OrderBy(x => x.NodeId, SqlSyntax);
                 var sql = SqlSyntax.SelectTop(query, groupSize);
                 var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql))
@@ -290,8 +292,24 @@ namespace Umbraco.Core.Persistence.Repositories
                         Logger.Error("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e);
                     }
                 }
-                baseId = xmlItems.Last().NodeId;
+                baseId = xmlItems[xmlItems.Count - 1].NodeId;
             }
+
+            //now delete the items that shouldn't be there
+            var allMediaIds = Database.Fetch(GetBaseQuery(BaseQueryType.Ids).Where(x => x.Trashed == false, SqlSyntax));
+            var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
+            var xmlIdsQuery = new Sql()
+                .Select("DISTINCT cmsContentXml.nodeId")
+                .From(SqlSyntax)
+                .InnerJoin(SqlSyntax)
+                .On(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .Where(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax);
+
+            var allXmlIds = Database.Fetch(xmlIdsQuery);
+
+            var toRemove = allXmlIds.Except(allMediaIds).ToArray();
+            if (toRemove.Length > 0)
+                Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = toRemove });
         }
 
         public void AddOrUpdateContentXml(IMedia content, Func xml)
diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs
index ac0ff94d3c..b5ef81c193 100644
--- a/src/Umbraco.Core/Services/IMediaService.cs
+++ b/src/Umbraco.Core/Services/IMediaService.cs
@@ -80,6 +80,7 @@ namespace Umbraco.Core.Services
         /// 
         void RebuildXmlStructures(params int[] contentTypeIds);
 
+        int CountNotTrashed(string contentTypeAlias = null);
         int Count(string contentTypeAlias = null);
         int CountChildren(int parentId, string contentTypeAlias = null);
         int CountDescendants(int parentId, string contentTypeAlias = null);
diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs
index 51f8ed5148..3d2271ae1c 100644
--- a/src/Umbraco.Core/Services/MediaService.cs
+++ b/src/Umbraco.Core/Services/MediaService.cs
@@ -245,6 +245,20 @@ namespace Umbraco.Core.Services
             }
         }
 
+        public int CountNotTrashed(string contentTypeAlias = null)
+        {
+            var uow = UowProvider.GetUnitOfWork();
+            using (var repository = RepositoryFactory.CreateMediaRepository(uow))
+            using (var mediaTypeRepository = RepositoryFactory.CreateMediaTypeRepository(uow))
+            {
+                var mediaType = mediaTypeRepository.Get(contentTypeAlias);
+                if (mediaType == null) return 0;
+
+                var query = Query.Builder.Where(media => media.Trashed == false && media.ContentTypeId == mediaType.Id);
+                return repository.Count(query);
+            }
+        }
+
         public int Count(string contentTypeAlias = null)
         {
             var uow = UowProvider.GetUnitOfWork();
diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
index eaa3aa1835..807208e905 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs
@@ -355,6 +355,62 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
         }
 
+        [Test]
+        public void Rebuild_All_Xml_Structures_Ensure_Orphaned_Are_Removed()
+        {
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            ContentTypeRepository contentTypeRepository;
+            using (var repository = CreateRepository(unitOfWork, out contentTypeRepository))
+            {
+                //delete all xml
+                unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
+
+                var contentType1 = MockedContentTypes.CreateSimpleContentType("Textpage1", "Textpage1");
+                contentTypeRepository.AddOrUpdate(contentType1);
+                var allCreated = new List();
+
+                for (var i = 0; i < 100; i++)
+                {
+                    //These will be non-published so shouldn't show up
+                    var c1 = MockedContent.CreateSimpleContent(contentType1);
+                    repository.AddOrUpdate(c1);
+                    allCreated.Add(c1);
+                }
+                for (var i = 0; i < 100; i++)
+                {
+                    var c1 = MockedContent.CreateSimpleContent(contentType1);
+                    c1.ChangePublishedState(PublishedState.Published);
+                    repository.AddOrUpdate(c1);
+                    allCreated.Add(c1);
+                }
+                unitOfWork.Commit();
+
+                //now create some versions of this content - this shouldn't affect the xml structures saved
+                for (int i = 0; i < allCreated.Count; i++)
+                {
+                    allCreated[i].Name = "blah" + i;
+                    repository.AddOrUpdate(allCreated[i]);
+                }
+                unitOfWork.Commit();
+
+                //Add some extra orphaned rows that shouldn't be there
+                var notPublished = MockedContent.CreateSimpleContent(contentType1);
+                repository.AddOrUpdate(notPublished);
+                unitOfWork.Commit();
+                //Force add it
+                unitOfWork.Database.Insert(new ContentXmlDto
+                {
+                    NodeId = notPublished.Id,
+                    Xml = ""
+                });
+
+                repository.RebuildXmlStructures(media => new XElement("test"), 10);
+
+                Assert.AreEqual(100, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml"));
+            }
+        }
+
         [Test]
         public void Rebuild_All_Xml_Structures_For_Content_Type()
         {
diff --git a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs
index 70e72eca9b..d07a0bfaaa 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs
@@ -71,6 +71,44 @@ namespace Umbraco.Tests.Persistence.Repositories
             }
         }
 
+        [Test]
+        public void Rebuild_All_Xml_Structures_Ensure_Orphaned_Are_Removed()
+        {
+            var provider = new PetaPocoUnitOfWorkProvider(Logger);
+            var unitOfWork = provider.GetUnitOfWork();
+            MediaTypeRepository mediaTypeRepository;
+            using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository))
+            {
+                //delete all xml
+                unitOfWork.Database.Execute("DELETE FROM cmsContentXml");
+
+                var mediaType = mediaTypeRepository.Get(1032);
+
+                for (var i = 0; i < 100; i++)
+                {
+                    var image = MockedMedia.CreateMediaImage(mediaType, -1);
+                    repository.AddOrUpdate(image);
+                }
+                unitOfWork.Commit();
+
+                //Add some extra orphaned rows that shouldn't be there
+                var trashed = MockedMedia.CreateMediaImage(mediaType, -1);
+                trashed.ChangeTrashedState(true, Constants.System.RecycleBinMedia);
+                repository.AddOrUpdate(trashed);
+                unitOfWork.Commit();
+                //Force add it
+                unitOfWork.Database.Insert(new ContentXmlDto
+                {
+                    NodeId = trashed.Id,
+                    Xml = ""
+                });
+
+                repository.RebuildXmlStructures(media => new XElement("test"), 10);
+
+                Assert.AreEqual(103, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml"));
+            }
+        }
+
         [Test]
         public void Rebuild_Some_Xml_Structures()
         {
diff --git a/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs b/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs
index 66951d12f2..3d752a287b 100644
--- a/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs
+++ b/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs
@@ -51,7 +51,7 @@ namespace Umbraco.Web.WebServices
         [HttpGet]
         public bool CheckMediaXmlTable()
         {
-            var total = Services.MediaService.Count();
+            var total = Services.MediaService.CountNotTrashed();
             var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
             var subQuery = new Sql()
                 .Select("Count(*)")

From a54680397099c2b695c1a56bf92147bfeda41eb0 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 17:40:39 +0100
Subject: [PATCH 32/41] Need to also update the health check check not just the
 old endpoint for non trashed media.

---
 .../Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs
index 5d888ef117..8b94271e29 100644
--- a/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs
+++ b/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs
@@ -103,7 +103,7 @@ namespace Umbraco.Web.HealthCheck.Checks.DataIntegrity
         /// 
         private HealthCheckStatus CheckMedia()
         {
-            var total = _services.MediaService.Count();
+            var total = _services.MediaService.CountNotTrashed();
             var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
 
             //count entries

From 01b48107f3673a6454a09ce4c47b3b165399b5ad Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 17:45:21 +0100
Subject: [PATCH 33/41] ensures that the content type is not used in the query
 when it is null.

---
 src/Umbraco.Core/Services/MediaService.cs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs
index 3d2271ae1c..c52b85c09e 100644
--- a/src/Umbraco.Core/Services/MediaService.cs
+++ b/src/Umbraco.Core/Services/MediaService.cs
@@ -251,10 +251,19 @@ namespace Umbraco.Core.Services
             using (var repository = RepositoryFactory.CreateMediaRepository(uow))
             using (var mediaTypeRepository = RepositoryFactory.CreateMediaTypeRepository(uow))
             {
-                var mediaType = mediaTypeRepository.Get(contentTypeAlias);
-                if (mediaType == null) return 0;
+                var mediaTypeId = 0;
+                if (contentTypeAlias.IsNullOrWhiteSpace() == false)
+                {
+                    var mediaType = mediaTypeRepository.Get(contentTypeAlias);
+                    if (mediaType == null) return 0;
+                    mediaTypeId = mediaType.Id;
+                }
 
-                var query = Query.Builder.Where(media => media.Trashed == false && media.ContentTypeId == mediaType.Id);
+                var query = Query.Builder.Where(media => media.Trashed == false);
+                if (mediaTypeId > 0)
+                {
+                    query.Where(media => media.ContentTypeId == mediaTypeId);
+                }
                 return repository.Count(query);
             }
         }

From 08d579ec878e9fd0c9aa086ad32b9d2bc0c54fa3 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 18:44:42 +0100
Subject: [PATCH 34/41] Fixes a bunch of id assignments in
 PublicAccessRepository along with clearing out old entries that are flagged
 for removal, adds a lot more assertions for retrieving items and their rules.

---
 src/Umbraco.Core/Models/PublicAccessEntry.cs  |  6 ++
 .../Repositories/PublicAccessRepository.cs    | 18 ++++--
 .../PublicAccessRepositoryTest.cs             | 63 +++++++++++++------
 3 files changed, 63 insertions(+), 24 deletions(-)

diff --git a/src/Umbraco.Core/Models/PublicAccessEntry.cs b/src/Umbraco.Core/Models/PublicAccessEntry.cs
index 1ed15a8fb4..5989f885cb 100644
--- a/src/Umbraco.Core/Models/PublicAccessEntry.cs
+++ b/src/Umbraco.Core/Models/PublicAccessEntry.cs
@@ -110,6 +110,12 @@ namespace Umbraco.Core.Models
             _ruleCollection.Clear();
         }
 
+
+        internal void ClearRemovedRules()
+        {
+            _removedRules.Clear();
+        }
+
         [DataMember]
         public int LoginNodeId
         {
diff --git a/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs b/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs
index 4ed29abd13..32a9d384af 100644
--- a/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs
@@ -44,10 +44,9 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 sql.Where("umbracoAccess.id IN (@ids)", new { ids = ids });
             }
-
-            sql.OrderBy(x => x.NodeId, SqlSyntax);
-
+            
             var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
             var dtos = Database.Fetch(new AccessRulesRelator().Map, sql);
             return dtos.Select(factory.BuildEntity);
         }
@@ -59,6 +58,7 @@ namespace Umbraco.Core.Persistence.Repositories
             var sql = translator.Translate();
 
             var factory = new PublicAccessEntryFactory();
+            //MUST be ordered by this GUID ID for the AccessRulesRelator to work
             var dtos = Database.Fetch(new AccessRulesRelator().Map, sql);
             return dtos.Select(factory.BuildEntity);
         }
@@ -70,7 +70,8 @@ namespace Umbraco.Core.Persistence.Repositories
                 .From(SqlSyntax)
                 .LeftJoin(SqlSyntax)
                 .On(SqlSyntax, left => left.Id, right => right.AccessId)
-                .OrderBy(new string[] { "umbracoAccess.id" });
+                //MUST be ordered by this GUID ID for the AccessRulesRelator to work
+                .OrderBy(dto => dto.Id, SqlSyntax);
 
             return sql;
         }
@@ -111,6 +112,11 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 rule.AccessId = entity.Key;
                 Database.Insert(rule);
+                //update the id so HasEntity is correct
+                var entityRule = entity.Rules.First(x => x.Key == rule.Id);
+                entityRule.Id = entityRule.Key.GetHashCode();
+                //double make sure that this is set since it is possible to add rules via ctor without AddRule
+                entityRule.AccessEntryId = entity.Key;
             }
 
             entity.ResetDirtyProperties();
@@ -131,7 +137,7 @@ namespace Umbraco.Core.Persistence.Repositories
             {
                 if (rule.HasIdentity)
                 {
-                    var count = Database.Update(dto.Rules.Single(x => x.Id == rule.Key));
+                    var count = Database.Update(dto.Rules.First(x => x.Id == rule.Key));
                     if (count == 0)
                     {
                         throw new InvalidOperationException("No rows were updated for the access rule");
@@ -157,6 +163,8 @@ namespace Umbraco.Core.Persistence.Repositories
                 Database.Delete("WHERE id=@Id", new {Id = removedRule});
             }
 
+            entity.ClearRemovedRules();
+
             entity.ResetDirtyProperties();
         }
 
diff --git a/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs
index 4fa212f9bc..00a508dee9 100644
--- a/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs
+++ b/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs
@@ -154,36 +154,61 @@ namespace Umbraco.Tests.Persistence.Repositories
         [Test]
         public void Get_All()
         {
-            var content = CreateTestData(3).ToArray();
+            var content = CreateTestData(30).ToArray();
 
             var provider = new PetaPocoUnitOfWorkProvider(Logger);
             var unitOfWork = provider.GetUnitOfWork();
             using (var repo = new PublicAccessRepository(unitOfWork, CacheHelper, Logger, SqlSyntax))
-            {                
-                var entry1 = new PublicAccessEntry(content[0], content[1], content[2], new[]
+            {
+                var allEntries = new List();
+                for (int i = 0; i < 10; i++)
                 {
-                    new PublicAccessRule
+                    var rules = new List();
+                    for (int j = 0; j < 50; j++)
                     {
-                        RuleValue = "test",
-                        RuleType = "RoleName"
-                    },
-                });
-                repo.AddOrUpdate(entry1);
+                        rules.Add(new PublicAccessRule
+                        {
+                            RuleValue = "test" + j,
+                            RuleType = "RoleName" + j
+                        });
+                    }                    
+                    var entry1 = new PublicAccessEntry(content[i], content[i+1], content[i+2], rules);
+                    repo.AddOrUpdate(entry1);                    
+                    unitOfWork.Commit();
+                    allEntries.Add(entry1);
+                }
 
-                var entry2 = new PublicAccessEntry(content[1], content[0], content[2], new[]
+                //now remove a few rules from a few of the items and then add some more, this will put things 'out of order' which 
+                //we need to verify our sort order is working for the relator
+                for (int i = 0; i < allEntries.Count; i++)
                 {
-                    new PublicAccessRule
+                    //all the even ones
+                    if (i % 2 == 0)
                     {
-                        RuleValue = "test",
-                        RuleType = "RoleName"
-                    },
-                });
-                repo.AddOrUpdate(entry2);
-                
-                unitOfWork.Commit();
+                        var rules = allEntries[i].Rules.ToArray();
+                        for (int j = 0; j < rules.Length; j++)
+                        {
+                            //all the even ones
+                            if (j % 2 == 0)
+                            {
+                                allEntries[i].RemoveRule(rules[j]);
+                            }
+                        }
+                        allEntries[i].AddRule("newrule" + i, "newrule" + i);
+                        repo.AddOrUpdate(allEntries[i]);
+                        unitOfWork.Commit();
+                    }
+                }
 
                 var found = repo.GetAll().ToArray();
-                Assert.AreEqual(2, found.Count());
+                Assert.AreEqual(10, found.Length);
+
+                foreach (var publicAccessEntry in found)
+                {
+                    var matched = allEntries.First(x => x.Key == publicAccessEntry.Key);
+
+                    Assert.AreEqual(matched.Rules.Count(), publicAccessEntry.Rules.Count());
+                }
             }
         }
 

From 0b530ce181275588720f965bfc5f4df36747bd65 Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Thu, 2 Mar 2017 19:17:39 +0100
Subject: [PATCH 35/41] Fixes rebuilding xml structures when a content type is
 specified, we need to filter on that when looking for orphaned items.

---
 .../Persistence/Repositories/ContentRepository.cs  | 14 ++++++++++----
 .../Persistence/Repositories/MediaRepository.cs    |  5 +++++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index 4b31cb6120..d693b86a28 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -265,13 +265,19 @@ namespace Umbraco.Core.Persistence.Repositories
             }
 
             //now delete the items that shouldn't be there
-            var allContentIds = Database.Fetch(translate(0, GetBaseQuery(BaseQueryType.Ids)));
-
+            var sqlAllIds = translate(0, GetBaseQuery(BaseQueryType.Ids));
+            var allContentIds = Database.Fetch(sqlAllIds);
+            var docObjectType = Guid.Parse(Constants.ObjectTypes.Document);
             var xmlIdsQuery = new Sql()
                 .Select("DISTINCT cmsContentXml.nodeId")
                 .From(SqlSyntax)
-                .InnerJoin(SqlSyntax)
-                .On(SqlSyntax, left => left.NodeId, right => right.NodeId);
+                .InnerJoin(SqlSyntax)
+                .On(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                .Where(dto => dto.NodeObjectType == docObjectType, SqlSyntax);
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
 
             var allXmlIds = Database.Fetch(xmlIdsQuery);
 
diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
index 51f61217fa..b8f0486198 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
@@ -304,6 +304,11 @@ namespace Umbraco.Core.Persistence.Repositories
                 .InnerJoin(SqlSyntax)
                 .On(SqlSyntax, left => left.NodeId, right => right.NodeId)
                 .Where(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax);
+            if (contentTypeIdsA.Length > 0)
+            {
+                xmlIdsQuery.WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+            }
+
 
             var allXmlIds = Database.Fetch(xmlIdsQuery);
 

From 9acfd8f5d23edb774bd682fa7e9af24b67cd69c7 Mon Sep 17 00:00:00 2001
From: James Coxhead 
Date: Thu, 2 Mar 2017 21:02:25 +0000
Subject: [PATCH 36/41] Updated dashboard config transform

Fix for dupicate dashboards after upgrading
---
 .../tools/Dashboard.config.install.xdt        | 32 +++++++++++++------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/build/NuSpecs/tools/Dashboard.config.install.xdt b/build/NuSpecs/tools/Dashboard.config.install.xdt
index 8368870186..036beeba29 100644
--- a/build/NuSpecs/tools/Dashboard.config.install.xdt
+++ b/build/NuSpecs/tools/Dashboard.config.install.xdt
@@ -41,16 +41,6 @@
         views/dashboard/developer/examinemanagement.html
       
     
-    
-      
-        views/dashboard/developer/healthcheck.html
-      
-    
-    
-      
-        views/dashboard/developer/redirecturls.html
-      
-    	
   
   
   

@@ -80,4 +70,26 @@
 	
 	
   

+
+  

+    
+      content
+    
+    
+      
+        views/dashboard/developer/redirecturls.html
+      
+    
+  

+
+  

+    
+      developer
+    
+    
+      
+        views/dashboard/developer/healthcheck.html
+      
+    
+  

 
\ No newline at end of file

From 7e4e2a1d5de243857596a40fce8f6c9d2702c18d Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Mon, 6 Mar 2017 14:42:02 +0100
Subject: [PATCH 37/41] Fixes query when rebuilding xml structures for a
 content type

---
 .../Persistence/Repositories/ContentRepository.cs   | 12 +++++++++---
 .../Persistence/Repositories/MediaRepository.cs     | 13 +++++++++----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index d693b86a28..7d354a85d1 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -272,13 +272,19 @@ namespace Umbraco.Core.Persistence.Repositories
                 .Select("DISTINCT cmsContentXml.nodeId")
                 .From(SqlSyntax)
                 .InnerJoin(SqlSyntax)
-                .On(SqlSyntax, left => left.NodeId, right => right.NodeId)
-                .Where(dto => dto.NodeObjectType == docObjectType, SqlSyntax);
+                .On(SqlSyntax, left => left.NodeId, right => right.NodeId);
+
             if (contentTypeIdsA.Length > 0)
             {
-                xmlIdsQuery.WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                xmlIdsQuery.InnerJoin(SqlSyntax)
+                    .On(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin(SqlSyntax)
+                    .On(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
             }
 
+            xmlIdsQuery.Where(dto => dto.NodeObjectType == docObjectType, SqlSyntax);
+            
             var allXmlIds = Database.Fetch(xmlIdsQuery);
 
             var toRemove = allXmlIds.Except(allContentIds).ToArray();
diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
index b8f0486198..155e9bae2f 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
@@ -302,14 +302,19 @@ namespace Umbraco.Core.Persistence.Repositories
                 .Select("DISTINCT cmsContentXml.nodeId")
                 .From(SqlSyntax)
                 .InnerJoin(SqlSyntax)
-                .On(SqlSyntax, left => left.NodeId, right => right.NodeId)
-                .Where(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax);
+                .On(SqlSyntax, left => left.NodeId, right => right.NodeId);
+                
             if (contentTypeIdsA.Length > 0)
             {
-                xmlIdsQuery.WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
+                xmlIdsQuery.InnerJoin(SqlSyntax)
+                    .On(SqlSyntax, left => left.NodeId, right => right.NodeId)
+                    .InnerJoin(SqlSyntax)
+                    .On(SqlSyntax, left => left.NodeId, right => right.ContentTypeId)
+                    .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax);
             }
 
-
+            xmlIdsQuery.Where(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax);
+            
             var allXmlIds = Database.Fetch(xmlIdsQuery);
 
             var toRemove = allXmlIds.Except(allMediaIds).ToArray();

From 55e84515136f904efb0016bc132c71d27dab09aa Mon Sep 17 00:00:00 2001
From: Shannon 
Date: Mon, 6 Mar 2017 14:44:41 +0100
Subject: [PATCH 38/41] changes to in groups of 2000

---
 .../Persistence/Repositories/ContentRepository.cs         | 8 +++++++-
 .../Persistence/Repositories/MediaRepository.cs           | 7 ++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
index 7d354a85d1..f95c295243 100644
--- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs
@@ -289,7 +289,13 @@ namespace Umbraco.Core.Persistence.Repositories
 
             var toRemove = allXmlIds.Except(allContentIds).ToArray();
             if (toRemove.Length > 0)
-                Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = toRemove });
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }                
+            }
+                
         }
 
         public override IEnumerable GetAllVersions(int id)
diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
index 155e9bae2f..4c09fc6a29 100644
--- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
+++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs
@@ -319,7 +319,12 @@ namespace Umbraco.Core.Persistence.Repositories
 
             var toRemove = allXmlIds.Except(allMediaIds).ToArray();
             if (toRemove.Length > 0)
-                Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = toRemove });
+            {
+                foreach (var idGroup in toRemove.InGroupsOf(2000))
+                {
+                    Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup });
+                }
+            }
         }
 
         public void AddOrUpdateContentXml(IMedia content, Func xml)

From b080825bb28a1d4bcbcb78e3bafae20b750fd4ad Mon Sep 17 00:00:00 2001
From: Michael Razmgah 
Date: Mon, 6 Mar 2017 21:31:42 +0100
Subject: [PATCH 39/41] Updating sv.xml with correct translation

---
 src/Umbraco.Web.UI/umbraco/config/lang/sv.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml b/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
index 259b914a72..24b9231e60 100644
--- a/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
+++ b/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
@@ -87,7 +87,7 @@
     Välj aktuell mapp
     Förhandsgranska
     Förhandsgranskning är avstängt på grund av att det inte finns någon mall tilldelad
-    Ångra
+    Annat
     Välj stil
     Visa stil
     Infoga tabell
@@ -936,4 +936,4 @@
     Översättare
     Din profil
   
-
\ No newline at end of file
+

From 8a133938e688ca23e3d327bd90b1d8a2e53ff93b Mon Sep 17 00:00:00 2001
From: nengberg 
Date: Wed, 8 Mar 2017 09:57:32 +0100
Subject: [PATCH 40/41] Adds missing swedish translation for document type
 without template

---
 src/Umbraco.Web.UI/umbraco/config/lang/sv.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml b/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
index 24b9231e60..aeaf8bd49b 100644
--- a/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
+++ b/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
@@ -171,6 +171,7 @@
     "dokumenttyper".]]>
     "mediatyper".]]>
     Välj typ och rubrik
+    Dokumenttyp utan sidmall
   
   
     Surfa på din webbplats

From b8c0d4c4ea83521abaae7b234292bbf04d594da1 Mon Sep 17 00:00:00 2001
From: Stephan 
Date: Thu, 9 Mar 2017 11:35:06 +0100
Subject: [PATCH 41/41] U4-9595 - improve log error msg on thread aborts on
 timeouts

---
 src/Umbraco.Core/Logging/Logger.cs | 31 ++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/Umbraco.Core/Logging/Logger.cs b/src/Umbraco.Core/Logging/Logger.cs
index ae8bb60fcd..66cad59733 100644
--- a/src/Umbraco.Core/Logging/Logger.cs
+++ b/src/Umbraco.Core/Logging/Logger.cs
@@ -2,6 +2,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Reflection;
 using System.Threading;
 using System.Web;
 using log4net;
@@ -62,12 +63,34 @@ namespace Umbraco.Core.Logging
         
 		public void Error(Type callingType, string message, Exception exception)
 		{
-			var logger = LogManager.GetLogger(callingType);
-			if (logger != null)
-				logger.Error((message), exception);
+            var logger = LogManager.GetLogger(callingType);
+		    if (logger == null) return;
+
+		    if (IsTimeoutThreadAbortException(exception))
+		    {
+		        message += "\r\nThe thread has been aborted, because the request has timed out.";
+		    }
+
+		    logger.Error(message, exception);
 		}
 
-		public void Warn(Type callingType, string message, params Func[] formatItems)
+        private static bool IsTimeoutThreadAbortException(Exception exception)
+        {
+            var abort = exception as ThreadAbortException;
+            if (abort == null) return false;
+
+            if (abort.ExceptionState == null) return false;
+
+            var stateType = abort.ExceptionState.GetType();
+            if (stateType.FullName != "System.Web.HttpApplication+CancelModuleException") return false;
+
+            var timeoutField = stateType.GetField("_timeout", BindingFlags.Instance | BindingFlags.NonPublic);
+            if (timeoutField == null) return false;
+
+            return (bool) timeoutField.GetValue(abort.ExceptionState);
+        }
+
+        public void Warn(Type callingType, string message, params Func[] formatItems)
 		{
 			var logger = LogManager.GetLogger(callingType);
 			if (logger == null || logger.IsWarnEnabled == false) return;