diff --git a/build/NuSpecs/tools/Dashboard.config.install.xdt b/build/NuSpecs/tools/Dashboard.config.install.xdt index 8368870186..036beeba29 100644 --- a/build/NuSpecs/tools/Dashboard.config.install.xdt +++ b/build/NuSpecs/tools/Dashboard.config.install.xdt @@ -41,16 +41,6 @@ views/dashboard/developer/examinemanagement.html - - - views/dashboard/developer/healthcheck.html - - - - - views/dashboard/developer/redirecturls.html - -
@@ -80,4 +70,26 @@
+ +
+ + content + + + + views/dashboard/developer/redirecturls.html + + +
+ +
+ + developer + + + + views/dashboard/developer/healthcheck.html + + +
\ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/ContentXmlStorage.cs b/src/Umbraco.Core/Configuration/ContentXmlStorage.cs new file mode 100644 index 0000000000..7cbbc70675 --- /dev/null +++ b/src/Umbraco.Core/Configuration/ContentXmlStorage.cs @@ -0,0 +1,9 @@ +namespace Umbraco.Core.Configuration +{ + internal enum ContentXmlStorage + { + Default, + AspNetTemp, + EnvironmentTemp + } +} \ No newline at end of file diff --git a/src/Umbraco.Core/Configuration/GlobalSettings.cs b/src/Umbraco.Core/Configuration/GlobalSettings.cs index c28f398333..acbf0065c0 100644 --- a/src/Umbraco.Core/Configuration/GlobalSettings.cs +++ b/src/Umbraco.Core/Configuration/GlobalSettings.cs @@ -517,12 +517,25 @@ namespace Umbraco.Core.Configuration } internal static bool ContentCacheXmlStoredInCodeGen + { + get { return ContentCacheXmlStorageLocation == ContentXmlStorage.AspNetTemp; } + } + + internal static ContentXmlStorage ContentCacheXmlStorageLocation { get { - //defaults to false - return ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp") - && bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]); //default to false + if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLStorage")) + { + return Enum.Parse(ConfigurationManager.AppSettings["umbracoContentXMLStorage"]); + } + if (ConfigurationManager.AppSettings.ContainsKey("umbracoContentXMLUseLocalTemp")) + { + return bool.Parse(ConfigurationManager.AppSettings["umbracoContentXMLUseLocalTemp"]) + ? ContentXmlStorage.AspNetTemp + : ContentXmlStorage.Default; + } + return ContentXmlStorage.Default; } } diff --git a/src/Umbraco.Core/IO/SystemFiles.cs b/src/Umbraco.Core/IO/SystemFiles.cs index 48bdea2884..437ddd3ef7 100644 --- a/src/Umbraco.Core/IO/SystemFiles.cs +++ b/src/Umbraco.Core/IO/SystemFiles.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Configuration; using System.IO; using System.Linq; @@ -72,15 +73,28 @@ namespace Umbraco.Core.IO { get { - if (GlobalSettings.ContentCacheXmlStoredInCodeGen && SystemUtilities.GetCurrentTrustLevel() == AspNetHostingPermissionLevel.Unrestricted) - { - return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config"); + switch (GlobalSettings.ContentCacheXmlStorageLocation) + { + case ContentXmlStorage.AspNetTemp: + return Path.Combine(HttpRuntime.CodegenDir, @"UmbracoData\umbraco.config"); + case ContentXmlStorage.EnvironmentTemp: + var appDomainHash = HttpRuntime.AppDomainAppId.ToSHA1(); + var cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "UmbracoXml", + //include the appdomain hash is just a safety check, for example if a website is moved from worker A to worker B and then back + // to worker A again, in theory the %temp% folder should already be empty but we really want to make sure that its not + // utilizing an old path + appDomainHash); + return Path.Combine(cachePath, "umbraco.config"); + case ContentXmlStorage.Default: + return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config"); + default: + throw new ArgumentOutOfRangeException(); } - return IOHelper.ReturnPath("umbracoContentXML", "~/App_Data/umbraco.config"); } } [Obsolete("Use GlobalSettings.ContentCacheXmlStoredInCodeGen instead")] + [EditorBrowsable(EditorBrowsableState.Never)] internal static bool ContentCacheXmlStoredInCodeGen { get { return GlobalSettings.ContentCacheXmlStoredInCodeGen; } diff --git a/src/Umbraco.Core/Logging/Logger.cs b/src/Umbraco.Core/Logging/Logger.cs index ae8bb60fcd..66cad59733 100644 --- a/src/Umbraco.Core/Logging/Logger.cs +++ b/src/Umbraco.Core/Logging/Logger.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; using System.Threading; using System.Web; using log4net; @@ -62,12 +63,34 @@ namespace Umbraco.Core.Logging public void Error(Type callingType, string message, Exception exception) { - var logger = LogManager.GetLogger(callingType); - if (logger != null) - logger.Error((message), exception); + var logger = LogManager.GetLogger(callingType); + if (logger == null) return; + + if (IsTimeoutThreadAbortException(exception)) + { + message += "\r\nThe thread has been aborted, because the request has timed out."; + } + + logger.Error(message, exception); } - public void Warn(Type callingType, string message, params Func[] formatItems) + private static bool IsTimeoutThreadAbortException(Exception exception) + { + var abort = exception as ThreadAbortException; + if (abort == null) return false; + + if (abort.ExceptionState == null) return false; + + var stateType = abort.ExceptionState.GetType(); + if (stateType.FullName != "System.Web.HttpApplication+CancelModuleException") return false; + + var timeoutField = stateType.GetField("_timeout", BindingFlags.Instance | BindingFlags.NonPublic); + if (timeoutField == null) return false; + + return (bool) timeoutField.GetValue(abort.ExceptionState); + } + + public void Warn(Type callingType, string message, params Func[] formatItems) { var logger = LogManager.GetLogger(callingType); if (logger == null || logger.IsWarnEnabled == false) return; diff --git a/src/Umbraco.Core/Models/PropertyTypeCollection.cs b/src/Umbraco.Core/Models/PropertyTypeCollection.cs index 38abd0c57d..a06e6d737d 100644 --- a/src/Umbraco.Core/Models/PropertyTypeCollection.cs +++ b/src/Umbraco.Core/Models/PropertyTypeCollection.cs @@ -69,6 +69,7 @@ namespace Umbraco.Core.Models OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } + //TODO: Instead of 'new' this should explicitly implement one of the collection interfaces members internal new void Add(PropertyType item) { using (new WriteLock(_addLocker)) diff --git a/src/Umbraco.Core/Models/PublicAccessEntry.cs b/src/Umbraco.Core/Models/PublicAccessEntry.cs index 1ed15a8fb4..5989f885cb 100644 --- a/src/Umbraco.Core/Models/PublicAccessEntry.cs +++ b/src/Umbraco.Core/Models/PublicAccessEntry.cs @@ -110,6 +110,12 @@ namespace Umbraco.Core.Models _ruleCollection.Clear(); } + + internal void ClearRemovedRules() + { + _removedRules.Clear(); + } + [DataMember] public int LoginNodeId { diff --git a/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs b/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs index 2901f48539..7b28808429 100644 --- a/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs +++ b/src/Umbraco.Core/Persistence/Factories/MemberFactory.cs @@ -28,17 +28,17 @@ namespace Umbraco.Core.Persistence.Factories #region Implementation of IEntityFactory - public IMember BuildEntity(MemberDto dto) + public static IMember BuildEntity(MemberDto dto, IMemberType contentType) { var member = new Member( dto.ContentVersionDto.ContentDto.NodeDto.Text, - dto.Email, dto.LoginName, dto.Password, _contentType); + dto.Email, dto.LoginName, dto.Password, contentType); try { member.DisableChangeTracking(); - member.Id = _id; + member.Id = dto.NodeId; member.Key = dto.ContentVersionDto.ContentDto.NodeDto.UniqueId; member.Path = dto.ContentVersionDto.ContentDto.NodeDto.Path; member.CreatorId = dto.ContentVersionDto.ContentDto.NodeDto.UserId.Value; @@ -62,6 +62,12 @@ namespace Umbraco.Core.Persistence.Factories } } + [Obsolete("Use the static BuildEntity instead so we don't have to allocate one of these objects everytime we want to map values")] + public IMember BuildEntity(MemberDto dto) + { + return BuildEntity(dto, _contentType); + } + public MemberDto BuildDto(IMember entity) { var dto = new MemberDto diff --git a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs index 4eb3fe0659..ded7c60676 100644 --- a/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs +++ b/src/Umbraco.Core/Persistence/Factories/UmbracoEntityFactory.cs @@ -6,6 +6,7 @@ using System.Reflection; using Umbraco.Core.Models; using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Persistence.Repositories; +using Umbraco.Core.Strings; namespace Umbraco.Core.Persistence.Factories { @@ -17,7 +18,7 @@ namespace Umbraco.Core.Persistence.Factories //figure out what extra properties we have that are not on the IUmbracoEntity and add them to additional data foreach (var k in originalEntityProperties.Keys - .Select(x => new { orig = x, title = x.ConvertCase(StringAliasCaseType.PascalCase) }) + .Select(x => new { orig = x, title = x.ToCleanString(CleanStringType.PascalCase | CleanStringType.Ascii | CleanStringType.ConvertCase) }) .Where(x => entityProps.InvariantContains(x.title) == false)) { entity.AdditionalData[k.title] = originalEntityProperties[k.orig]; @@ -75,65 +76,6 @@ namespace Umbraco.Core.Persistence.Factories entity.EnableChangeTracking(); } } - - public UmbracoEntity BuildEntity(EntityRepository.UmbracoEntityDto dto) - { - var entity = new UmbracoEntity(dto.Trashed) - { - CreateDate = dto.CreateDate, - CreatorId = dto.UserId.Value, - Id = dto.NodeId, - Key = dto.UniqueId, - Level = dto.Level, - Name = dto.Text, - NodeObjectTypeId = dto.NodeObjectType.Value, - ParentId = dto.ParentId, - Path = dto.Path, - SortOrder = dto.SortOrder, - HasChildren = dto.Children > 0, - ContentTypeAlias = dto.Alias ?? string.Empty, - ContentTypeIcon = dto.Icon ?? string.Empty, - ContentTypeThumbnail = dto.Thumbnail ?? string.Empty, - }; - - entity.IsPublished = dto.PublishedVersion != default(Guid) || (dto.NewestVersion != default(Guid) && dto.PublishedVersion == dto.NewestVersion); - entity.IsDraft = dto.NewestVersion != default(Guid) && (dto.PublishedVersion == default(Guid) || dto.PublishedVersion != dto.NewestVersion); - entity.HasPendingChanges = (dto.PublishedVersion != default(Guid) && dto.NewestVersion != default(Guid)) && dto.PublishedVersion != dto.NewestVersion; - - if (dto.UmbracoPropertyDtos != null) - { - foreach (var propertyDto in dto.UmbracoPropertyDtos) - { - entity.AdditionalData[propertyDto.PropertyAlias] = new UmbracoEntity.EntityProperty - { - PropertyEditorAlias = propertyDto.PropertyEditorAlias, - Value = propertyDto.NTextValue.IsNullOrWhiteSpace() - ? propertyDto.NVarcharValue - : propertyDto.NTextValue.ConvertToJsonIfPossible() - }; - } - } - - return entity; - } - - public EntityRepository.UmbracoEntityDto BuildDto(UmbracoEntity entity) - { - var node = new EntityRepository.UmbracoEntityDto - { - CreateDate = entity.CreateDate, - Level = short.Parse(entity.Level.ToString(CultureInfo.InvariantCulture)), - NodeId = entity.Id, - NodeObjectType = entity.NodeObjectTypeId, - ParentId = entity.ParentId, - Path = entity.Path, - SortOrder = entity.SortOrder, - Text = entity.Name, - Trashed = entity.Trashed, - UniqueId = entity.Key, - UserId = entity.CreatorId - }; - return node; - } + } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs b/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs index 6909c77744..a4dee60ee9 100644 --- a/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs +++ b/src/Umbraco.Core/Persistence/Mappers/MappingResolver.cs @@ -31,7 +31,7 @@ namespace Umbraco.Core.Persistence.Mappers /// /// /// - internal BaseMapper ResolveMapperByType(Type type) + public virtual BaseMapper ResolveMapperByType(Type type) { return _mapperCache.GetOrAdd(type, type1 => { @@ -67,7 +67,7 @@ namespace Umbraco.Core.Persistence.Mappers return Attempt.Succeed(mapper); } - internal string GetMapping(Type type, string propertyName) + public virtual string GetMapping(Type type, string propertyName) { var mapper = ResolveMapperByType(type); var result = mapper.Map(propertyName); diff --git a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs index 9ebb3d17b0..fe73638b4b 100644 --- a/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs +++ b/src/Umbraco.Core/Persistence/Querying/ExpressionVisitorBase.cs @@ -546,7 +546,7 @@ namespace Umbraco.Core.Persistence.Querying && methodArgs.Length == 1 && methodArgs[0].NodeType == ExpressionType.MemberAccess && TypeHelper.IsTypeAssignableFrom(m.Arguments[0].Type)) - { + { goto case "SqlIn"; } diff --git a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs index 7f5e479af6..da1f93a032 100644 --- a/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs +++ b/src/Umbraco.Core/Persistence/Querying/ModelToSqlExpressionVisitor.cs @@ -1,5 +1,6 @@ using System; using System.Linq.Expressions; +using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Persistence.Mappers; using Umbraco.Core.Persistence.SqlSyntax; @@ -12,17 +13,19 @@ namespace Umbraco.Core.Persistence.Querying /// This object is stateful and cannot be re-used to parse an expression. internal class ModelToSqlExpressionVisitor : ExpressionVisitorBase { + private readonly MappingResolver _mappingResolver; private readonly BaseMapper _mapper; - public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, BaseMapper mapper) + public ModelToSqlExpressionVisitor(ISqlSyntaxProvider sqlSyntax, MappingResolver mappingResolver) : base(sqlSyntax) { - _mapper = mapper; + _mapper = mappingResolver.ResolveMapperByType(typeof(T)); + _mappingResolver = mappingResolver; } [Obsolete("Use the overload the specifies a SqlSyntaxProvider")] public ModelToSqlExpressionVisitor() - : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current.ResolveMapperByType(typeof(T))) + : this(SqlSyntaxContext.SqlSyntaxProvider, MappingResolver.Current) { } protected override string VisitMemberAccess(MemberExpression m) @@ -36,7 +39,7 @@ namespace Umbraco.Core.Persistence.Querying { var field = _mapper.Map(m.Member.Name, true); if (field.IsNullOrWhiteSpace()) - throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name); + throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type)); return field; } //already compiled, return @@ -50,13 +53,42 @@ namespace Umbraco.Core.Persistence.Querying { var field = _mapper.Map(m.Member.Name, true); if (field.IsNullOrWhiteSpace()) - throw new InvalidOperationException("The mapper returned an empty field for the member name: " + m.Member.Name); + throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type)); return field; } //already compiled, return return string.Empty; } + if (m.Expression != null + && m.Expression.Type != typeof(T) + && TypeHelper.IsTypeAssignableFrom(m.Expression.Type) + && EndsWithConstant(m) == false) + { + //if this is the case, it means we have a sub expression / nested property access, such as: x.ContentType.Alias == "Test"; + //and since the sub type (x.ContentType) is not the same as x, we need to resolve a mapper for x.ContentType to get it's mapped SQL column + + //don't execute if compiled + if (Visited == false) + { + var subMapper = _mappingResolver.ResolveMapperByType(m.Expression.Type); + if (subMapper == null) + throw new NullReferenceException("No mapper found for type " + m.Expression.Type); + var field = subMapper.Map(m.Member.Name, true); + if (field.IsNullOrWhiteSpace()) + throw new InvalidOperationException(string.Format("The mapper returned an empty field for the member name: {0} for type: {1}", m.Member.Name, m.Expression.Type)); + return field; + } + //already compiled, return + return string.Empty; + } + + //TODO: When m.Expression.NodeType == ExpressionType.Constant and it's an expression like: content => aliases.Contains(content.ContentType.Alias); + // then an SQL parameter will be added for aliases as an array, however in SqlIn on the subclass it will manually add these SqlParameters anyways, + // however the query will still execute because the SQL that is written will only contain the correct indexes of SQL parameters, this would be ignored, + // I'm just unsure right now due to time constraints how to make it correct. It won't matter right now and has been working already with this bug but I've + // only just discovered what it is actually doing. + var member = Expression.Convert(m, typeof(object)); var lambda = Expression.Lambda>(member); var getter = lambda.Compile(); @@ -71,5 +103,24 @@ namespace Umbraco.Core.Persistence.Querying return string.Empty; } + + /// + /// Determines if the MemberExpression ends in a Constant value + /// + /// + /// + private bool EndsWithConstant(MemberExpression m) + { + Expression expr = m; + + while (expr is MemberExpression) + { + var memberExpr = expr as MemberExpression; + expr = memberExpr.Expression; + } + + var constExpr = expr as ConstantExpression; + return constExpr != null; + } } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs b/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs index 5a0452c3aa..593955734e 100644 --- a/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs +++ b/src/Umbraco.Core/Persistence/Querying/SqlExpressionExtensions.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs index 37e5e80fe9..ae656b3f5b 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentRepository.cs @@ -64,7 +64,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateContentFromDto(dto, dto.ContentVersionDto.VersionId, sql); + var content = CreateContentFromDto(dto, sql); return content; } @@ -134,6 +134,9 @@ namespace Umbraco.Core.Persistence.Repositories .On(SqlSyntax, left => left.NodeId, right => right.NodeId) .InnerJoin(SqlSyntax) .On(SqlSyntax, left => left.NodeId, right => right.NodeId); + //TODO: IF we want to enable querying on content type information this will need to be joined + //.InnerJoin(SqlSyntax) + //.On(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax); if (queryType == BaseQueryType.FullSingle) { @@ -260,6 +263,39 @@ namespace Umbraco.Core.Persistence.Repositories } baseId = xmlItems[xmlItems.Count - 1].NodeId; } + + //now delete the items that shouldn't be there + var sqlAllIds = translate(0, GetBaseQuery(BaseQueryType.Ids)); + var allContentIds = Database.Fetch(sqlAllIds); + var docObjectType = Guid.Parse(Constants.ObjectTypes.Document); + var xmlIdsQuery = new Sql() + .Select("DISTINCT cmsContentXml.nodeId") + .From(SqlSyntax) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId); + + if (contentTypeIdsA.Length > 0) + { + xmlIdsQuery.InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.ContentTypeId) + .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); + } + + xmlIdsQuery.Where(dto => dto.NodeObjectType == docObjectType, SqlSyntax); + + var allXmlIds = Database.Fetch(xmlIdsQuery); + + var toRemove = allXmlIds.Except(allContentIds).ToArray(); + if (toRemove.Length > 0) + { + foreach (var idGroup in toRemove.InGroupsOf(2000)) + { + Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup }); + } + } + } public override IEnumerable GetAllVersions(int id) @@ -273,12 +309,13 @@ namespace Umbraco.Core.Persistence.Repositories var sqlFull = translate(GetBaseQuery(BaseQueryType.FullMultiple)); var sqlIds = translate(GetBaseQuery(BaseQueryType.Ids)); - return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true); + return ProcessQuery(sqlFull, new PagingSqlQuery(sqlIds), true, includeAllVersions:true); } public override IContent GetByVersion(Guid versionId) { var sql = GetBaseQuery(BaseQueryType.FullSingle); + //TODO: cmsContentVersion.VersionId has a Unique Index constraint applied, seems silly then to also add OrderByDescending since it would be impossible to return more than one. sql.Where("cmsContentVersion.VersionId = @VersionId", new { VersionId = versionId }); sql.OrderByDescending(x => x.VersionDate, SqlSyntax); @@ -287,7 +324,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateContentFromDto(dto, versionId, sql); + var content = CreateContentFromDto(dto, sql); return content; } @@ -297,7 +334,8 @@ namespace Umbraco.Core.Persistence.Repositories var sql = new Sql() .Select("*") .From(SqlSyntax) - .InnerJoin(SqlSyntax).On(SqlSyntax, left => left.VersionId, right => right.VersionId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.VersionId, right => right.VersionId) .Where(x => x.VersionId == versionId, SqlSyntax) .Where(x => x.Newest != true, SqlSyntax); var dto = Database.Fetch(sql).FirstOrDefault(); @@ -317,7 +355,8 @@ namespace Umbraco.Core.Persistence.Repositories var sql = new Sql() .Select("*") .From() - .InnerJoin().On(left => left.VersionId, right => right.VersionId) + .InnerJoin() + .On(left => left.VersionId, right => right.VersionId) .Where(x => x.NodeId == id) .Where(x => x.VersionDate < versionDate) .Where(x => x.Newest != true); @@ -558,6 +597,7 @@ namespace Umbraco.Core.Persistence.Repositories //if (((ICanBeDirty)entity).IsPropertyDirty("Published") && (entity.Published || publishedState == PublishedState.Unpublished)) if (entity.ShouldClearPublishedFlagForPreviousVersions(publishedState, shouldCreateNewVersion)) { + //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many) var publishedDocs = Database.Fetch("WHERE nodeId = @Id AND published = @IsPublished", new { Id = entity.Id, IsPublished = true }); foreach (var doc in publishedDocs) { @@ -571,6 +611,7 @@ namespace Umbraco.Core.Persistence.Repositories } //Look up (newest) entries by id in cmsDocument table to set newest = false + //TODO: This perf can be improved, it could easily be UPDATE WHERE.... (one SQL call instead of many) var documentDtos = Database.Fetch("WHERE nodeId = @Id AND newest = @IsNewest", new { Id = entity.Id, IsNewest = true }); foreach (var documentDto in documentDtos) { @@ -898,7 +939,7 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", return base.GetDatabaseFieldNameForOrderBy(orderBy); } - + /// /// This is the underlying method that processes most queries for this repository /// @@ -909,8 +950,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", /// The Id SQL without the outer join to just return all document ids - used to process the properties for the content item /// /// + /// + /// Generally when querying for content we only want to return the most recent version of the content item, however in some cases like when + /// we want to return all versions of a content item, we can't simply return the latest + /// /// - private IEnumerable ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false) + private IEnumerable ProcessQuery(Sql sqlFull, PagingSqlQuery pagingSqlQuery, bool withCache = false, bool includeAllVersions = false) { // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); @@ -929,13 +974,12 @@ order by umbracoNode.{2}, umbracoNode.parentID, umbracoNode.sortOrder", parsedOriginalSql = parsedOriginalSql.Substring(0, parsedOriginalSql.LastIndexOf("ORDER BY ", StringComparison.Ordinal)); } - var publishedSql = new Sql(@"SELECT * -FROM cmsDocument AS doc2 -INNER JOIN - (" + parsedOriginalSql + @") as docData -ON doc2.nodeId = docData.nodeId -WHERE doc2.published = 1 -ORDER BY doc2.nodeId + //order by update date DESC, if there is corrupted published flags we only want the latest! + var publishedSql = new Sql(@"SELECT cmsDocument.nodeId, cmsDocument.published, cmsDocument.versionId, cmsDocument.newest +FROM cmsDocument INNER JOIN cmsContentVersion ON cmsContentVersion.VersionId = cmsDocument.versionId +WHERE cmsDocument.published = 1 AND cmsDocument.nodeId IN +(" + parsedOriginalSql + @") +ORDER BY cmsContentVersion.id DESC ", sqlFull.Arguments); //go and get the published version data, we do a Query here and not a Fetch so we are @@ -950,9 +994,9 @@ ORDER BY doc2.nodeId publishedDataCollection.Add(publishedDto); } - - var content = new IContent[dtos.Count]; - var defs = new List(); + //This is a tuple list identifying if the content item came from the cache or not + var content = new List>(); + var defs = new DocumentDefinitionCollection(includeAllVersions); var templateIds = new List(); //track the looked up content types, even though the content types are cached @@ -960,9 +1004,8 @@ ORDER BY doc2.nodeId // the overhead of deep cloning them on every item in this loop var contentTypes = new Dictionary(); - for (var i = 0; i < dtos.Count; i++) + foreach (var dto in dtos) { - var dto = dtos[i]; DocumentPublishedReadOnlyDto publishedDto; publishedDataCollection.TryGetValue(dto.NodeId, out publishedDto); @@ -970,10 +1013,10 @@ ORDER BY doc2.nodeId if (withCache) { var cached = IsolatedCache.GetCacheItem(GetCacheIdKey(dto.NodeId)); - //only use this cached version if the dto returned is also the publish version, they must match - if (cached != null && cached.Published && dto.Published) + //only use this cached version if the dto returned is also the publish version, they must match and be teh same version + if (cached != null && cached.Version == dto.VersionId && cached.Published && dto.Published) { - content[i] = cached; + content.Add(new Tuple(cached, true)); continue; } } @@ -991,21 +1034,16 @@ ORDER BY doc2.nodeId contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); contentTypes[dto.ContentVersionDto.ContentDto.ContentTypeId] = contentType; } - - content[i] = ContentFactory.BuildEntity(dto, contentType, publishedDto); - // need template - if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0) - templateIds.Add(dto.TemplateId.Value); + // track the definition and if it's successfully added or updated then processed + if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType))) + { + // assign template + if (dto.TemplateId.HasValue && dto.TemplateId.Value > 0) + templateIds.Add(dto.TemplateId.Value); - // need properties - defs.Add(new DocumentDefinition( - dto.NodeId, - dto.VersionId, - dto.ContentVersionDto.VersionDate, - dto.ContentVersionDto.ContentDto.NodeDto.CreateDate, - contentType - )); + content.Add(new Tuple(ContentFactory.BuildEntity(dto, contentType, publishedDto), false)); + } } // load all required templates in 1 query @@ -1015,38 +1053,38 @@ ORDER BY doc2.nodeId // load all properties for all documents from database in 1 query var propertyData = GetPropertyCollection(pagingSqlQuery, defs); - // assign - var dtoIndex = 0; - foreach (var def in defs) + // assign template and property data + foreach (var contentItem in content) { - // move to corresponding item (which has to exist) - while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++; + var cc = contentItem.Item1; + var fromCache = contentItem.Item2; + + //if this has come from cache, we do not need to build up it's structure + if (fromCache) continue; + + var def = defs[includeAllVersions ? (ValueType)cc.Version : cc.Id]; - // complete the item - var cc = content[dtoIndex]; - var dto = dtos[dtoIndex]; ITemplate template = null; - if (dto.TemplateId.HasValue) - templates.TryGetValue(dto.TemplateId.Value, out template); // else null + if (def.DocumentDto.TemplateId.HasValue) + templates.TryGetValue(def.DocumentDto.TemplateId.Value, out template); // else null cc.Template = template; - cc.Properties = propertyData[cc.Id]; + cc.Properties = propertyData[cc.Version]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 cc.ResetDirtyProperties(false); } - return content; + return content.Select(x => x.Item1).ToArray(); } /// /// Private method to create a content object from a DocumentDto, which is used by Get and GetByVersion. /// /// - /// /// /// - private IContent CreateContentFromDto(DocumentDto dto, Guid versionId, Sql docSql) + private IContent CreateContentFromDto(DocumentDto dto, Sql docSql) { var contentType = _contentTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); @@ -1058,11 +1096,11 @@ ORDER BY doc2.nodeId content.Template = _templateRepository.Get(dto.TemplateId.Value); } - var docDef = new DocumentDefinition(dto.NodeId, versionId, content.UpdateDate, content.CreateDate, contentType); + var docDef = new DocumentDefinition(dto, contentType); var properties = GetPropertyCollection(docSql, new[] { docDef }); - content.Properties = properties[dto.NodeId]; + content.Properties = properties[dto.VersionId]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 diff --git a/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs index b440d477a5..985f9446b7 100644 --- a/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/ContentTypeRepository.cs @@ -104,7 +104,7 @@ namespace Umbraco.Core.Persistence.Repositories if (objectTypes.Any()) { - sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", objectTypes); + sql = sql.Where("umbracoNode.nodeObjectType IN (@objectTypes)", new {objectTypes = objectTypes}); } return Database.Fetch(sql); diff --git a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs index eb372eda56..a5bf9c1807 100644 --- a/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/EntityRepository.cs @@ -1,19 +1,14 @@ using System; using System.Collections.Generic; -using System.Dynamic; -using System.Globalization; +using System.Collections.ObjectModel; using System.Linq; -using System.Reflection; -using System.Text; using Umbraco.Core.Models; -using Umbraco.Core; using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Factories; using Umbraco.Core.Persistence.Querying; using Umbraco.Core.Persistence.UnitOfWork; -using Umbraco.Core.Strings; namespace Umbraco.Core.Persistence.Repositories { @@ -187,10 +182,12 @@ namespace Umbraco.Core.Persistence.Repositories bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media); var sql = GetFullSqlForEntityType(key, isContent, isMedia, objectTypeId); - + + var factory = new UmbracoEntityFactory(); + if (isMedia) { - //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! + //for now treat media differently and include all property data too var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, sql); @@ -198,14 +195,16 @@ namespace Umbraco.Core.Persistence.Repositories } else { - var nodeDto = _work.Database.FirstOrDefault(sql); - if (nodeDto == null) - return null; - var factory = new UmbracoEntityFactory(); - var entity = factory.BuildEntityFromDynamic(nodeDto); - - return entity; + //query = read forward data reader, do not load everything into mem + var dtos = _work.Database.Query(sql); + var collection = new EntityDefinitionCollection(); + foreach (var dto in dtos) + { + collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false)); + } + var found = collection.FirstOrDefault(); + return found != null ? found.BuildFromDynamic() : null; } @@ -230,28 +229,29 @@ namespace Umbraco.Core.Persistence.Repositories bool isMedia = objectTypeId == new Guid(Constants.ObjectTypes.Media); var sql = GetFullSqlForEntityType(id, isContent, isMedia, objectTypeId); - + + var factory = new UmbracoEntityFactory(); + if (isMedia) { - //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! + //for now treat media differently and include all property data too var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, sql); return entities.FirstOrDefault(); } else - { - var nodeDto = _work.Database.FirstOrDefault(sql); - if (nodeDto == null) - return null; - - var factory = new UmbracoEntityFactory(); - var entity = factory.BuildEntityFromDynamic(nodeDto); - - return entity; + { + //query = read forward data reader, do not load everything into mem + var dtos = _work.Database.Query(sql); + var collection = new EntityDefinitionCollection(); + foreach (var dto in dtos) + { + collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false)); + } + var found = collection.FirstOrDefault(); + return found != null ? found.BuildFromDynamic() : null; } - - } public virtual IEnumerable GetAll(Guid objectTypeId, params int[] ids) @@ -288,21 +288,21 @@ namespace Umbraco.Core.Persistence.Repositories if (isMedia) { - //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! + //for now treat media differently and include all property data too var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, sql); - foreach (var entity in entities) - { - yield return entity; - } + return entities; } else { - var dtos = _work.Database.Fetch(sql); - foreach (var entity in dtos.Select(dto => factory.BuildEntityFromDynamic(dto))) + //query = read forward data reader, do not load everything into mem + var dtos = _work.Database.Query(sql); + var collection = new EntityDefinitionCollection(); + foreach (var dto in dtos) { - yield return entity; + collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false)); } + return collection.Select(x => x.BuildFromDynamic()).ToList(); } } @@ -347,7 +347,7 @@ namespace Umbraco.Core.Persistence.Repositories } }); - //Treat media differently for now, as an Entity it will be returned with ALL of it's properties in the AdditionalData bag! + //for now treat media differently and include all property data too var entities = _work.Database.Fetch( new UmbracoEntityRelator().Map, mediaSql); return entities; @@ -356,8 +356,15 @@ namespace Umbraco.Core.Persistence.Repositories { //use dynamic so that we can get ALL properties from the SQL so we can chuck that data into our AdditionalData var finalSql = entitySql.Append(GetGroupBy(isContent, false)); - var dtos = _work.Database.Fetch(finalSql); - return dtos.Select(factory.BuildEntityFromDynamic).Cast().ToList(); + + //query = read forward data reader, do not load everything into mem + var dtos = _work.Database.Query(finalSql); + var collection = new EntityDefinitionCollection(); + foreach (var dto in dtos) + { + collection.AddOrUpdate(new EntityDefinition(factory, dto, isContent, false)); + } + return collection.Select(x => x.BuildFromDynamic()).ToList(); } } @@ -449,30 +456,31 @@ namespace Umbraco.Core.Persistence.Repositories else { columns.AddRange(new List - { - "umbracoNode.id", - "umbracoNode.trashed", - "umbracoNode.parentID", - "umbracoNode.nodeUser", - "umbracoNode.level", - "umbracoNode.path", - "umbracoNode.sortOrder", - "umbracoNode.uniqueID", - "umbracoNode.text", - "umbracoNode.nodeObjectType", - "umbracoNode.createDate", - "COUNT(parent.parentID) as children" - }); + { + "umbracoNode.id", + "umbracoNode.trashed", + "umbracoNode.parentID", + "umbracoNode.nodeUser", + "umbracoNode.level", + "umbracoNode.path", + "umbracoNode.sortOrder", + "umbracoNode.uniqueID", + "umbracoNode.text", + "umbracoNode.nodeObjectType", + "umbracoNode.createDate", + "COUNT(parent.parentID) as children" + }; - if (isContent || isMedia) + if (isContent || isMedia) + { + if (isContent) { - if (isContent) - { - //only content has this info - columns.Add("published.versionId as publishedVersion"); - columns.Add("document.versionId as newestVersion"); - } - + //only content has/needs this info + columns.Add("published.versionId as publishedVersion"); + columns.Add("document.versionId as newestVersion"); + columns.Add("contentversion.id as versionId"); + } + columns.Add("contenttype.alias"); columns.Add("contenttype.icon"); columns.Add("contenttype.thumbnail"); @@ -485,20 +493,24 @@ namespace Umbraco.Core.Persistence.Repositories var entitySql = new Sql() .Select(columns.ToArray()) .From("umbracoNode umbracoNode"); - + if (isContent || isMedia) { entitySql.InnerJoin("cmsContent content").On("content.nodeId = umbracoNode.id"); if (isContent) { - //only content has this info - entitySql + //only content has/needs this info + entitySql .InnerJoin("cmsDocument document").On("document.nodeId = umbracoNode.id") + .InnerJoin("cmsContentVersion contentversion").On("contentversion.VersionId = document.versionId") .LeftJoin("(SELECT nodeId, versionId FROM cmsDocument WHERE published = 1) as published") .On("umbracoNode.id = published.nodeId"); } + entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType"); + } + entitySql.LeftJoin("cmsContentType contenttype").On("contenttype.nodeId = content.contentType"); } @@ -546,7 +558,7 @@ namespace Umbraco.Core.Persistence.Repositories protected virtual Sql GetBaseWhere(Func, Sql> baseQuery, bool isContent, bool isMedia, Guid key) { var sql = baseQuery(isContent, isMedia, null) - .Where("umbracoNode.uniqueID = @UniqueID", new { UniqueID = key }); + .Where("umbracoNode.uniqueID = @UniqueID", new {UniqueID = key}); if (isContent) { @@ -609,13 +621,14 @@ namespace Umbraco.Core.Persistence.Repositories { columns.Add("published.versionId"); columns.Add("document.versionId"); + columns.Add("contentversion.id"); } columns.Add("contenttype.alias"); columns.Add("contenttype.icon"); columns.Add("contenttype.thumbnail"); - columns.Add("contenttype.isContainer"); + columns.Add("contenttype.isContainer"); } - + var sql = new Sql() .GroupBy(columns.ToArray()); @@ -626,7 +639,7 @@ namespace Umbraco.Core.Persistence.Repositories return sql; } - + #endregion /// @@ -652,33 +665,7 @@ namespace Umbraco.Core.Persistence.Repositories return _work.Database.ExecuteScalar(sql) > 0; } - #region umbracoNode POCO - Extends NodeDto - [TableName("umbracoNode")] - [PrimaryKey("id")] - [ExplicitColumns] - internal class UmbracoEntityDto : NodeDto - { - [Column("children")] - public int Children { get; set; } - - [Column("publishedVersion")] - public Guid PublishedVersion { get; set; } - - [Column("newestVersion")] - public Guid NewestVersion { get; set; } - - [Column("alias")] - public string Alias { get; set; } - - [Column("icon")] - public string Icon { get; set; } - - [Column("thumbnail")] - public string Thumbnail { get; set; } - - [ResultColumn] - public List UmbracoPropertyDtos { get; set; } - } + #region private classes [ExplicitColumns] internal class UmbracoPropertyDto @@ -762,6 +749,99 @@ namespace Umbraco.Core.Persistence.Repositories return prev; } } + + private class EntityDefinitionCollection : KeyedCollection + { + protected override int GetKeyForItem(EntityDefinition item) + { + return item.Id; + } + + /// + /// if this key already exists if it does then we need to check + /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one + /// + /// + /// + public bool AddOrUpdate(EntityDefinition item) + { + if (Dictionary == null) + { + base.Add(item); + return true; + } + + var key = GetKeyForItem(item); + EntityDefinition found; + if (TryGetValue(key, out found)) + { + //it already exists and it's older so we need to replace it + if (item.VersionId > found.VersionId) + { + var currIndex = Items.IndexOf(found); + if (currIndex == -1) + throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Id); + + //replace the current one with the newer one + SetItem(currIndex, item); + return true; + } + //could not add or update + return false; + } + + base.Add(item); + return true; + } + + private bool TryGetValue(int key, out EntityDefinition val) + { + if (Dictionary == null) + { + val = null; + return false; + } + return Dictionary.TryGetValue(key, out val); + } + } + + private class EntityDefinition + { + private readonly UmbracoEntityFactory _factory; + private readonly dynamic _entity; + private readonly bool _isContent; + private readonly bool _isMedia; + + public EntityDefinition(UmbracoEntityFactory factory, dynamic entity, bool isContent, bool isMedia) + { + _factory = factory; + _entity = entity; + _isContent = isContent; + _isMedia = isMedia; + } + + public IUmbracoEntity BuildFromDynamic() + { + return _factory.BuildEntityFromDynamic(_entity); + } + + public int Id + { + get { return _entity.id; } + } + + public int VersionId + { + get + { + if (_isContent || _isMedia) + { + return _entity.versionId; + } + return _entity.id; + } + } + } #endregion } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs index df8f880ea2..ea3194cb4d 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Interfaces/IRepositoryVersionable.cs @@ -23,7 +23,7 @@ namespace Umbraco.Core.Persistence.Repositories /// The serializer to convert TEntity to Xml /// Structures will be rebuilt in chunks of this size /// - void RebuildXmlStructures(Func serializer, int groupSize = 5000, IEnumerable contentTypeIds = null); + void RebuildXmlStructures(Func serializer, int groupSize = 200, IEnumerable contentTypeIds = null); /// /// Get the total count of entities diff --git a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs index 4b7afe9000..c9f06c6c75 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MediaRepository.cs @@ -55,7 +55,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateMediaFromDto(dto, dto.VersionId, sql); + var content = CreateMediaFromDto(dto, sql); return content; } @@ -94,6 +94,9 @@ namespace Umbraco.Core.Persistence.Repositories .On(SqlSyntax, left => left.NodeId, right => right.NodeId) .InnerJoin(SqlSyntax) .On(SqlSyntax, left => left.NodeId, right => right.NodeId, SqlSyntax) + //TODO: IF we want to enable querying on content type information this will need to be joined + //.InnerJoin(SqlSyntax) + //.On(SqlSyntax, left => left.ContentTypeId, right => right.NodeId, SqlSyntax); .Where(x => x.NodeObjectType == NodeObjectTypeId, SqlSyntax); return sql; } @@ -161,25 +164,27 @@ namespace Umbraco.Core.Persistence.Repositories { // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); - var content = new IMedia[dtos.Count]; - var defs = new List(); + + //This is a tuple list identifying if the content item came from the cache or not + var content = new List>(); + var defs = new DocumentDefinitionCollection(); //track the looked up content types, even though the content types are cached // they still need to be deep cloned out of the cache and we don't want to add // the overhead of deep cloning them on every item in this loop var contentTypes = new Dictionary(); - for (var i = 0; i < dtos.Count; i++) + foreach (var dto in dtos) { - var dto = dtos[i]; - // if the cache contains the item, use it if (withCache) { var cached = IsolatedCache.GetCacheItem(GetCacheIdKey(dto.NodeId)); - if (cached != null) + //only use this cached version if the dto returned is the same version - this is just a safety check, media doesn't + //store different versions, but just in case someone corrupts some data we'll double check to be sure. + if (cached != null && cached.Version == dto.VersionId) { - content[i] = cached; + content.Add(new Tuple(cached, true)); continue; } } @@ -197,39 +202,34 @@ namespace Umbraco.Core.Persistence.Repositories contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); contentTypes[dto.ContentDto.ContentTypeId] = contentType; } - - content[i] = MediaFactory.BuildEntity(dto, contentType); - // need properties - defs.Add(new DocumentDefinition( - dto.NodeId, - dto.VersionId, - dto.VersionDate, - dto.ContentDto.NodeDto.CreateDate, - contentType - )); + // track the definition and if it's successfully added or updated then processed + if (defs.AddOrUpdate(new DocumentDefinition(dto, contentType))) + { + content.Add(new Tuple(MediaFactory.BuildEntity(dto, contentType), false)); + } } // load all properties for all documents from database in 1 query var propertyData = GetPropertyCollection(pagingSqlQuery, defs); - // assign - var dtoIndex = 0; - foreach (var def in defs) + // assign property data + foreach (var contentItem in content) { - // move to corresponding item (which has to exist) - while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++; + var cc = contentItem.Item1; + var fromCache = contentItem.Item2; - // complete the item - var cc = content[dtoIndex]; - cc.Properties = propertyData[cc.Id]; + //if this has come from cache, we do not need to build up it's structure + if (fromCache) continue; + + cc.Properties = propertyData[cc.Version]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 cc.ResetDirtyProperties(false); } - return content; + return content.Select(x => x.Item1).ToArray(); } public override IMedia GetByVersion(Guid versionId) @@ -243,7 +243,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateMediaFromDto(dto, versionId, sql); + var content = CreateMediaFromDto(dto, sql); return content; } @@ -263,10 +263,12 @@ namespace Umbraco.Core.Persistence.Repositories // get the next group of nodes var query = GetBaseQuery(false); if (contentTypeIdsA.Length > 0) - query = query - .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); + { + query = query.WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); + } query = query .Where(x => x.NodeId > baseId, SqlSyntax) + .Where(x => x.Trashed == false, SqlSyntax) .OrderBy(x => x.NodeId, SqlSyntax); var sql = SqlSyntax.SelectTop(query, groupSize); var xmlItems = ProcessQuery(sql, new PagingSqlQuery(sql)) @@ -290,7 +292,38 @@ namespace Umbraco.Core.Persistence.Repositories Logger.Error("Could not rebuild XML for nodeId=" + xmlItem.NodeId, e); } } - baseId = xmlItems.Last().NodeId; + baseId = xmlItems[xmlItems.Count - 1].NodeId; + } + + //now delete the items that shouldn't be there + var allMediaIds = Database.Fetch(GetBaseQuery(BaseQueryType.Ids).Where(x => x.Trashed == false, SqlSyntax)); + var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media); + var xmlIdsQuery = new Sql() + .Select("DISTINCT cmsContentXml.nodeId") + .From(SqlSyntax) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId); + + if (contentTypeIdsA.Length > 0) + { + xmlIdsQuery.InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.NodeId) + .InnerJoin(SqlSyntax) + .On(SqlSyntax, left => left.NodeId, right => right.ContentTypeId) + .WhereIn(x => x.ContentTypeId, contentTypeIdsA, SqlSyntax); + } + + xmlIdsQuery.Where(dto => dto.NodeObjectType == mediaObjectType, SqlSyntax); + + var allXmlIds = Database.Fetch(xmlIdsQuery); + + var toRemove = allXmlIds.Except(allMediaIds).ToArray(); + if (toRemove.Length > 0) + { + foreach (var idGroup in toRemove.InGroupsOf(2000)) + { + Database.Execute("DELETE FROM cmsContentXml WHERE nodeId IN (@ids)", new { ids = idGroup }); + } } } @@ -526,20 +559,19 @@ namespace Umbraco.Core.Persistence.Repositories /// Private method to create a media object from a ContentDto /// /// - /// /// /// - private IMedia CreateMediaFromDto(ContentVersionDto dto, Guid versionId, Sql docSql) + private IMedia CreateMediaFromDto(ContentVersionDto dto, Sql docSql) { var contentType = _mediaTypeRepository.Get(dto.ContentDto.ContentTypeId); var media = MediaFactory.BuildEntity(dto, contentType); - var docDef = new DocumentDefinition(dto.NodeId, versionId, media.UpdateDate, media.CreateDate, contentType); + var docDef = new DocumentDefinition(dto, contentType); var properties = GetPropertyCollection(new PagingSqlQuery(docSql), new[] { docDef }); - media.Properties = properties[dto.NodeId]; + media.Properties = properties[dto.VersionId]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 diff --git a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs index 95028f99cd..2ef795282b 100644 --- a/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/MemberRepository.cs @@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories if (dto == null) return null; - var content = CreateMemberFromDto(dto, dto.ContentVersionDto.VersionId, sql); + var content = CreateMemberFromDto(dto, sql); return content; @@ -448,16 +448,16 @@ namespace Umbraco.Core.Persistence.Repositories var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.NodeId); - var media = factory.BuildEntity(dto); + var member = factory.BuildEntity(dto); - var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.NodeId, dto.ContentVersionDto.VersionId, media.UpdateDate, media.CreateDate, memberType) }); + var properties = GetPropertyCollection(new PagingSqlQuery(sql), new[] { new DocumentDefinition(dto.ContentVersionDto, memberType) }); - media.Properties = properties[dto.NodeId]; + member.Properties = properties[dto.ContentVersionDto.VersionId]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 - ((Entity)media).ResetDirtyProperties(false); - return media; + ((Entity)member).ResetDirtyProperties(false); + return member; } @@ -658,20 +658,21 @@ namespace Umbraco.Core.Persistence.Repositories // fetch returns a list so it's ok to iterate it in this method var dtos = Database.Fetch(sqlFull); - var content = new IMember[dtos.Count]; - var defs = new List(); + //This is a tuple list identifying if the content item came from the cache or not + var content = new List>(); + var defs = new DocumentDefinitionCollection(); - for (var i = 0; i < dtos.Count; i++) + foreach (var dto in dtos) { - var dto = dtos[i]; - // if the cache contains the item, use it if (withCache) { var cached = IsolatedCache.GetCacheItem(GetCacheIdKey(dto.NodeId)); - if (cached != null) + //only use this cached version if the dto returned is the same version - this is just a safety check, members dont + //store different versions, but just in case someone corrupts some data we'll double check to be sure. + if (cached != null && cached.Version == dto.ContentVersionDto.VersionId) { - content[i] = cached; + content.Add(new Tuple(cached, true)); continue; } } @@ -679,60 +680,54 @@ namespace Umbraco.Core.Persistence.Repositories // else, need to fetch from the database // content type repository is full-cache so OK to get each one independently var contentType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); - var factory = new MemberFactory(contentType, NodeObjectTypeId, dto.NodeId); - content[i] = factory.BuildEntity(dto); // need properties - defs.Add(new DocumentDefinition( - dto.NodeId, - dto.ContentVersionDto.VersionId, - dto.ContentVersionDto.VersionDate, - dto.ContentVersionDto.ContentDto.NodeDto.CreateDate, - contentType - )); + if (defs.AddOrUpdate(new DocumentDefinition(dto.ContentVersionDto, contentType))) + { + content.Add(new Tuple(MemberFactory.BuildEntity(dto, contentType), false)); + } } // load all properties for all documents from database in 1 query var propertyData = GetPropertyCollection(pagingSqlQuery, defs); - // assign - var dtoIndex = 0; - foreach (var def in defs) + // assign property data + foreach (var contentItem in content) { - // move to corresponding item (which has to exist) - while (dtos[dtoIndex].NodeId != def.Id) dtoIndex++; + var cc = contentItem.Item1; + var fromCache = contentItem.Item2; - // complete the item - var cc = content[dtoIndex]; - cc.Properties = propertyData[cc.Id]; + //if this has come from cache, we do not need to build up it's structure + if (fromCache) continue; + + cc.Properties = propertyData[cc.Version]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 - ((Entity)cc).ResetDirtyProperties(false); + cc.ResetDirtyProperties(false); } - return content; + return content.Select(x => x.Item1).ToArray(); } /// /// Private method to create a member object from a MemberDto /// /// - /// /// /// - private IMember CreateMemberFromDto(MemberDto dto, Guid versionId, Sql docSql) + private IMember CreateMemberFromDto(MemberDto dto, Sql docSql) { var memberType = _memberTypeRepository.Get(dto.ContentVersionDto.ContentDto.ContentTypeId); var factory = new MemberFactory(memberType, NodeObjectTypeId, dto.ContentVersionDto.NodeId); var member = factory.BuildEntity(dto); - var docDef = new DocumentDefinition(dto.ContentVersionDto.NodeId, versionId, member.UpdateDate, member.CreateDate, memberType); + var docDef = new DocumentDefinition(dto.ContentVersionDto, memberType); var properties = GetPropertyCollection(docSql, new[] { docDef }); - member.Properties = properties[dto.ContentVersionDto.NodeId]; + member.Properties = properties[dto.ContentVersionDto.VersionId]; //on initial construction we don't want to have dirty properties tracked // http://issues.umbraco.org/issue/U4-1946 diff --git a/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs b/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs index ee045c725f..8a09a47988 100644 --- a/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/PublicAccessRepository.cs @@ -38,10 +38,9 @@ namespace Umbraco.Core.Persistence.Repositories { sql.Where("umbracoAccess.id IN (@ids)", new { ids = ids }); } - - sql.OrderBy(x => x.NodeId, SqlSyntax); - + var factory = new PublicAccessEntryFactory(); + //MUST be ordered by this GUID ID for the AccessRulesRelator to work var dtos = Database.Fetch(new AccessRulesRelator().Map, sql); return dtos.Select(factory.BuildEntity); } @@ -53,6 +52,7 @@ namespace Umbraco.Core.Persistence.Repositories var sql = translator.Translate(); var factory = new PublicAccessEntryFactory(); + //MUST be ordered by this GUID ID for the AccessRulesRelator to work var dtos = Database.Fetch(new AccessRulesRelator().Map, sql); return dtos.Select(factory.BuildEntity); } @@ -63,7 +63,9 @@ namespace Umbraco.Core.Persistence.Repositories sql.Select("*") .From(SqlSyntax) .LeftJoin(SqlSyntax) - .On(SqlSyntax, left => left.Id, right => right.AccessId); + .On(SqlSyntax, left => left.Id, right => right.AccessId) + //MUST be ordered by this GUID ID for the AccessRulesRelator to work + .OrderBy(dto => dto.Id, SqlSyntax); return sql; } @@ -104,6 +106,11 @@ namespace Umbraco.Core.Persistence.Repositories { rule.AccessId = entity.Key; Database.Insert(rule); + //update the id so HasEntity is correct + var entityRule = entity.Rules.First(x => x.Key == rule.Id); + entityRule.Id = entityRule.Key.GetHashCode(); + //double make sure that this is set since it is possible to add rules via ctor without AddRule + entityRule.AccessEntryId = entity.Key; } entity.ResetDirtyProperties(); @@ -124,7 +131,7 @@ namespace Umbraco.Core.Persistence.Repositories { if (rule.HasIdentity) { - var count = Database.Update(dto.Rules.Single(x => x.Id == rule.Key)); + var count = Database.Update(dto.Rules.First(x => x.Id == rule.Key)); if (count == 0) { throw new InvalidOperationException("No rows were updated for the access rule"); @@ -150,6 +157,8 @@ namespace Umbraco.Core.Persistence.Repositories Database.Delete("WHERE id=@Id", new {Id = removedRule}); } + entity.ClearRemovedRules(); + entity.ResetDirtyProperties(); } @@ -158,4 +167,4 @@ namespace Umbraco.Core.Persistence.Repositories return entity.Key; } } -} \ No newline at end of file +} diff --git a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs index d802d10c66..98743a6a63 100644 --- a/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/VersionableRepositoryBase.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Data.SqlTypes; using System.Diagnostics; using System.Linq; using System.Text; @@ -33,6 +34,11 @@ namespace Umbraco.Core.Persistence.Repositories { private readonly IContentSection _contentSection; + /// + /// This is used for unit tests ONLY + /// + internal static bool ThrowOnWarning = false; + protected VersionableRepositoryBase(IScopeUnitOfWork work, CacheHelper cache, ILogger logger, ISqlSyntaxProvider sqlSyntax, IContentSection contentSection) : base(work, cache, logger, sqlSyntax) { @@ -503,7 +509,7 @@ namespace Umbraco.Core.Persistence.Repositories /// /// /// - protected IDictionary GetPropertyCollection( + protected IDictionary GetPropertyCollection( Sql sql, IReadOnlyCollection documentDefs) { @@ -516,11 +522,11 @@ namespace Umbraco.Core.Persistence.Repositories /// /// /// - protected IDictionary GetPropertyCollection( + protected IDictionary GetPropertyCollection( PagingSqlQuery pagingSqlQuery, IReadOnlyCollection documentDefs) { - if (documentDefs.Count == 0) return new Dictionary(); + if (documentDefs.Count == 0) return new Dictionary(); //initialize to the query passed in var docSql = pagingSqlQuery.PrePagedSql; @@ -576,16 +582,16 @@ ON cmsPropertyData.propertytypeid = cmsPropertyType.id INNER JOIN (" + string.Format(parsedOriginalSql, "cmsContent.nodeId, cmsContentVersion.VersionId") + @") as docData ON cmsPropertyData.versionId = docData.VersionId AND cmsPropertyData.contentNodeId = docData.nodeId -ORDER BY contentNodeId, propertytypeid +ORDER BY contentNodeId, versionId, propertytypeid ", docSql.Arguments); - + //This does NOT fetch all data into memory in a list, this will read // over the records as a data reader, this is much better for performance and memory, // but it means that during the reading of this data set, nothing else can be read // from SQL server otherwise we'll get an exception. var allPropertyData = Database.Query(propSql); - var result = new Dictionary(); + var result = new Dictionary(); var propertiesWithTagSupport = new Dictionary(); //used to track the resolved composition property types per content type so we don't have to re-resolve (ToArray) the list every time var resolvedCompositionProperties = new Dictionary(); @@ -594,11 +600,13 @@ ORDER BY contentNodeId, propertytypeid var propertyDataSetEnumerator = allPropertyData.GetEnumerator(); var hasCurrent = false; // initially there is no enumerator.Current + var comparer = new DocumentDefinitionComparer(SqlSyntax); + try { //This must be sorted by node id because this is how we are sorting the query to lookup property types above, // which allows us to more efficiently iterate over the large data set of property values - foreach (var def in documentDefs.OrderBy(x => x.Id)) + foreach (var def in documentDefs.OrderBy(x => x.Id).ThenBy(x => x.Version, comparer)) { // get the resolved properties from our local cache, or resolve them and put them in cache PropertyType[] compositionProperties; @@ -617,15 +625,17 @@ ORDER BY contentNodeId, propertytypeid var propertyDataDtos = new List(); while (hasCurrent || propertyDataSetEnumerator.MoveNext()) { - if (propertyDataSetEnumerator.Current.NodeId == def.Id) + //Not checking null on VersionId because it can never be null - no idea why it's set to nullable + // ReSharper disable once PossibleInvalidOperationException + if (propertyDataSetEnumerator.Current.VersionId.Value == def.Version) { hasCurrent = false; // enumerator.Current is not available propertyDataDtos.Add(propertyDataSetEnumerator.Current); } else { - hasCurrent = true; // enumerator.Current is available for another def - break; // no more propertyDataDto for this def + hasCurrent = true; // enumerator.Current is available for another def + break; // no more propertyDataDto for this def } } @@ -660,11 +670,19 @@ ORDER BY contentNodeId, propertytypeid } } - if (result.ContainsKey(def.Id)) + if (result.ContainsKey(def.Version)) { - Logger.Warn>("The query returned multiple property sets for document definition " + def.Id + ", " + def.Composition.Name); + var msg = string.Format("The query returned multiple property sets for document definition {0}, {1}, {2}", def.Id, def.Version, def.Composition.Name); + if (ThrowOnWarning) + { + throw new InvalidOperationException(msg); + } + else + { + Logger.Warn>(msg); + } } - result[def.Id] = new PropertyCollection(properties); + result[def.Version] = new PropertyCollection(properties); } } finally @@ -687,8 +705,8 @@ ORDER BY contentNodeId, propertytypeid case "NAME": return "umbracoNode.text"; case "PUBLISHED": - return "cmsDocument.published"; case "OWNER": + return "cmsDocument.published"; //TODO: This isn't going to work very nicely because it's going to order by ID, not by letter return "umbracoNode.nodeUser"; // Members only @@ -772,25 +790,159 @@ ORDER BY contentNodeId, propertytypeid /// protected abstract Sql GetBaseQuery(BaseQueryType queryType); + internal class DocumentDefinitionCollection : KeyedCollection + { + private readonly bool _includeAllVersions; + + /// + /// Constructor specifying if all versions should be allowed, in that case the key for the collection becomes the versionId (GUID) + /// + /// + public DocumentDefinitionCollection(bool includeAllVersions = false) + { + _includeAllVersions = includeAllVersions; + } + + protected override ValueType GetKeyForItem(DocumentDefinition item) + { + return _includeAllVersions ? (ValueType)item.Version : item.Id; + } + + /// + /// if this key already exists if it does then we need to check + /// if the existing item is 'older' than the new item and if that is the case we'll replace the older one + /// + /// + /// + public bool AddOrUpdate(DocumentDefinition item) + { + //if we are including all versions then just add, we aren't checking for latest + if (_includeAllVersions) + { + base.Add(item); + return true; + } + + if (Dictionary == null) + { + base.Add(item); + return true; + } + + var key = GetKeyForItem(item); + DocumentDefinition found; + if (TryGetValue(key, out found)) + { + //it already exists and it's older so we need to replace it + if (item.VersionId > found.VersionId) + { + var currIndex = Items.IndexOf(found); + if (currIndex == -1) + throw new IndexOutOfRangeException("Could not find the item in the list: " + found.Version); + + //replace the current one with the newer one + SetItem(currIndex, item); + return true; + } + //could not add or update + return false; + } + + base.Add(item); + return true; + } + + public bool TryGetValue(ValueType key, out DocumentDefinition val) + { + if (Dictionary == null) + { + val = null; + return false; + } + return Dictionary.TryGetValue(key, out val); + } + } + + /// + /// A custom comparer required for sorting entities by GUIDs to match how the sorting of GUIDs works on SQL server + /// + /// + /// MySql sorts GUIDs as a string, MSSQL sorts based on byte sections, this comparer will allow sorting GUIDs to be the same as how SQL server does + /// + private class DocumentDefinitionComparer : IComparer + { + private readonly ISqlSyntaxProvider _sqlSyntax; + + public DocumentDefinitionComparer(ISqlSyntaxProvider sqlSyntax) + { + _sqlSyntax = sqlSyntax; + } + + public int Compare(Guid x, Guid y) + { + //MySql sorts on GUIDs as strings (i.e. normal) + if (_sqlSyntax is MySqlSyntaxProvider) + { + return x.CompareTo(y); + } + + //MSSQL doesn't it sorts them on byte sections! + return new SqlGuid(x).CompareTo(new SqlGuid(y)); + } + } + internal class DocumentDefinition { /// /// Initializes a new instance of the class. /// - public DocumentDefinition(int id, Guid version, DateTime versionDate, DateTime createDate, IContentTypeComposition composition) + public DocumentDefinition(DocumentDto dto, IContentTypeComposition composition) { - Id = id; - Version = version; - VersionDate = versionDate; - CreateDate = createDate; + DocumentDto = dto; + ContentVersionDto = dto.ContentVersionDto; Composition = composition; } - public int Id { get; set; } - public Guid Version { get; set; } - public DateTime VersionDate { get; set; } - public DateTime CreateDate { get; set; } - public IContentTypeComposition Composition { get; set; } + public DocumentDefinition(ContentVersionDto dto, IContentTypeComposition composition) + { + ContentVersionDto = dto; + Composition = composition; + } + + public DocumentDto DocumentDto { get; private set; } + public ContentVersionDto ContentVersionDto { get; private set; } + + public int Id + { + get { return ContentVersionDto.NodeId; } + } + + public Guid Version + { + get { return DocumentDto != null ? DocumentDto.VersionId : ContentVersionDto.VersionId; } + } + + /// + /// This is used to determien which version is the most recent + /// + public int VersionId + { + get { return ContentVersionDto.Id; } + } + + public DateTime VersionDate + { + get { return ContentVersionDto.VersionDate; } + } + + public DateTime CreateDate + { + get { return ContentVersionDto.ContentDto.NodeDto.CreateDate; } + } + + public IContentTypeComposition Composition { get; set; } + + } /// diff --git a/src/Umbraco.Core/Services/ContentService.cs b/src/Umbraco.Core/Services/ContentService.cs index f76ecad963..69deb04ee5 100644 --- a/src/Umbraco.Core/Services/ContentService.cs +++ b/src/Umbraco.Core/Services/ContentService.cs @@ -2243,6 +2243,7 @@ namespace Umbraco.Core.Services //We need to check if children and their publish state to ensure that we 'republish' content that was previously published if (published && previouslyPublished == false && HasChildren(content.Id)) { + //TODO: Horrible for performance if there are lots of descendents! We should page if anything but this is crazy var descendants = GetPublishedDescendants(content); _publishingStrategy.PublishingFinalized(uow, descendants, false); diff --git a/src/Umbraco.Core/Services/IMediaService.cs b/src/Umbraco.Core/Services/IMediaService.cs index 4fdf3d9b84..47e214d734 100644 --- a/src/Umbraco.Core/Services/IMediaService.cs +++ b/src/Umbraco.Core/Services/IMediaService.cs @@ -81,6 +81,7 @@ namespace Umbraco.Core.Services /// void RebuildXmlStructures(params int[] contentTypeIds); + int CountNotTrashed(string contentTypeAlias = null); int Count(string contentTypeAlias = null); int CountChildren(int parentId, string contentTypeAlias = null); int CountDescendants(int parentId, string contentTypeAlias = null); @@ -167,6 +168,22 @@ namespace Umbraco.Core.Services IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords, string orderBy, Direction orderDirection, bool orderBySystemField, string filter); + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Children from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Flag to indicate when ordering by system field + /// Search text filter + /// A list of content type Ids to filter the list by + /// An Enumerable list of objects + IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalRecords, + string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter); + [Obsolete("Use the overload with 'long' parameter types instead")] [EditorBrowsable(EditorBrowsableState.Never)] IEnumerable GetPagedDescendants(int id, int pageIndex, int pageSize, out int totalRecords, diff --git a/src/Umbraco.Core/Services/MediaService.cs b/src/Umbraco.Core/Services/MediaService.cs index 48db7f8ced..36a5ce3591 100644 --- a/src/Umbraco.Core/Services/MediaService.cs +++ b/src/Umbraco.Core/Services/MediaService.cs @@ -261,6 +261,29 @@ namespace Umbraco.Core.Services } } + public int CountNotTrashed(string contentTypeAlias = null) + { + var uow = UowProvider.GetUnitOfWork(); + using (var repository = RepositoryFactory.CreateMediaRepository(uow)) + using (var mediaTypeRepository = RepositoryFactory.CreateMediaTypeRepository(uow)) + { + var mediaTypeId = 0; + if (contentTypeAlias.IsNullOrWhiteSpace() == false) + { + var mediaType = mediaTypeRepository.Get(contentTypeAlias); + if (mediaType == null) return 0; + mediaTypeId = mediaType.Id; + } + + var query = Query.Builder.Where(media => media.Trashed == false); + if (mediaTypeId > 0) + { + query.Where(media => media.ContentTypeId == mediaTypeId); + } + return repository.Count(query); + } + } + public int Count(string contentTypeAlias = null) { using (var uow = UowProvider.GetUnitOfWork(commit: true)) @@ -448,6 +471,25 @@ namespace Umbraco.Core.Services /// An Enumerable list of objects public IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren, string orderBy, Direction orderDirection, bool orderBySystemField, string filter) + { + return GetPagedChildren(id, pageIndex, pageSize, out totalChildren, orderBy, orderDirection, true, filter, null); + } + + /// + /// Gets a collection of objects by Parent Id + /// + /// Id of the Parent to retrieve Children from + /// Page number + /// Page size + /// Total records query would return without paging + /// Field to order by + /// Direction to order by + /// Flag to indicate when ordering by system field + /// Search text filter + /// A list of content type Ids to filter the list by + /// An Enumerable list of objects + public IEnumerable GetPagedChildren(int id, long pageIndex, int pageSize, out long totalChildren, + string orderBy, Direction orderDirection, bool orderBySystemField, string filter, int[] contentTypeFilter) { Mandate.ParameterCondition(pageIndex >= 0, "pageIndex"); Mandate.ParameterCondition(pageSize > 0, "pageSize"); @@ -460,6 +502,11 @@ namespace Umbraco.Core.Services // always check for a parent - else it will also get decendants (and then you should use the GetPagedDescendants method) query.Where(x => x.ParentId == id); + if (contentTypeFilter != null && contentTypeFilter.Length > 0) + { + query.Where(x => contentTypeFilter.Contains(x.ContentTypeId)); + } + IQuery filterQuery = null; if (filter.IsNullOrWhiteSpace() == false) { diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index b9aaed7251..a41a560161 100644 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -198,6 +198,7 @@ + @@ -552,6 +553,7 @@ + diff --git a/src/Umbraco.Core/XmlExtensions.cs b/src/Umbraco.Core/XmlExtensions.cs index e2518c791a..8707523615 100644 --- a/src/Umbraco.Core/XmlExtensions.cs +++ b/src/Umbraco.Core/XmlExtensions.cs @@ -16,32 +16,6 @@ namespace Umbraco.Core /// internal static class XmlExtensions { - /// - /// Saves the xml document async - /// - /// - /// - /// - public static async Task SaveAsync(this XmlDocument xdoc, string filename) - { - if (xdoc.DocumentElement == null) - throw new XmlException("Cannot save xml document, there is no root element"); - - using (var fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 4096, useAsync: true)) - using (var xmlWriter = XmlWriter.Create(fs, new XmlWriterSettings - { - Async = true, - Encoding = Encoding.UTF8, - Indent = true - })) - { - //NOTE: There are no nice methods to write it async, only flushing it async. We - // could implement this ourselves but it'd be a very manual process. - xdoc.WriteTo(xmlWriter); - await xmlWriter.FlushAsync().ConfigureAwait(false); - } - } - public static bool HasAttribute(this XmlAttributeCollection attributes, string attributeName) { return attributes.Cast().Any(x => x.Name == attributeName); diff --git a/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs b/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs index 03aa3206ee..5fa395d2f1 100644 --- a/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs +++ b/src/Umbraco.Tests.Benchmarks/ModelToSqlExpressionHelperBenchmarks.cs @@ -16,6 +16,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; +using Moq; using Umbraco.Core; using Umbraco.Core.Logging; using Umbraco.Core.Models; @@ -43,15 +44,18 @@ namespace Umbraco.Tests.Benchmarks public ModelToSqlExpressionHelperBenchmarks() { - _contentMapper = new ContentMapper(_syntaxProvider); - _contentMapper.BuildMap(); + var contentMapper = new ContentMapper(_syntaxProvider); + contentMapper.BuildMap(); _cachedExpression = new CachedExpression(); + var mappingResolver = new Mock(); + mappingResolver.Setup(resolver => resolver.ResolveMapperByType(It.IsAny())).Returns(contentMapper); + _mappingResolver = mappingResolver.Object; } private readonly ISqlSyntaxProvider _syntaxProvider = new SqlCeSyntaxProvider(); - private readonly BaseMapper _contentMapper; private readonly CachedExpression _cachedExpression; - + private readonly MappingResolver _mappingResolver; + [Benchmark(Baseline = true)] public void WithNonCached() { @@ -62,7 +66,7 @@ namespace Umbraco.Tests.Benchmarks Expression> predicate = content => content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b); - var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _contentMapper); + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _mappingResolver); var result = modelToSqlExpressionHelper.Visit(predicate); } @@ -80,7 +84,7 @@ namespace Umbraco.Tests.Benchmarks Expression> predicate = content => content.Path.StartsWith("-1") && content.Published && (content.ContentTypeId == a || content.ContentTypeId == b); - var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _contentMapper); + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(_syntaxProvider, _mappingResolver); //wrap it! _cachedExpression.Wrap(predicate); diff --git a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj index ba954bc57c..aa165e4e00 100644 --- a/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj +++ b/src/Umbraco.Tests.Benchmarks/Umbraco.Tests.Benchmarks.csproj @@ -54,6 +54,9 @@ ..\packages\BenchmarkDotNet.Toolchains.Roslyn.0.9.9\lib\net45\BenchmarkDotNet.Toolchains.Roslyn.dll True + + ..\packages\Castle.Core.4.0.0\lib\net45\Castle.Core.dll + ..\packages\Microsoft.CodeAnalysis.Common.1.3.2\lib\net45\Microsoft.CodeAnalysis.dll True @@ -66,6 +69,9 @@ ..\packages\Microsoft.Diagnostics.Tracing.TraceEvent.1.0.41\lib\net40\Microsoft.Diagnostics.Tracing.TraceEvent.dll True + + ..\packages\Moq.4.7.0\lib\net45\Moq.dll + ..\packages\System.Collections.Immutable.1.2.0\lib\portable-net45+win8+wp8+wpa81\System.Collections.Immutable.dll diff --git a/src/Umbraco.Tests.Benchmarks/packages.config b/src/Umbraco.Tests.Benchmarks/packages.config index fd59cfcb10..db9e0b6248 100644 --- a/src/Umbraco.Tests.Benchmarks/packages.config +++ b/src/Umbraco.Tests.Benchmarks/packages.config @@ -4,11 +4,13 @@ + + diff --git a/src/Umbraco.Tests/Issues/U9560.cs b/src/Umbraco.Tests/Issues/U9560.cs new file mode 100644 index 0000000000..f9174498ff --- /dev/null +++ b/src/Umbraco.Tests/Issues/U9560.cs @@ -0,0 +1,92 @@ +using System; +using NUnit.Framework; +using Umbraco.Core.Models; +using Umbraco.Tests.TestHelpers; + +namespace Umbraco.Tests.Issues +{ + [TestFixture] + [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] + public class U9560 : BaseDatabaseFactoryTest + { + [Test] + public void Test() + { + // create a content type and some properties + var contentType = new ContentType(-1); + contentType.Alias = "test"; + contentType.Name = "test"; + var propertyType = new PropertyType("test", DataTypeDatabaseType.Ntext, "prop") { Name = "Prop", Description = "", Mandatory = false, SortOrder = 1, DataTypeDefinitionId = -88 }; + contentType.PropertyTypeCollection.Add(propertyType); + ServiceContext.ContentTypeService.Save(contentType); + + var aliasName = string.Empty; + + // read fields, same as what we do with PetaPoco Fetch + var db = DatabaseContext.Database; + db.OpenSharedConnection(); + try + { + var conn = db.Connection; + var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT mandatory, dataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias, name, validationRegExp, description from cmsPropertyType where id=" + propertyType.Id; + using (var reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + for (var i = 0; i < reader.FieldCount; i++) + Console.WriteLine(reader.GetName(i)); + aliasName = reader.GetName(5); + } + } + } + finally + { + db.CloseSharedConnection(); + } + + // note that although the query is for 'alias' the field is named 'Alias' + Assert.AreEqual("Alias", aliasName); + + // try differently + db.OpenSharedConnection(); + try + { + var conn = db.Connection; + var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT mandatory, dataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias as alias, name, validationRegExp, description from cmsPropertyType where id=" + propertyType.Id; + using (var reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + for (var i = 0; i < reader.FieldCount; i++) + Console.WriteLine(reader.GetName(i)); + aliasName = reader.GetName(5); + } + } + } + finally + { + db.CloseSharedConnection(); + } + + // and now it is OK + Assert.AreEqual("alias", aliasName); + + // get the legacy content type + var legacyContentType = new umbraco.cms.businesslogic.ContentType(contentType.Id); + Assert.AreEqual("test", legacyContentType.Alias); + + // get the legacy properties + var legacyProperties = legacyContentType.PropertyTypes; + + // without the fix, due to some (swallowed) inner exception, we have no properties + //Assert.IsNull(legacyProperties); + + // thanks to the fix, it works + Assert.IsNotNull(legacyProperties); + Assert.AreEqual(1, legacyProperties.Count); + Assert.AreEqual("prop", legacyProperties[0].Alias); + } + } +} diff --git a/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs b/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs index 7bd4086d9c..8304b1824c 100644 --- a/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs +++ b/src/Umbraco.Tests/Persistence/DatabaseContextTests.cs @@ -48,6 +48,13 @@ namespace Umbraco.Tests.Persistence ApplicationContext.Current = null; } + [Test] + public void Database_Connection() + { + var db = _dbContext.Database; + Assert.IsNull(db.Connection); + } + [Test] public void Can_Verify_Single_Database_Instance() { diff --git a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs index 51de34efc2..7d50f0c036 100644 --- a/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs +++ b/src/Umbraco.Tests/Persistence/Querying/ExpressionTests.cs @@ -17,19 +17,52 @@ namespace Umbraco.Tests.Persistence.Querying [TestFixture] public class ExpressionTests : BaseUsingSqlCeSyntax { - // [Test] - // public void Can_Query_With_Content_Type_Alias() - // { - // //Arrange - // Expression> predicate = content => content.ContentType.Alias == "Test"; - // var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(); - // var result = modelToSqlExpressionHelper.Visit(predicate); + [Test] + public void Equals_Claus_With_Two_Entity_Values() + { + var dataType = new DataTypeDefinition(-1, "Test") + { + Id = 12345 + }; + Expression> predicate = p => p.DataTypeDefinitionId == dataType.Id; + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(); + var result = modelToSqlExpressionHelper.Visit(predicate); - // Debug.Print("Model to Sql ExpressionHelper: \n" + result); + Debug.Print("Model to Sql ExpressionHelper: \n" + result); - // Assert.AreEqual("[cmsContentType].[alias] = @0", result); - // Assert.AreEqual("Test", modelToSqlExpressionHelper.GetSqlParameters()[0]); - // } + Assert.AreEqual("([cmsPropertyType].[dataTypeId] = @0)", result); + Assert.AreEqual(12345, modelToSqlExpressionHelper.GetSqlParameters()[0]); + } + + [Test] + public void Can_Query_With_Content_Type_Alias() + { + //Arrange + Expression> predicate = content => content.ContentType.Alias == "Test"; + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(); + var result = modelToSqlExpressionHelper.Visit(predicate); + + Debug.Print("Model to Sql ExpressionHelper: \n" + result); + + Assert.AreEqual("([cmsContentType].[alias] = @0)", result); + Assert.AreEqual("Test", modelToSqlExpressionHelper.GetSqlParameters()[0]); + } + + [Test] + public void Can_Query_With_Content_Type_Aliases() + { + //Arrange + var aliases = new[] {"Test1", "Test2"}; + Expression> predicate = content => aliases.Contains(content.ContentType.Alias); + var modelToSqlExpressionHelper = new ModelToSqlExpressionVisitor(); + var result = modelToSqlExpressionHelper.Visit(predicate); + + Debug.Print("Model to Sql ExpressionHelper: \n" + result); + + Assert.AreEqual("[cmsContentType].[alias] IN (@1,@2)", result); + Assert.AreEqual("Test1", modelToSqlExpressionHelper.GetSqlParameters()[1]); + Assert.AreEqual("Test2", modelToSqlExpressionHelper.GetSqlParameters()[2]); + } [Test] public void CachedExpression_Can_Verify_Path_StartsWith_Predicate_In_Same_Result() diff --git a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs index 4e95af39cf..6cb97feec1 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/ContentRepositoryTest.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text.RegularExpressions; using System.Xml.Linq; @@ -21,7 +22,6 @@ using Umbraco.Tests.TestHelpers; using Umbraco.Tests.TestHelpers.Entities; using umbraco.editorControls.tinyMCE3; using umbraco.interfaces; -using Umbraco.Core.Models.EntityBase; using Umbraco.Core.Persistence.DatabaseModelDefinitions; namespace Umbraco.Tests.Persistence.Repositories @@ -36,11 +36,15 @@ namespace Umbraco.Tests.Persistence.Repositories base.Initialize(); CreateTestData(); + + VersionableRepositoryBase.ThrowOnWarning = true; } [TearDown] public override void TearDown() { + VersionableRepositoryBase.ThrowOnWarning = false; + base.TearDown(); } @@ -67,6 +71,131 @@ namespace Umbraco.Tests.Persistence.Repositories return repository; } + [Test] + public void Get_Always_Returns_Latest_Version() + { + // Arrange + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + IContent content1; + + var versions = new List(); + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage"); + content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType); + + //save version + contentTypeRepository.AddOrUpdate(hasPropertiesContentType); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + versions.Add(content1.Version); + + //publish version + content1.ChangePublishedState(PublishedState.Published); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + versions.Add(content1.Version); + + //change something and make a pending version + content1.Name = "new name"; + content1.ChangePublishedState(PublishedState.Saved); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + versions.Add(content1.Version); + } + + // Assert + Assert.AreEqual(3, versions.Distinct().Count()); + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray()[0]; + Assert.AreEqual(versions[2], content.Version); + + content = repository.Get(content1.Id); + Assert.AreEqual(versions[2], content.Version); + + foreach (var version in versions) + { + content = repository.GetByVersion(version); + Assert.IsNotNull(content); + Assert.AreEqual(version, content.Version); + } + } + } + + [Test] + public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags() + { + // Arrange + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + IContent content1; + + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage"); + content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType); + + contentTypeRepository.AddOrUpdate(hasPropertiesContentType); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + } + + var versionDtos = new List(); + + //Now manually corrupt the data + var versions = new[] { Guid.NewGuid(), Guid.NewGuid() }; + for (var index = 0; index < versions.Length; index++) + { + var version = versions[index]; + var versionDate = DateTime.Now.AddMinutes(index); + var versionDto = new ContentVersionDto + { + NodeId = content1.Id, + VersionDate = versionDate, + VersionId = version + }; + this.DatabaseContext.Database.Insert(versionDto); + versionDtos.Add(versionDto); + this.DatabaseContext.Database.Insert(new DocumentDto + { + Newest = true, + NodeId = content1.Id, + Published = true, + Text = content1.Name, + VersionId = version, + WriterUserId = 0, + UpdateDate = versionDate, + TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?) content1.Template.Id + }); + } + + // Assert + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray(); + Assert.AreEqual(1, content.Length); + Assert.AreEqual(content[0].Version, versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionId); + Assert.AreEqual(content[0].UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture)); + + var contentItem = repository.GetByVersion(content1.Version); + Assert.IsNotNull(contentItem); + + contentItem = repository.Get(content1.Id); + Assert.IsNotNull(contentItem); + Assert.AreEqual(contentItem.UpdateDate.ToString(CultureInfo.InvariantCulture), versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionDate.ToString(CultureInfo.InvariantCulture)); + Assert.AreEqual(contentItem.Version, versionDtos.Single(x => x.Id == versionDtos.Max(y => y.Id)).VersionId); + + var allVersions = repository.GetAllVersions(content[0].Id); + var allKnownVersions = versionDtos.Select(x => x.VersionId).Union(new[]{ content1.Version }).ToArray(); + Assert.IsTrue(allKnownVersions.ContainsAll(allVersions.Select(x => x.Version))); + Assert.IsTrue(allVersions.Select(x => x.Version).ContainsAll(allKnownVersions)); + } + } + /// /// This tests the regression issue of U4-9438 /// @@ -226,6 +355,62 @@ namespace Umbraco.Tests.Persistence.Repositories } } + [Test] + public void Rebuild_All_Xml_Structures_Ensure_Orphaned_Are_Removed() + { + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + using (var repository = CreateRepository(unitOfWork, out contentTypeRepository)) + { + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + + var contentType1 = MockedContentTypes.CreateSimpleContentType("Textpage1", "Textpage1"); + contentTypeRepository.AddOrUpdate(contentType1); + var allCreated = new List(); + + for (var i = 0; i < 100; i++) + { + //These will be non-published so shouldn't show up + var c1 = MockedContent.CreateSimpleContent(contentType1); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + for (var i = 0; i < 100; i++) + { + var c1 = MockedContent.CreateSimpleContent(contentType1); + c1.ChangePublishedState(PublishedState.Published); + repository.AddOrUpdate(c1); + allCreated.Add(c1); + } + unitOfWork.Commit(); + + //now create some versions of this content - this shouldn't affect the xml structures saved + for (int i = 0; i < allCreated.Count; i++) + { + allCreated[i].Name = "blah" + i; + repository.AddOrUpdate(allCreated[i]); + } + unitOfWork.Commit(); + + //Add some extra orphaned rows that shouldn't be there + var notPublished = MockedContent.CreateSimpleContent(contentType1); + repository.AddOrUpdate(notPublished); + unitOfWork.Commit(); + //Force add it + unitOfWork.Database.Insert(new ContentXmlDto + { + NodeId = notPublished.Id, + Xml = "" + }); + + repository.RebuildXmlStructures(media => new XElement("test"), 10); + + Assert.AreEqual(100, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + [Test] public void Rebuild_All_Xml_Structures_For_Content_Type() { diff --git a/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs new file mode 100644 index 0000000000..baf574e091 --- /dev/null +++ b/src/Umbraco.Tests/Persistence/Repositories/EntityRepositoryTest.cs @@ -0,0 +1,183 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Moq; +using NUnit.Framework; +using Umbraco.Core; +using Umbraco.Core.Configuration.UmbracoSettings; +using Umbraco.Core.IO; +using Umbraco.Core.Models; +using Umbraco.Core.Models.EntityBase; +using Umbraco.Core.Models.Rdbms; +using Umbraco.Core.Persistence.Querying; +using Umbraco.Core.Persistence.Repositories; +using Umbraco.Core.Persistence.UnitOfWork; +using Umbraco.Tests.TestHelpers; +using Umbraco.Tests.TestHelpers.Entities; + +namespace Umbraco.Tests.Persistence.Repositories +{ + [DatabaseTestBehavior(DatabaseBehavior.NewDbFileAndSchemaPerTest)] + [TestFixture] + public class EntityRepositoryTest : BaseDatabaseFactoryTest + { + [SetUp] + public override void Initialize() + { + base.Initialize(); + } + + [TearDown] + public override void TearDown() + { + base.TearDown(); + } + + private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository) + { + TemplateRepository tr; + return CreateContentRepository(unitOfWork, out contentTypeRepository, out tr); + } + + private ContentRepository CreateContentRepository(IDatabaseUnitOfWork unitOfWork, out ContentTypeRepository contentTypeRepository, out TemplateRepository templateRepository) + { + templateRepository = new TemplateRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, Mock.Of(), Mock.Of(), Mock.Of()); + var tagRepository = new TagRepository(unitOfWork, CacheHelper, Logger, SqlSyntax); + contentTypeRepository = new ContentTypeRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, templateRepository); + var repository = new ContentRepository(unitOfWork, CacheHelper, Logger, SqlSyntax, contentTypeRepository, templateRepository, tagRepository, Mock.Of()); + return repository; + } + + [Test] + public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags_Full_Model() + { + // Arrange + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + IContent content1; + + using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository)) + { + var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage"); + content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType); + + contentTypeRepository.AddOrUpdate(hasPropertiesContentType); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + } + + var versionDtos = new List(); + + //Now manually corrupt the data + var versions = new[] {Guid.NewGuid(), Guid.NewGuid()}; + for (var index = 0; index < versions.Length; index++) + { + var version = versions[index]; + var versionDate = DateTime.Now.AddMinutes(index); + var versionDto = new ContentVersionDto + { + NodeId = content1.Id, + VersionDate = versionDate, + VersionId = version + }; + this.DatabaseContext.Database.Insert(versionDto); + versionDtos.Add(versionDto); + this.DatabaseContext.Database.Insert(new DocumentDto + { + Newest = true, + NodeId = content1.Id, + Published = true, + Text = content1.Name, + VersionId = version, + WriterUserId = 0, + UpdateDate = versionDate, + TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id + }); + } + + // Assert + using (var repository = new EntityRepository(unitOfWork)) + { + var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id), Constants.ObjectTypes.DocumentGuid).ToArray(); + Assert.AreEqual(1, content.Length); + Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]); + + content = repository.GetAll(Constants.ObjectTypes.DocumentGuid, content[0].Key).ToArray(); + Assert.AreEqual(1, content.Length); + Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]); + + content = repository.GetAll(Constants.ObjectTypes.DocumentGuid, content[0].Id).ToArray(); + Assert.AreEqual(1, content.Length); + Assert.AreEqual(versionDtos.Max(x => x.Id), content[0].AdditionalData["VersionId"]); + + var contentItem = repository.Get(content[0].Id, Constants.ObjectTypes.DocumentGuid); + Assert.AreEqual(versionDtos.Max(x => x.Id), contentItem.AdditionalData["VersionId"]); + + contentItem = repository.GetByKey(content[0].Key, Constants.ObjectTypes.DocumentGuid); + Assert.AreEqual(versionDtos.Max(x => x.Id), contentItem.AdditionalData["VersionId"]); + } + } + + /// + /// The Slim model will test the EntityRepository when it doesn't know the object type so it will + /// make the simplest (slim) query + /// + [Test] + public void Deal_With_Corrupt_Duplicate_Newest_Published_Flags_Slim_Model() + { + // Arrange + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + ContentTypeRepository contentTypeRepository; + IContent content1; + + using (var repository = CreateContentRepository(unitOfWork, out contentTypeRepository)) + { + var hasPropertiesContentType = MockedContentTypes.CreateSimpleContentType("umbTextpage1", "Textpage"); + content1 = MockedContent.CreateSimpleContent(hasPropertiesContentType); + + contentTypeRepository.AddOrUpdate(hasPropertiesContentType); + repository.AddOrUpdate(content1); + unitOfWork.Commit(); + } + + //Now manually corrupt the data + var versions = new[] { Guid.NewGuid(), Guid.NewGuid() }; + for (var index = 0; index < versions.Length; index++) + { + var version = versions[index]; + var versionDate = DateTime.Now.AddMinutes(index); + var versionDto = new ContentVersionDto + { + NodeId = content1.Id, + VersionDate = versionDate, + VersionId = version + }; + this.DatabaseContext.Database.Insert(versionDto); + this.DatabaseContext.Database.Insert(new DocumentDto + { + Newest = true, + NodeId = content1.Id, + Published = true, + Text = content1.Name, + VersionId = version, + WriterUserId = 0, + UpdateDate = versionDate, + TemplateId = content1.Template == null || content1.Template.Id <= 0 ? null : (int?)content1.Template.Id + }); + } + + // Assert + using (var repository = new EntityRepository(unitOfWork)) + { + var content = repository.GetByQuery(new Query().Where(c => c.Id == content1.Id)).ToArray(); + Assert.AreEqual(1, content.Length); + var contentItem = repository.Get(content[0].Id); + Assert.IsNotNull(contentItem); + contentItem = repository.GetByKey(content[0].Key); + Assert.IsNotNull(contentItem); + } + } + } +} \ No newline at end of file diff --git a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs index d9a228dfd7..c9f78fb201 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/MediaRepositoryTest.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Xml.Linq; using Moq; using NUnit.Framework; +using Umbraco.Core; using Umbraco.Core.Configuration.UmbracoSettings; using Umbraco.Core.Logging; using Umbraco.Core.Models; @@ -70,6 +71,44 @@ namespace Umbraco.Tests.Persistence.Repositories } } + [Test] + public void Rebuild_All_Xml_Structures_Ensure_Orphaned_Are_Removed() + { + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + MediaTypeRepository mediaTypeRepository; + using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository)) + { + //delete all xml + unitOfWork.Database.Execute("DELETE FROM cmsContentXml"); + + var mediaType = mediaTypeRepository.Get(1032); + + for (var i = 0; i < 100; i++) + { + var image = MockedMedia.CreateMediaImage(mediaType, -1); + repository.AddOrUpdate(image); + } + unitOfWork.Commit(); + + //Add some extra orphaned rows that shouldn't be there + var trashed = MockedMedia.CreateMediaImage(mediaType, -1); + trashed.ChangeTrashedState(true, Constants.System.RecycleBinMedia); + repository.AddOrUpdate(trashed); + unitOfWork.Commit(); + //Force add it + unitOfWork.Database.Insert(new ContentXmlDto + { + NodeId = trashed.Id, + Xml = "" + }); + + repository.RebuildXmlStructures(media => new XElement("test"), 10); + + Assert.AreEqual(103, unitOfWork.Database.ExecuteScalar("SELECT COUNT(*) FROM cmsContentXml")); + } + } + [Test] public void Rebuild_Some_Xml_Structures() { @@ -345,6 +384,61 @@ namespace Umbraco.Tests.Persistence.Repositories } } + [Test] + public void Can_Perform_GetByQuery_On_MediaRepository_With_ContentType_Id_Filter() + { + // Arrange + var folderMediaType = ServiceContext.ContentTypeService.GetMediaType(1031); + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + MediaTypeRepository mediaTypeRepository; + using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository)) + { + // Act + for (int i = 0; i < 10; i++) + { + var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1); + repository.AddOrUpdate(folder); + } + unitOfWork.Commit(); + + var types = new[] { 1031 }; + var query = Query.Builder.Where(x => types.Contains(x.ContentTypeId)); + var result = repository.GetByQuery(query); + + // Assert + Assert.That(result.Count(), Is.GreaterThanOrEqualTo(11)); + } + } + + [Ignore("We could allow this to work but it requires an extra join on the query used which currently we don't absolutely need so leaving this out for now")] + [Test] + public void Can_Perform_GetByQuery_On_MediaRepository_With_ContentType_Alias_Filter() + { + // Arrange + var folderMediaType = ServiceContext.ContentTypeService.GetMediaType(1031); + var provider = new PetaPocoUnitOfWorkProvider(Logger); + var unitOfWork = provider.GetUnitOfWork(); + MediaTypeRepository mediaTypeRepository; + using (var repository = CreateRepository(unitOfWork, out mediaTypeRepository)) + { + // Act + for (int i = 0; i < 10; i++) + { + var folder = MockedMedia.CreateMediaFolder(folderMediaType, -1); + repository.AddOrUpdate(folder); + } + unitOfWork.Commit(); + + var types = new[] { "Folder" }; + var query = Query.Builder.Where(x => types.Contains(x.ContentType.Alias)); + var result = repository.GetByQuery(query); + + // Assert + Assert.That(result.Count(), Is.GreaterThanOrEqualTo(11)); + } + } + [Test] public void Can_Perform_GetPagedResultsByQuery_ForFirstPage_On_MediaRepository() { @@ -472,7 +566,7 @@ namespace Umbraco.Tests.Persistence.Repositories Assert.That(result.First().Name, Is.EqualTo("Test File")); } } - + [Test] public void Can_Perform_GetPagedResultsByQuery_WithFilterMatchingAll_On_MediaRepository() { diff --git a/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs b/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs index 6d446e9174..7f7e8ac14f 100644 --- a/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs +++ b/src/Umbraco.Tests/Persistence/Repositories/PublicAccessRepositoryTest.cs @@ -154,36 +154,61 @@ namespace Umbraco.Tests.Persistence.Repositories [Test] public void Get_All() { - var content = CreateTestData(3).ToArray(); + var content = CreateTestData(30).ToArray(); var provider = new PetaPocoUnitOfWorkProvider(Logger); var unitOfWork = provider.GetUnitOfWork(); using (var repo = new PublicAccessRepository(unitOfWork, CacheHelper, Logger, SqlSyntax)) - { - var entry1 = new PublicAccessEntry(content[0], content[1], content[2], new[] + { + var allEntries = new List(); + for (int i = 0; i < 10; i++) { - new PublicAccessRule + var rules = new List(); + for (int j = 0; j < 50; j++) { - RuleValue = "test", - RuleType = "RoleName" - }, - }); - repo.AddOrUpdate(entry1); + rules.Add(new PublicAccessRule + { + RuleValue = "test" + j, + RuleType = "RoleName" + j + }); + } + var entry1 = new PublicAccessEntry(content[i], content[i+1], content[i+2], rules); + repo.AddOrUpdate(entry1); + unitOfWork.Commit(); + allEntries.Add(entry1); + } - var entry2 = new PublicAccessEntry(content[1], content[0], content[2], new[] + //now remove a few rules from a few of the items and then add some more, this will put things 'out of order' which + //we need to verify our sort order is working for the relator + for (int i = 0; i < allEntries.Count; i++) { - new PublicAccessRule + //all the even ones + if (i % 2 == 0) { - RuleValue = "test", - RuleType = "RoleName" - }, - }); - repo.AddOrUpdate(entry2); - - unitOfWork.Commit(); + var rules = allEntries[i].Rules.ToArray(); + for (int j = 0; j < rules.Length; j++) + { + //all the even ones + if (j % 2 == 0) + { + allEntries[i].RemoveRule(rules[j]); + } + } + allEntries[i].AddRule("newrule" + i, "newrule" + i); + repo.AddOrUpdate(allEntries[i]); + unitOfWork.Commit(); + } + } var found = repo.GetAll().ToArray(); - Assert.AreEqual(2, found.Count()); + Assert.AreEqual(10, found.Length); + + foreach (var publicAccessEntry in found) + { + var matched = allEntries.First(x => x.Key == publicAccessEntry.Key); + + Assert.AreEqual(matched.Rules.Count(), publicAccessEntry.Rules.Count()); + } } } diff --git a/src/Umbraco.Tests/Services/ContentServiceTests.cs b/src/Umbraco.Tests/Services/ContentServiceTests.cs index 651e07a9a5..c598b272d7 100644 --- a/src/Umbraco.Tests/Services/ContentServiceTests.cs +++ b/src/Umbraco.Tests/Services/ContentServiceTests.cs @@ -36,11 +36,15 @@ namespace Umbraco.Tests.Services public override void Initialize() { base.Initialize(); + + VersionableRepositoryBase.ThrowOnWarning = true; } [TearDown] public override void TearDown() { + VersionableRepositoryBase.ThrowOnWarning = false; + base.TearDown(); } @@ -762,16 +766,26 @@ namespace Umbraco.Tests.Services var parent = ServiceContext.ContentService.GetById(NodeDto.NodeIdSeed + 1); ServiceContext.ContentService.Publish(parent);//Publishing root, so Text Page 2 can be updated. var subpage2 = contentService.GetById(NodeDto.NodeIdSeed + 3); + subpage2.Name = "Text Page 2 Updated"; subpage2.SetValue("author", "Jane Doe"); contentService.SaveAndPublishWithStatus(subpage2, 0);//NOTE New versions are only added between publish-state-changed, so publishing to ensure addition version. + subpage2.Name = "Text Page 2 Updated again"; + subpage2.SetValue("author", "Bob Hope"); + contentService.SaveAndPublishWithStatus(subpage2, 0);//NOTE New versions are only added between publish-state-changed, so publishing to ensure addition version. + // Act var versions = contentService.GetVersions(NodeDto.NodeIdSeed + 3).ToList(); // Assert Assert.That(versions.Any(), Is.True); Assert.That(versions.Count(), Is.GreaterThanOrEqualTo(2)); + + //ensure each version contains the correct property values + Assert.AreEqual("John Doe", versions[2].GetValue("author")); + Assert.AreEqual("Jane Doe", versions[1].GetValue("author")); + Assert.AreEqual("Bob Hope", versions[0].GetValue("author")); } [Test] diff --git a/src/Umbraco.Tests/Services/MediaServiceTests.cs b/src/Umbraco.Tests/Services/MediaServiceTests.cs index eec1f7ac6f..d9ecc66e8f 100644 --- a/src/Umbraco.Tests/Services/MediaServiceTests.cs +++ b/src/Umbraco.Tests/Services/MediaServiceTests.cs @@ -7,6 +7,7 @@ using Umbraco.Core; using Umbraco.Core.Models; using Umbraco.Core.Models.Rdbms; using Umbraco.Core.Persistence; +using Umbraco.Core.Persistence.DatabaseModelDefinitions; using Umbraco.Core.Persistence.Repositories; using Umbraco.Core.Persistence.UnitOfWork; using Umbraco.Tests.TestHelpers; @@ -30,6 +31,33 @@ namespace Umbraco.Tests.Services base.TearDown(); } + [Test] + public void Get_Paged_Children_With_Media_Type_Filter() + { + var mediaService = ServiceContext.MediaService; + var mediaType1 = MockedContentTypes.CreateImageMediaType("Image2"); + ServiceContext.ContentTypeService.Save(mediaType1); + var mediaType2 = MockedContentTypes.CreateImageMediaType("Image3"); + ServiceContext.ContentTypeService.Save(mediaType2); + + for (int i = 0; i < 10; i++) + { + var m1 = MockedMedia.CreateMediaImage(mediaType1, -1); + mediaService.Save(m1); + var m2 = MockedMedia.CreateMediaImage(mediaType2, -1); + mediaService.Save(m2); + } + + long total; + var result = ServiceContext.MediaService.GetPagedChildren(-1, 0, 11, out total, "SortOrder", Direction.Ascending, true, null, new[] {mediaType1.Id, mediaType2.Id}); + Assert.AreEqual(11, result.Count()); + Assert.AreEqual(20, total); + + result = ServiceContext.MediaService.GetPagedChildren(-1, 1, 11, out total, "SortOrder", Direction.Ascending, true, null, new[] { mediaType1.Id, mediaType2.Id }); + Assert.AreEqual(9, result.Count()); + Assert.AreEqual(20, total); + } + [Test] public void Can_Move_Media() { diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index d6f2393c13..eb126eb68e 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -159,6 +159,7 @@ + @@ -170,6 +171,7 @@ + diff --git a/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js b/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js index bbb78cd616..d1810f56ee 100644 --- a/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js +++ b/src/Umbraco.Web.UI.Client/src/common/resources/media.resource.js @@ -427,7 +427,9 @@ function mediaResource($q, $http, umbDataFormatter, umbRequestHelper) { * Retrieves all media children with types used as folders. * Uses the convention of looking for media items with mediaTypes ending in * *Folder so will match "Folder", "bannerFolder", "secureFolder" etc, - * + * + * NOTE: This will return a max of 500 folders, if more is required it needs to be paged + * * ##usage * 
           * mediaResource.getChildFolders(1234)
@@ -445,14 +447,15 @@ function mediaResource($q, $http, umbDataFormatter, umbRequestHelper) {
                 parentId = -1;
             }
 
+            //NOTE: This will return a max of 500 folders, if more is required it needs to be paged
             return umbRequestHelper.resourcePromise(
                   $http.get(
                         umbRequestHelper.getApiUrl(
                               "mediaApiBaseUrl",
                               "GetChildFolders",
-                              [
-                                    { id: parentId }
-                              ])),
+                            {
+                                id: parentId
+                            })),
                   'Failed to retrieve child folders for media item ' + parentId);
         },
 
diff --git a/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js b/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js
index e47f0663d8..0c18656c83 100644
--- a/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js
+++ b/src/Umbraco.Web.UI.Client/src/common/security/securityinterceptor.js
@@ -19,15 +19,20 @@ angular.module('umbraco.security.interceptor')
                     return promise;
                 }, function(originalResponse) {
                     // Intercept failed requests
+                    
+                    // Make sure we have the configuration of the request (don't we always?)
+                    var config = originalResponse.config ? originalResponse.config : {};
+                    
+                    // Make sure we have an object for the headers of the request
+                    var headers = config.headers ? config.headers : {};
 
-                    //Here we'll check if we should ignore the error, this will be based on an original header set
-                    var headers = originalResponse.config ? originalResponse.config.headers : {};
-                    if (headers["x-umb-ignore-error"] === "ignore") {
+                    //Here we'll check if we should ignore the error (either based on the original header set or the request configuration)
+                    if (headers["x-umb-ignore-error"] === "ignore" || config.umbIgnoreErrors === true) {
                         //exit/ignore
                         return promise;
                     }
                     var filtered = _.find(requestInterceptorFilter(), function(val) {
-                        return originalResponse.config.url.indexOf(val) > 0;
+                        return config.url.indexOf(val) > 0;
                     });
                     if (filtered) {
                         return promise;
@@ -99,4 +104,4 @@ angular.module('umbraco.security.interceptor')
     // We have to add the interceptor to the queue as a string because the interceptor depends upon service instances that are not available in the config block.
     .config(['$httpProvider', function ($httpProvider) {
         $httpProvider.responseInterceptors.push('securityInterceptor');
-    }]);
\ No newline at end of file
+    }]);
diff --git a/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js b/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js
index 20e5e3799b..b06a5ca8e3 100644
--- a/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js
+++ b/src/Umbraco.Web.UI.Client/src/common/services/mediatypehelper.service.js
@@ -7,6 +7,19 @@ function mediaTypeHelper(mediaTypeResource, $q) {
 
     var mediaTypeHelperService = {
 
+        isFolderType: function(mediaEntity) {
+            if (!mediaEntity) {
+                throw "mediaEntity is null";
+            }
+            if (!mediaEntity.contentTypeAlias) {
+                throw "mediaEntity.contentTypeAlias is null";
+            }
+
+            //if you create a media type, which has an alias that ends with ...Folder then its a folder: ex: "secureFolder", "bannerFolder", "Folder"
+            //this is the exact same logic that is performed in MediaController.GetChildFolders
+            return mediaEntity.contentTypeAlias.endsWith("Folder");
+        },
+
         getAllowedImagetypes: function (mediaId){
 				
             // Get All allowedTypes
diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html
index 16c5efe799..d09b04f97a 100644
--- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html
+++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.html
@@ -51,7 +51,7 @@
       
 
       
diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js
index be1c5856ae..b8ba4f880b 100644
--- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js
+++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/layouts/grid/grid.listviewlayout.controller.js
@@ -37,7 +37,8 @@
       function activate() {
           vm.itemsWithoutFolders = filterOutFolders($scope.items);
 
-          if($scope.entityType === 'media') {
+          //no need to make another REST/DB call if this data is not used when we are browsing the bin
+          if ($scope.entityType === 'media' && !vm.isRecycleBin) {
             mediaTypeHelper.getAllowedImagetypes(vm.nodeId).then(function (types) {
                 vm.acceptedMediatypes = types;
             });
diff --git a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js
index 8542157607..a9eab75a04 100644
--- a/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js
+++ b/src/Umbraco.Web.UI.Client/src/views/propertyeditors/listview/listview.controller.js
@@ -53,7 +53,9 @@ function listViewController($rootScope, $scope, $routeParams, $injector, $cookie
    $scope.isNew = false;
    $scope.actionInProgress = false;
    $scope.selection = [];
-   $scope.folders = [];
+   $scope.folders = [];   
+   //tracks if we've already loaded the folders for the current node
+   var foldersLoaded = false;
    $scope.listViewResultSet = {
       totalPages: 0,
       items: []
@@ -268,12 +270,13 @@ function listViewController($rootScope, $scope, $routeParams, $injector, $cookie
             });
          }
 
-         if ($scope.entityType === 'media') {
-
+         if (!foldersLoaded && $scope.entityType === 'media') {
+            //The folders aren't loaded - we only need to do this once since we're never changing node ids
             mediaResource.getChildFolders($scope.contentId)
                     .then(function (folders) {
                        $scope.folders = folders;
                        $scope.viewLoaded = true;
+                       foldersLoaded = true;
                     });
 
          } else {
diff --git a/src/Umbraco.Web.UI/config/umbracoSettings.Release.config b/src/Umbraco.Web.UI/config/umbracoSettings.Release.config
index 1dc6f8aae1..846622843b 100644
--- a/src/Umbraco.Web.UI/config/umbracoSettings.Release.config
+++ b/src/Umbraco.Web.UI/config/umbracoSettings.Release.config
@@ -51,7 +51,7 @@
     throw
 
     
-    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,html,htm,svg,php,htaccess
+    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,xhtml,html,htm,svg,php,htaccess
 
     
     Textstring
diff --git a/src/Umbraco.Web.UI/config/umbracoSettings.config b/src/Umbraco.Web.UI/config/umbracoSettings.config
index 42e21e1b3d..429cc30d20 100644
--- a/src/Umbraco.Web.UI/config/umbracoSettings.config
+++ b/src/Umbraco.Web.UI/config/umbracoSettings.config
@@ -100,7 +100,7 @@
     throw
     
     
-    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,html,htm,svg,php,htaccess
+    ashx,aspx,ascx,config,cshtml,vbhtml,asmx,air,axd,swf,xml,xhtml,html,htm,svg,php,htaccess
 
     
     Textstring
diff --git a/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml b/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
index 259b914a72..aeaf8bd49b 100644
--- a/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
+++ b/src/Umbraco.Web.UI/umbraco/config/lang/sv.xml
@@ -87,7 +87,7 @@
     Välj aktuell mapp
     Förhandsgranska
     Förhandsgranskning är avstängt på grund av att det inte finns någon mall tilldelad
-    Ångra
+    Annat
     Välj stil
     Visa stil
     Infoga tabell
@@ -171,6 +171,7 @@
     "dokumenttyper".]]>
     "mediatyper".]]>
     Välj typ och rubrik
+    Dokumenttyp utan sidmall
   
   
     Surfa på din webbplats
@@ -936,4 +937,4 @@
     Översättare
     Din profil
   
-
\ No newline at end of file
+
diff --git a/src/Umbraco.Web/Editors/EntityController.cs b/src/Umbraco.Web/Editors/EntityController.cs
index f5af30a57f..1bb0cdfdf6 100644
--- a/src/Umbraco.Web/Editors/EntityController.cs
+++ b/src/Umbraco.Web/Editors/EntityController.cs
@@ -5,30 +5,17 @@ using System.Globalization;
 using System.Net;
 using System.Text;
 using System.Web.Http;
-using System.Web.Http.ModelBinding;
 using AutoMapper;
-using ClientDependency.Core;
-using Examine.LuceneEngine;
-using Examine.LuceneEngine.Providers;
-using Newtonsoft.Json;
 using Umbraco.Core;
-using Umbraco.Core.Logging;
 using Umbraco.Core.Models.Membership;
-using Umbraco.Core.Services;
 using Umbraco.Web.Models.ContentEditing;
 using Umbraco.Web.Mvc;
 using System.Linq;
 using System.Net.Http;
-using Umbraco.Core.Models.EntityBase;
 using Umbraco.Core.Models;
-using Umbraco.Web.WebApi.Filters;
-using umbraco.cms.businesslogic.packager;
 using Constants = Umbraco.Core.Constants;
 using Examine;
-using Examine.LuceneEngine.SearchCriteria;
-using Examine.SearchCriteria;
 using Umbraco.Web.Dynamics;
-using umbraco;
 using System.Text.RegularExpressions;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using System.Web.Http.Controllers;
diff --git a/src/Umbraco.Web/Editors/MediaController.cs b/src/Umbraco.Web/Editors/MediaController.cs
index 3db1656c34..64c619ee6f 100644
--- a/src/Umbraco.Web/Editors/MediaController.cs
+++ b/src/Umbraco.Web/Editors/MediaController.cs
@@ -6,39 +6,30 @@ using System.IO;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Formatting;
-using System.Security.AccessControl;
 using System.Text;
 using System.Threading.Tasks;
-using System.Web;
 using System.Web.Http;
 using System.Web.Http.ModelBinding;
 using AutoMapper;
 using Umbraco.Core;
-using Umbraco.Core.Dynamics;
 using Umbraco.Core.IO;
 using Umbraco.Core.Logging;
 using Umbraco.Core.Models;
-using Umbraco.Core.Models.Editors;
 using Umbraco.Core.Models.Membership;
 using Umbraco.Core.Persistence.DatabaseModelDefinitions;
 using Umbraco.Core.Services;
-using Umbraco.Web.Models;
 using Umbraco.Web.Models.ContentEditing;
 using Umbraco.Web.Models.Mapping;
 using Umbraco.Web.Mvc;
 using Umbraco.Web.WebApi;
 using System.Linq;
-using System.Runtime.Serialization;
 using System.Text.RegularExpressions;
 using System.Web.Http.Controllers;
 using Examine;
 using Umbraco.Web.WebApi.Binders;
 using Umbraco.Web.WebApi.Filters;
-using umbraco;
-using umbraco.BusinessLogic.Actions;
 using Constants = Umbraco.Core.Constants;
 using Umbraco.Core.Configuration;
-using Umbraco.Core.Persistence.FaultHandling;
 using Umbraco.Web.UI;
 using Notification = Umbraco.Web.Models.ContentEditing.Notification;
 using Umbraco.Core.Persistence;
@@ -163,19 +154,48 @@ namespace Umbraco.Web.Editors
         }
 
         /// 
-        /// Returns media items known to be a container of other media items
+        /// Returns media items known to be of a "Folder" type
         /// 
         /// 
         /// 
+        [Obsolete("This is no longer used and shouldn't be because it performs poorly when there are a lot of media items")]
         [FilterAllowedOutgoingMedia(typeof(IEnumerable>))]
         public IEnumerable> GetChildFolders(int id = -1)
+        {
+            //we are only allowing a max of 500 to be returned here, if more is required it needs to be paged
+            var result = GetChildFolders(id, 1, 500);
+            return result.Items;
+        }
+
+        /// 
+        /// Returns a paged result of media items known to be of a "Folder" type
+        /// 
+        /// 
+        /// 
+        /// 
+        /// 
+        public PagedResult> GetChildFolders(int id, int pageNumber, int pageSize)
         {
             //Suggested convention for folder mediatypes - we can make this more or less complicated as long as we document it...
             //if you create a media type, which has an alias that ends with ...Folder then its a folder: ex: "secureFolder", "bannerFolder", "Folder"
-            var folderTypes = Services.ContentTypeService.GetAllMediaTypes().ToArray().Where(x => x.Alias.EndsWith("Folder")).Select(x => x.Id);
+            var folderTypes = Services.ContentTypeService
+                .GetAllMediaTypes()
+                .Where(x => x.Alias.EndsWith("Folder"))
+                .Select(x => x.Id)
+                .ToArray();
 
-            var children = (id < 0) ? Services.MediaService.GetRootMedia() : Services.MediaService.GetById(id).Children();
-            return children.Where(x => folderTypes.Contains(x.ContentTypeId)).Select(Mapper.Map>);
+            if (folderTypes.Length == 0)
+            {
+                return new PagedResult>(0, pageNumber, pageSize);
+            }
+
+            long total;
+            var children = Services.MediaService.GetPagedChildren(id, pageNumber - 1, pageSize, out total, "Name", Direction.Ascending, true, null, folderTypes.ToArray());
+            
+            return new PagedResult>(total, pageNumber, pageSize)
+            {
+                Items = children.Select(Mapper.Map>)
+            };
         }
 
         /// 
diff --git a/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs b/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs
index 5d888ef117..8b94271e29 100644
--- a/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs
+++ b/src/Umbraco.Web/HealthCheck/Checks/DataIntegrity/XmlDataIntegrityHealthCheck.cs
@@ -103,7 +103,7 @@ namespace Umbraco.Web.HealthCheck.Checks.DataIntegrity
         /// 
         private HealthCheckStatus CheckMedia()
         {
-            var total = _services.MediaService.Count();
+            var total = _services.MediaService.CountNotTrashed();
             var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
 
             //count entries
diff --git a/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs b/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs
index 66951d12f2..3d752a287b 100644
--- a/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs
+++ b/src/Umbraco.Web/WebServices/XmlDataIntegrityController.cs
@@ -51,7 +51,7 @@ namespace Umbraco.Web.WebServices
         [HttpGet]
         public bool CheckMediaXmlTable()
         {
-            var total = Services.MediaService.Count();
+            var total = Services.MediaService.CountNotTrashed();
             var mediaObjectType = Guid.Parse(Constants.ObjectTypes.Media);
             var subQuery = new Sql()
                 .Select("Count(*)")
diff --git a/src/Umbraco.Web/umbraco.presentation/content.cs b/src/Umbraco.Web/umbraco.presentation/content.cs
index edfffe03eb..c9d4308313 100644
--- a/src/Umbraco.Web/umbraco.presentation/content.cs
+++ b/src/Umbraco.Web/umbraco.presentation/content.cs
@@ -775,7 +775,6 @@ namespace umbraco
         internal void SaveXmlToFile()
         {
             LogHelper.Info("Save Xml to file...");
-
             try
             {
                 // ok to access _xmlContent here - capture (atomic + volatile), immutable anyway
@@ -793,7 +792,7 @@ namespace umbraco
                     Directory.CreateDirectory(directoryName);
 
                 // save
-                using (var fs = new FileStream(_xmlFileName, FileMode.Create, FileAccess.Write, FileShare.Read, bufferSize: 4096, useAsync: true))
+                using (var fs = new FileStream(_xmlFileName, FileMode.Create, FileAccess.Write, FileShare.Read))
                 {
                     SaveXmlToStream(xml, fs);
                 }
diff --git a/src/Umbraco.Web/umbraco.presentation/template.cs b/src/Umbraco.Web/umbraco.presentation/template.cs
index 4290ce2acf..50d3876f40 100644
--- a/src/Umbraco.Web/umbraco.presentation/template.cs
+++ b/src/Umbraco.Web/umbraco.presentation/template.cs
@@ -88,19 +88,20 @@ namespace umbraco
                 string originalPath = IOHelper.MapPath(VirtualPathUtility.ToAbsolute(MasterPageFile));
                 string copyPath = IOHelper.MapPath(VirtualPathUtility.ToAbsolute(path));
 
-                FileStream fs = new FileStream(originalPath, FileMode.Open, FileAccess.ReadWrite);
-                StreamReader f = new StreamReader(fs);
-                String newfile = f.ReadToEnd();
-                f.Close();
-                fs.Close();
+                string newFile;
+                using (var fs = new FileStream(originalPath, FileMode.Open, FileAccess.ReadWrite))
+                using (var f = new StreamReader(fs))
+                {
+                    newFile = f.ReadToEnd();                    
+                }
 
-                newfile = newfile.Replace("MasterPageFile=\"~/masterpages/", "MasterPageFile=\"");
+                newFile = newFile.Replace("MasterPageFile=\"~/masterpages/", "MasterPageFile=\"");
 
-                fs = new FileStream(copyPath, FileMode.Create, FileAccess.Write);
-
-                StreamWriter replacement = new StreamWriter(fs);
-                replacement.Write(newfile);
-                replacement.Close();
+                using (var fs = new FileStream(copyPath, FileMode.Create, FileAccess.Write))
+                using (var replacement = new StreamWriter(fs))
+                {
+                    replacement.Write(newFile);
+                }
             }
 
             return path;
diff --git a/src/umbraco.businesslogic/IO/SystemFiles.cs b/src/umbraco.businesslogic/IO/SystemFiles.cs
index 991ec0b29f..3c5841a31b 100644
--- a/src/umbraco.businesslogic/IO/SystemFiles.cs
+++ b/src/umbraco.businesslogic/IO/SystemFiles.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Configuration;
 using System.IO;
 using System.Linq;
@@ -54,7 +55,9 @@ namespace umbraco.IO
 			get { return Umbraco.Core.IO.SystemFiles.ContentCacheXml; }
 		}
 
-		public static bool ContentCacheXmlIsEphemeral
+        [Obsolete("This is not used and will be removed in future versions")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        public static bool ContentCacheXmlIsEphemeral
 		{
 			get { return Umbraco.Core.IO.SystemFiles.ContentCacheXmlStoredInCodeGen; }
 		}
diff --git a/src/umbraco.cms/businesslogic/propertytype/propertytype.cs b/src/umbraco.cms/businesslogic/propertytype/propertytype.cs
index 8d5f538fd9..63a4d80e91 100644
--- a/src/umbraco.cms/businesslogic/propertytype/propertytype.cs
+++ b/src/umbraco.cms/businesslogic/propertytype/propertytype.cs
@@ -56,7 +56,7 @@ namespace umbraco.cms.businesslogic.propertytype
         {
             var found = ApplicationContext.Current.DatabaseContext.Database
                 .SingleOrDefault(
-                    "Select mandatory, DataTypeId, propertyTypeGroupId, contentTypeId, sortOrder, alias, name, validationRegExp, description from cmsPropertyType where id=@id",
+                    "Select mandatory as mandatory, dataTypeId as dataTypeId, propertyTypeGroupId as propertyTypeGroupId, contentTypeId as contentTypeId, sortOrder as sortOrder, alias as alias, name as name, validationRegExp as validationRegExp, description as description from cmsPropertyType where id=@id",
                     new {id = id});
 
             if (found == null)
@@ -72,14 +72,13 @@ namespace umbraco.cms.businesslogic.propertytype
                 _tabId = _propertyTypeGroup;
             }
 
-            //Fixed issue U4-9493 Case issues
             _sortOrder = found.sortOrder;
-            _alias = found.Alias;
-            _name = found.Name;
+            _alias = found.alias;
+            _name = found.name;
             _validationRegExp = found.validationRegExp;
             _DataTypeId = found.dataTypeId;
             _contenttypeid = found.contentTypeId;
-            _description = found.Description;
+            _description = found.description;
         }
 
         #endregion