yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #7907 from umbraco/v8/bugfix/7868-nucache-null-parent

Browse Source 
Fix moving large amounts of content/media
This commit is contained in:
Shannon Deminick
2020-04-17 15:04:40 +10:00
committed by GitHub
parent d5e2c12685 d1524cdafe
commit 75655fd1ee
18 changed files with 699 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/Umbraco.Core/Constants-SqlTemplates.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace Umbraco.Core
{
public static partial class Constants
{
public static class SqlTemplates
{
public static class VersionableRepository
{
public const string GetVersionIds = "Umbraco.Core.VersionableRepository.GetVersionIds";
public const string GetVersion = "Umbraco.Core.VersionableRepository.GetVersion";
public const string GetVersions = "Umbraco.Core.VersionableRepository.GetVersions";
public const string EnsureUniqueNodeName = "Umbraco.Core.VersionableRepository.EnsureUniqueNodeName";
public const string GetSortOrder = "Umbraco.Core.VersionableRepository.GetSortOrder";
public const string GetParentNode = "Umbraco.Core.VersionableRepository.GetParentNode";
public const string GetReservedId = "Umbraco.Core.VersionableRepository.GetReservedId";
}
}
}
}

13
src/Umbraco.Core/ContentExtensions.cs
View File

@@ -60,6 +60,19 @@ namespace Umbraco.Core
#endregion
internal static bool IsMoving(this IContentBase entity)
{
// Check if this entity is being moved as a descendant as part of a bulk moving operations.
// When this occurs, only Path + Level + UpdateDate are being changed. In this case we can bypass a lot of the below
// operations which will make this whole operation go much faster. When moving we don't need to create
// new versions, etc... because we cannot roll this operation back anyways.
var isMoving = entity.IsPropertyDirty(nameof(entity.Path))
&& entity.IsPropertyDirty(nameof(entity.Level))
&& entity.IsPropertyDirty(nameof(entity.UpdateDate));
return isMoving;
}
/// <summary>
/// Removes characters that are not valid XML characters from all entity properties
/// of type string. See: http://stackoverflow.com/a/961504/5018

48
src/Umbraco.Core/Models/ContentDataIntegrityReport.cs Normal file
View File

@@ -0,0 +1,48 @@
using System.Collections.Generic;
using System.Linq;
namespace Umbraco.Core.Models
{
public class ContentDataIntegrityReport
{
public ContentDataIntegrityReport(IReadOnlyDictionary<int, ContentDataIntegrityReportEntry> detectedIssues)
{
DetectedIssues = detectedIssues;
}
public bool Ok => DetectedIssues.Count == 0 || DetectedIssues.Count == DetectedIssues.Values.Count(x => x.Fixed);
public IReadOnlyDictionary<int, ContentDataIntegrityReportEntry> DetectedIssues { get; }
public IReadOnlyDictionary<int, ContentDataIntegrityReportEntry> FixedIssues
=> DetectedIssues.Where(x => x.Value.Fixed).ToDictionary(x => x.Key, x => x.Value);
public enum IssueType
{
/// <summary>
/// The item's level and path are inconsistent with it's parent's path and level
/// </summary>
InvalidPathAndLevelByParentId,
/// <summary>
/// The item's path doesn't contain all required parts
/// </summary>
InvalidPathEmpty,
/// <summary>
/// The item's path parts are inconsistent with it's level value
/// </summary>
InvalidPathLevelMismatch,
/// <summary>
/// The item's path does not end with it's own ID
/// </summary>
InvalidPathById,
/// <summary>
/// The item's path does not have it's parent Id as the 2nd last entry
/// </summary>
InvalidPathByParentId,
}
}
}

13
src/Umbraco.Core/Models/ContentDataIntegrityReportEntry.cs Normal file
View File

@@ -0,0 +1,13 @@
namespace Umbraco.Core.Models
{
public class ContentDataIntegrityReportEntry
{
public ContentDataIntegrityReportEntry(ContentDataIntegrityReport.IssueType issueType)
{
IssueType = issueType;
}
public ContentDataIntegrityReport.IssueType IssueType { get; }
public bool Fixed { get; set; }
}
}

13
src/Umbraco.Core/Models/ContentDataIntegrityReportOptions.cs Normal file
View File

@@ -0,0 +1,13 @@
namespace Umbraco.Core.Models
{
public class ContentDataIntegrityReportOptions
{
/// <summary>
/// Set to true to try to automatically resolve data integrity issues
/// </summary>
public bool FixIssues { get; set; }
// TODO: We could define all sorts of options for the data integrity check like what to check for, what to fix, etc...
// things like Tag data consistency, etc...
}
}

3
src/Umbraco.Core/Persistence/Repositories/IContentRepository.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Entities;
using Umbraco.Core.Persistence.DatabaseModelDefinitions;
using Umbraco.Core.Persistence.Querying;
@@ -77,5 +78,7 @@ namespace Umbraco.Core.Persistence.Repositories
/// <remarks>Here, <paramref name="filter"/> can be null but <paramref name="ordering"/> cannot.</remarks>
IEnumerable<TEntity> GetPage(IQuery<TEntity> query, long pageIndex, int pageSize, out long totalRecords,
IQuery<TEntity> filter, Ordering ordering);
ContentDataIntegrityReport CheckDataIntegrity(ContentDataIntegrityReportOptions options);
}
}

133
src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs
View File

@@ -83,7 +83,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// gets all version ids, current first
public virtual IEnumerable<int> GetVersionIds(int nodeId, int maxRows)
{
var template = SqlContext.Templates.Get("Umbraco.Core.VersionableRepository.GetVersionIds", tsql =>
var template = SqlContext.Templates.Get(Constants.SqlTemplates.VersionableRepository.GetVersionIds, tsql =>
tsql.Select<ContentVersionDto>(x => x.Id)
.From<ContentVersionDto>()
.Where<ContentVersionDto>(x => x.NodeId == SqlTemplate.Arg<int>("nodeId"))
@@ -99,7 +99,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// TODO: test object node type?
// get the version we want to delete
var template = SqlContext.Templates.Get("Umbraco.Core.VersionableRepository.GetVersion", tsql =>
var template = SqlContext.Templates.Get(Constants.SqlTemplates.VersionableRepository.GetVersion, tsql =>
tsql.Select<ContentVersionDto>().From<ContentVersionDto>().Where<ContentVersionDto>(x => x.Id == SqlTemplate.Arg<int>("versionId"))
);
var versionDto = Database.Fetch<ContentVersionDto>(template.Sql(new { versionId })).FirstOrDefault();
@@ -121,7 +121,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// TODO: test object node type?
// get the versions we want to delete, excluding the current one
var template = SqlContext.Templates.Get("Umbraco.Core.VersionableRepository.GetVersions", tsql =>
var template = SqlContext.Templates.Get(Constants.SqlTemplates.VersionableRepository.GetVersions, tsql =>
tsql.Select<ContentVersionDto>().From<ContentVersionDto>().Where<ContentVersionDto>(x => x.NodeId == SqlTemplate.Arg<int>("nodeId") && !x.Current && x.VersionDate < SqlTemplate.Arg<DateTime>("versionDate"))
);
var versionDtos = Database.Fetch<ContentVersionDto>(template.Sql(new { nodeId, versionDate }));
@@ -403,7 +403,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
}
// content type alias is invariant
if(ordering.OrderBy.InvariantEquals("contentTypeAlias"))
if (ordering.OrderBy.InvariantEquals("contentTypeAlias"))
{
var joins = Sql()
.InnerJoin<ContentTypeDto>("ctype").On<ContentDto, ContentTypeDto>((content, contentType) => content.ContentTypeId == contentType.NodeId, aliasRight: "ctype");
@@ -477,6 +477,123 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
IQuery<TEntity> filter,
Ordering ordering);
public ContentDataIntegrityReport CheckDataIntegrity(ContentDataIntegrityReportOptions options)
{
var report = new Dictionary<int, ContentDataIntegrityReportEntry>();
var sql = SqlContext.Sql()
.Select<NodeDto>()
.From<NodeDto>()
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var nodesToRebuild = new Dictionary<int, List<NodeDto>>();
var validNodes = new Dictionary<int, NodeDto>();
var rootIds = new[] {Constants.System.Root, Constants.System.RecycleBinContent, Constants.System.RecycleBinMedia};
var currentParentIds = new HashSet<int>(rootIds);
var prevParentIds = currentParentIds;
var lastLevel = -1;
// use a forward cursor (query)
foreach (var node in Database.Query<NodeDto>(sql))
{
if (node.Level != lastLevel)
{
// changing levels
prevParentIds = currentParentIds;
currentParentIds = null;
lastLevel = node.Level;
}
if (currentParentIds == null)
{
// we're reset
currentParentIds = new HashSet<int>();
}
currentParentIds.Add(node.NodeId);
// paths parts without the roots
var pathParts = node.Path.Split(',').Where(x => !rootIds.Contains(int.Parse(x))).ToArray();
if (!prevParentIds.Contains(node.ParentId))
{
// invalid, this will be because the level is wrong (which prob means path is wrong too)
report.Add(node.NodeId, new ContentDataIntegrityReportEntry(ContentDataIntegrityReport.IssueType.InvalidPathAndLevelByParentId));
AppendNodeToFix(nodesToRebuild, node);
}
else if (pathParts.Length == 0)
{
// invalid path
report.Add(node.NodeId, new ContentDataIntegrityReportEntry(ContentDataIntegrityReport.IssueType.InvalidPathEmpty));
AppendNodeToFix(nodesToRebuild, node);
}
else if (pathParts.Length != node.Level)
{
// invalid, either path or level is wrong
report.Add(node.NodeId, new ContentDataIntegrityReportEntry(ContentDataIntegrityReport.IssueType.InvalidPathLevelMismatch));
AppendNodeToFix(nodesToRebuild, node);
}
else if (pathParts[pathParts.Length - 1] != node.NodeId.ToString())
{
// invalid path
report.Add(node.NodeId, new ContentDataIntegrityReportEntry(ContentDataIntegrityReport.IssueType.InvalidPathById));
AppendNodeToFix(nodesToRebuild, node);
}
else if (!rootIds.Contains(node.ParentId) && pathParts[pathParts.Length - 2] != node.ParentId.ToString())
{
// invalid path
report.Add(node.NodeId, new ContentDataIntegrityReportEntry(ContentDataIntegrityReport.IssueType.InvalidPathByParentId));
AppendNodeToFix(nodesToRebuild, node);
}
else
{
// it's valid!
// don't track unless we are configured to fix
if (options.FixIssues)
validNodes.Add(node.NodeId, node);
}
}
var updated = new List<NodeDto>();
if (options.FixIssues)
{
// iterate all valid nodes to see if these are parents for invalid nodes
foreach (var (nodeId, node) in validNodes)
{
if (!nodesToRebuild.TryGetValue(nodeId, out var invalidNodes)) continue;
// now we can try to rebuild the invalid paths.
foreach (var invalidNode in invalidNodes)
{
invalidNode.Level = (short)(node.Level + 1);
invalidNode.Path = node.Path + "," + invalidNode.NodeId;
updated.Add(invalidNode);
}
}
foreach (var node in updated)
{
Database.Update(node);
if (report.TryGetValue(node.NodeId, out var entry))
entry.Fixed = true;
}
}
return new ContentDataIntegrityReport(report);
}
private static void AppendNodeToFix(IDictionary<int, List<NodeDto>> nodesToRebuild, NodeDto node)
{
if (nodesToRebuild.TryGetValue(node.ParentId, out var childIds))
childIds.Add(node);
else
nodesToRebuild[node.ParentId] = new List<NodeDto> { node };
}
// here, filter can be null and ordering cannot
protected IEnumerable<TEntity> GetPage<TDto>(IQuery<TEntity> query,
long pageIndex, int pageSize, out long totalRecords,
@@ -770,7 +887,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected virtual string EnsureUniqueNodeName(int parentId, string nodeName, int id = 0)
{
var template = SqlContext.Templates.Get("Umbraco.Core.VersionableRepository.EnsureUniqueNodeName", tsql => tsql
var template = SqlContext.Templates.Get(Constants.SqlTemplates.VersionableRepository.EnsureUniqueNodeName, tsql => tsql
.Select<NodeDto>(x => Alias(x.NodeId, "id"), x => Alias(x.Text, "name"))
.From<NodeDto>()
.Where<NodeDto>(x => x.NodeObjectType == SqlTemplate.Arg<Guid>("nodeObjectType") && x.ParentId == SqlTemplate.Arg<int>("parentId")));
@@ -783,7 +900,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected virtual int GetNewChildSortOrder(int parentId, int first)
{
var template = SqlContext.Templates.Get("Umbraco.Core.VersionableRepository.GetSortOrder", tsql =>
var template = SqlContext.Templates.Get(Constants.SqlTemplates.VersionableRepository.GetSortOrder, tsql =>
tsql.Select($"COALESCE(MAX(sortOrder),{first - 1})").From<NodeDto>().Where<NodeDto>(x => x.ParentId == SqlTemplate.Arg<int>("parentId") && x.NodeObjectType == NodeObjectTypeId)
);
@@ -792,7 +909,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected virtual NodeDto GetParentNodeDto(int parentId)
{
var template = SqlContext.Templates.Get("Umbraco.Core.VersionableRepository.GetParentNode", tsql =>
var template = SqlContext.Templates.Get(Constants.SqlTemplates.VersionableRepository.GetParentNode, tsql =>
tsql.Select<NodeDto>().From<NodeDto>().Where<NodeDto>(x => x.NodeId == SqlTemplate.Arg<int>("parentId"))
);
@@ -801,7 +918,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected virtual int GetReservedId(Guid uniqueId)
{
var template = SqlContext.Templates.Get("Umbraco.Core.VersionableRepository.GetReservedId", tsql =>
var template = SqlContext.Templates.Get(Constants.SqlTemplates.VersionableRepository.GetReservedId, tsql =>
tsql.Select<NodeDto>(x => x.NodeId).From<NodeDto>().Where<NodeDto>(x => x.UniqueId == SqlTemplate.Arg<Guid>("uniqueId") && x.NodeObjectType == Constants.ObjectTypes.IdReservation)
);
var id = Database.ExecuteScalar<int?>(template.Sql(new { uniqueId = uniqueId }));

305
src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs
View File

@@ -321,7 +321,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
.InnerJoin<DocumentVersionDto>()
.On<ContentVersionDto, DocumentVersionDto>((c, d) => c.Id == d.Id)
.Where<ContentVersionDto>(x => x.NodeId == SqlTemplate.Arg<int>("nodeId") && !x.Current && x.VersionDate < SqlTemplate.Arg<DateTime>("versionDate"))
.Where<DocumentVersionDto>( x => !x.Published)
.Where<DocumentVersionDto>(x => !x.Published)
);
var versionDtos = Database.Fetch<ContentVersionDto>(template.Sql(new { nodeId, versionDate }));
foreach (var versionDto in versionDtos)
@@ -519,8 +519,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected override void PersistUpdatedItem(IContent entity)
{
var entityBase = entity as EntityBase;
var isEntityDirty = entityBase != null && entityBase.IsDirty();
var isEntityDirty = entity.IsDirty();
// check if we need to make any database changes at all
if ((entity.PublishedState == PublishedState.Published || entity.PublishedState == PublishedState.Unpublished)
@@ -535,29 +534,41 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// update
entity.UpdatingEntity();
// Check if this entity is being moved as a descendant as part of a bulk moving operations.
// In this case we can bypass a lot of the below operations which will make this whole operation go much faster.
// When moving we don't need to create new versions, etc... because we cannot roll this operation back anyways.
var isMoving = entity.IsMoving();
// TODO: I'm sure we can also detect a "Copy" (of a descendant) operation and probably perform similar checks below.
// There is probably more stuff that would be required for copying but I'm sure not all of this logic would be, we could more than likely boost
// copy performance by 95% just like we did for Move
var publishing = entity.PublishedState == PublishedState.Publishing;
// check if we need to create a new version
if (publishing && entity.PublishedVersionId > 0)
if (!isMoving)
{
// published version is not published anymore
Database.Execute(Sql().Update<DocumentVersionDto>(u => u.Set(x => x.Published, false)).Where<DocumentVersionDto>(x => x.Id == entity.PublishedVersionId));
}
// check if we need to create a new version
if (publishing && entity.PublishedVersionId > 0)
{
// published version is not published anymore
Database.Execute(Sql().Update<DocumentVersionDto>(u => u.Set(x => x.Published, false)).Where<DocumentVersionDto>(x => x.Id == entity.PublishedVersionId));
}
// sanitize names
SanitizeNames(entity, publishing);
// sanitize names
SanitizeNames(entity, publishing);
// ensure that strings don't contain characters that are invalid in xml
// TODO: do we really want to keep doing this here?
entity.SanitizeEntityPropertiesForXmlStorage();
// ensure that strings don't contain characters that are invalid in xml
// TODO: do we really want to keep doing this here?
entity.SanitizeEntityPropertiesForXmlStorage();
// if parent has changed, get path, level and sort order
if (entity.IsPropertyDirty("ParentId"))
{
var parent = GetParentNodeDto(entity.ParentId);
entity.Path = string.Concat(parent.Path, ",", entity.Id);
entity.Level = parent.Level + 1;
entity.SortOrder = GetNewChildSortOrder(entity.ParentId, 0);
// if parent has changed, get path, level and sort order
if (entity.IsPropertyDirty("ParentId"))
{
var parent = GetParentNodeDto(entity.ParentId);
entity.Path = string.Concat(parent.Path, ",", entity.Id);
entity.Level = parent.Level + 1;
entity.SortOrder = GetNewChildSortOrder(entity.ParentId, 0);
}
}
// create the dto
@@ -568,146 +579,152 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
nodeDto.ValidatePathWithException();
Database.Update(nodeDto);
// update the content dto
Database.Update(dto.ContentDto);
// update the content & document version dtos
var contentVersionDto = dto.DocumentVersionDto.ContentVersionDto;
var documentVersionDto = dto.DocumentVersionDto;
if (publishing)
if (!isMoving)
{
documentVersionDto.Published = true; // now published
contentVersionDto.Current = false; // no more current
}
Database.Update(contentVersionDto);
Database.Update(documentVersionDto);
// update the content dto
Database.Update(dto.ContentDto);
// and, if publishing, insert new content & document version dtos
if (publishing)
{
entity.PublishedVersionId = entity.VersionId;
contentVersionDto.Id = 0; // want a new id
contentVersionDto.Current = true; // current version
contentVersionDto.Text = entity.Name;
Database.Insert(contentVersionDto);
entity.VersionId = documentVersionDto.Id = contentVersionDto.Id; // get the new id
documentVersionDto.Published = false; // non-published version
Database.Insert(documentVersionDto);
}
// replace the property data (rather than updating)
// only need to delete for the version that existed, the new version (if any) has no property data yet
var versionToDelete = publishing ? entity.PublishedVersionId : entity.VersionId;
var deletePropertyDataSql = Sql().Delete<PropertyDataDto>().Where<PropertyDataDto>(x => x.VersionId == versionToDelete);
Database.Execute(deletePropertyDataSql);
// insert property data
var propertyDataDtos = PropertyFactory.BuildDtos(entity.ContentType.Variations, entity.VersionId, publishing ? entity.PublishedVersionId : 0,
entity.Properties, LanguageRepository, out var edited, out var editedCultures);
foreach (var propertyDataDto in propertyDataDtos)
Database.Insert(propertyDataDto);
// if !publishing, we may have a new name != current publish name,
// also impacts 'edited'
if (!publishing && entity.PublishName != entity.Name)
edited = true;
if (entity.ContentType.VariesByCulture())
{
// bump dates to align cultures to version
// update the content & document version dtos
var contentVersionDto = dto.DocumentVersionDto.ContentVersionDto;
var documentVersionDto = dto.DocumentVersionDto;
if (publishing)
entity.AdjustDates(contentVersionDto.VersionDate);
{
documentVersionDto.Published = true; // now published
contentVersionDto.Current = false; // no more current
}
Database.Update(contentVersionDto);
Database.Update(documentVersionDto);
// names also impact 'edited'
// ReSharper disable once UseDeconstruction
foreach (var cultureInfo in entity.CultureInfos)
if (cultureInfo.Name != entity.GetPublishName(cultureInfo.Culture))
{
edited = true;
(editedCultures ?? (editedCultures = new HashSet<string>(StringComparer.OrdinalIgnoreCase))).Add(cultureInfo.Culture);
// and, if publishing, insert new content & document version dtos
if (publishing)
{
entity.PublishedVersionId = entity.VersionId;
// TODO: change tracking
// at the moment, we don't do any dirty tracking on property values, so we don't know whether the
// culture has just been edited or not, so we don't update its update date - that date only changes
// when the name is set, and it all works because the controller does it - but, if someone uses a
// service to change a property value and save (without setting name), the update date does not change.
}
contentVersionDto.Id = 0; // want a new id
contentVersionDto.Current = true; // current version
contentVersionDto.Text = entity.Name;
Database.Insert(contentVersionDto);
entity.VersionId = documentVersionDto.Id = contentVersionDto.Id; // get the new id
// replace the content version variations (rather than updating)
documentVersionDto.Published = false; // non-published version
Database.Insert(documentVersionDto);
}
// replace the property data (rather than updating)
// only need to delete for the version that existed, the new version (if any) has no property data yet
var deleteContentVariations = Sql().Delete<ContentVersionCultureVariationDto>().Where<ContentVersionCultureVariationDto>(x => x.VersionId == versionToDelete);
Database.Execute(deleteContentVariations);
var versionToDelete = publishing ? entity.PublishedVersionId : entity.VersionId;
var deletePropertyDataSql = Sql().Delete<PropertyDataDto>().Where<PropertyDataDto>(x => x.VersionId == versionToDelete);
Database.Execute(deletePropertyDataSql);
// replace the document version variations (rather than updating)
var deleteDocumentVariations = Sql().Delete<DocumentCultureVariationDto>().Where<DocumentCultureVariationDto>(x => x.NodeId == entity.Id);
Database.Execute(deleteDocumentVariations);
// insert property data
var propertyDataDtos = PropertyFactory.BuildDtos(entity.ContentType.Variations, entity.VersionId, publishing ? entity.PublishedVersionId : 0,
entity.Properties, LanguageRepository, out var edited, out var editedCultures);
foreach (var propertyDataDto in propertyDataDtos)
Database.Insert(propertyDataDto);
// TODO: NPoco InsertBulk issue?
// we should use the native NPoco InsertBulk here but it causes problems (not sure exactly all scenarios)
// but by using SQL Server and updating a variants name will cause: Unable to cast object of type
// 'Umbraco.Core.Persistence.FaultHandling.RetryDbConnection' to type 'System.Data.SqlClient.SqlConnection'.
// (same in PersistNewItem above)
// if !publishing, we may have a new name != current publish name,
// also impacts 'edited'
if (!publishing && entity.PublishName != entity.Name)
edited = true;
// insert content variations
Database.BulkInsertRecords(GetContentVariationDtos(entity, publishing));
if (entity.ContentType.VariesByCulture())
{
// bump dates to align cultures to version
if (publishing)
entity.AdjustDates(contentVersionDto.VersionDate);
// insert document variations
Database.BulkInsertRecords(GetDocumentVariationDtos(entity, editedCultures));
// names also impact 'edited'
// ReSharper disable once UseDeconstruction
foreach (var cultureInfo in entity.CultureInfos)
if (cultureInfo.Name != entity.GetPublishName(cultureInfo.Culture))
{
edited = true;
(editedCultures ?? (editedCultures = new HashSet<string>(StringComparer.OrdinalIgnoreCase))).Add(cultureInfo.Culture);
// TODO: change tracking
// at the moment, we don't do any dirty tracking on property values, so we don't know whether the
// culture has just been edited or not, so we don't update its update date - that date only changes
// when the name is set, and it all works because the controller does it - but, if someone uses a
// service to change a property value and save (without setting name), the update date does not change.
}
// replace the content version variations (rather than updating)
// only need to delete for the version that existed, the new version (if any) has no property data yet
var deleteContentVariations = Sql().Delete<ContentVersionCultureVariationDto>().Where<ContentVersionCultureVariationDto>(x => x.VersionId == versionToDelete);
Database.Execute(deleteContentVariations);
// replace the document version variations (rather than updating)
var deleteDocumentVariations = Sql().Delete<DocumentCultureVariationDto>().Where<DocumentCultureVariationDto>(x => x.NodeId == entity.Id);
Database.Execute(deleteDocumentVariations);
// TODO: NPoco InsertBulk issue?
// we should use the native NPoco InsertBulk here but it causes problems (not sure exactly all scenarios)
// but by using SQL Server and updating a variants name will cause: Unable to cast object of type
// 'Umbraco.Core.Persistence.FaultHandling.RetryDbConnection' to type 'System.Data.SqlClient.SqlConnection'.
// (same in PersistNewItem above)
// insert content variations
Database.BulkInsertRecords(GetContentVariationDtos(entity, publishing));
// insert document variations
Database.BulkInsertRecords(GetDocumentVariationDtos(entity, editedCultures));
}
// refresh content
entity.SetCultureEdited(editedCultures);
// update the document dto
// at that point, when un/publishing, the entity still has its old Published value
// so we need to explicitly update the dto to persist the correct value
if (entity.PublishedState == PublishedState.Publishing)
dto.Published = true;
else if (entity.PublishedState == PublishedState.Unpublishing)
dto.Published = false;
entity.Edited = dto.Edited = !dto.Published || edited; // if not published, always edited
Database.Update(dto);
//update the schedule
if (entity.IsPropertyDirty("ContentSchedule"))
PersistContentSchedule(entity, true);
// if entity is publishing, update tags, else leave tags there
// means that implicitly unpublished, or trashed, entities *still* have tags in db
if (entity.PublishedState == PublishedState.Publishing)
SetEntityTags(entity, _tagRepository);
}
// refresh content
entity.SetCultureEdited(editedCultures);
// update the document dto
// at that point, when un/publishing, the entity still has its old Published value
// so we need to explicitly update the dto to persist the correct value
if (entity.PublishedState == PublishedState.Publishing)
dto.Published = true;
else if (entity.PublishedState == PublishedState.Unpublishing)
dto.Published = false;
entity.Edited = dto.Edited = !dto.Published || edited; // if not published, always edited
Database.Update(dto);
//update the schedule
if (entity.IsPropertyDirty("ContentSchedule"))
PersistContentSchedule(entity, true);
// if entity is publishing, update tags, else leave tags there
// means that implicitly unpublished, or trashed, entities *still* have tags in db
if (entity.PublishedState == PublishedState.Publishing)
SetEntityTags(entity, _tagRepository);
// trigger here, before we reset Published etc
OnUowRefreshedEntity(new ScopedEntityEventArgs(AmbientScope, entity));
// flip the entity's published property
// this also flips its published state
if (entity.PublishedState == PublishedState.Publishing)
if (!isMoving)
{
entity.Published = true;
entity.PublishTemplateId = entity.TemplateId;
entity.PublisherId = entity.WriterId;
entity.PublishName = entity.Name;
entity.PublishDate = entity.UpdateDate;
// flip the entity's published property
// this also flips its published state
if (entity.PublishedState == PublishedState.Publishing)
{
entity.Published = true;
entity.PublishTemplateId = entity.TemplateId;
entity.PublisherId = entity.WriterId;
entity.PublishName = entity.Name;
entity.PublishDate = entity.UpdateDate;
SetEntityTags(entity, _tagRepository);
SetEntityTags(entity, _tagRepository);
}
else if (entity.PublishedState == PublishedState.Unpublishing)
{
entity.Published = false;
entity.PublishTemplateId = null;
entity.PublisherId = null;
entity.PublishName = null;
entity.PublishDate = null;
ClearEntityTags(entity, _tagRepository);
}
PersistRelations(entity);
// TODO: note re. tags: explicitly unpublished entities have cleared tags, but masked or trashed entities *still* have tags in the db - so what?
}
else if (entity.PublishedState == PublishedState.Unpublishing)
{
entity.Published = false;
entity.PublishTemplateId = null;
entity.PublisherId = null;
entity.PublishName = null;
entity.PublishDate = null;
ClearEntityTags(entity, _tagRepository);
}
PersistRelations(entity);
// TODO: note re. tags: explicitly unpublished entities have cleared tags, but masked or trashed entities *still* have tags in the db - so what?
entity.ResetDirtyProperties();
@@ -1183,7 +1200,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
if (temp.Template2Id.HasValue && templates.ContainsKey(temp.Template2Id.Value))
temp.Content.PublishTemplateId = temp.Template2Id;
}
// set properties
if (loadProperties)
@@ -1216,7 +1233,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
SetVariations(temp.Content, contentVariations, documentVariations);
}
}
foreach (var c in content)
@@ -1430,7 +1447,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
yield return dto;
}
}
private class ContentVariation

78
src/Umbraco.Core/Persistence/Repositories/Implement/MediaRepository.cs
View File

@@ -219,7 +219,6 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected override void PersistNewItem(IMedia entity)
{
var media = (Models.Media) entity;
entity.AddingEntity();
// ensure unique name on the same level
@@ -274,15 +273,15 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
contentVersionDto.NodeId = nodeDto.NodeId;
contentVersionDto.Current = true;
Database.Insert(contentVersionDto);
media.VersionId = contentVersionDto.Id;
entity.VersionId = contentVersionDto.Id;
// persist the media version dto
var mediaVersionDto = dto.MediaVersionDto;
mediaVersionDto.Id = media.VersionId;
mediaVersionDto.Id = entity.VersionId;
Database.Insert(mediaVersionDto);
// persist the property data
var propertyDataDtos = PropertyFactory.BuildDtos(media.ContentType.Variations, media.VersionId, 0, entity.Properties, LanguageRepository, out _, out _);
var propertyDataDtos = PropertyFactory.BuildDtos(entity.ContentType.Variations, entity.VersionId, 0, entity.Properties, LanguageRepository, out _, out _);
foreach (var propertyDataDto in propertyDataDtos)
Database.Insert(propertyDataDto);
@@ -298,26 +297,32 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected override void PersistUpdatedItem(IMedia entity)
{
var media = (Models.Media) entity;
// update
media.UpdatingEntity();
entity.UpdatingEntity();
// ensure unique name on the same level
entity.Name = EnsureUniqueNodeName(entity.ParentId, entity.Name, entity.Id);
// Check if this entity is being moved as a descendant as part of a bulk moving operations.
// In this case we can bypass a lot of the below operations which will make this whole operation go much faster.
// When moving we don't need to create new versions, etc... because we cannot roll this operation back anyways.
var isMoving = entity.IsMoving();
// ensure that strings don't contain characters that are invalid in xml
// TODO: do we really want to keep doing this here?
entity.SanitizeEntityPropertiesForXmlStorage();
// if parent has changed, get path, level and sort order
if (entity.IsPropertyDirty("ParentId"))
if (!isMoving)
{
var parent = GetParentNodeDto(entity.ParentId);
// ensure unique name on the same level
entity.Name = EnsureUniqueNodeName(entity.ParentId, entity.Name, entity.Id);
entity.Path = string.Concat(parent.Path, ",", entity.Id);
entity.Level = parent.Level + 1;
entity.SortOrder = GetNewChildSortOrder(entity.ParentId, 0);
// ensure that strings don't contain characters that are invalid in xml
// TODO: do we really want to keep doing this here?
entity.SanitizeEntityPropertiesForXmlStorage();
// if parent has changed, get path, level and sort order
if (entity.IsPropertyDirty(nameof(entity.ParentId)))
{
var parent = GetParentNodeDto(entity.ParentId);
entity.Path = string.Concat(parent.Path, ",", entity.Id);
entity.Level = parent.Level + 1;
entity.SortOrder = GetNewChildSortOrder(entity.ParentId, 0);
}
}
// create the dto
@@ -328,26 +333,29 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
nodeDto.ValidatePathWithException();
Database.Update(nodeDto);
// update the content dto
Database.Update(dto.ContentDto);
if (!isMoving)
{
// update the content dto
Database.Update(dto.ContentDto);
// update the content & media version dtos
var contentVersionDto = dto.MediaVersionDto.ContentVersionDto;
var mediaVersionDto = dto.MediaVersionDto;
contentVersionDto.Current = true;
Database.Update(contentVersionDto);
Database.Update(mediaVersionDto);
// update the content & media version dtos
var contentVersionDto = dto.MediaVersionDto.ContentVersionDto;
var mediaVersionDto = dto.MediaVersionDto;
contentVersionDto.Current = true;
Database.Update(contentVersionDto);
Database.Update(mediaVersionDto);
// replace the property data
var deletePropertyDataSql = SqlContext.Sql().Delete<PropertyDataDto>().Where<PropertyDataDto>(x => x.VersionId == media.VersionId);
Database.Execute(deletePropertyDataSql);
var propertyDataDtos = PropertyFactory.BuildDtos(media.ContentType.Variations, media.VersionId, 0, entity.Properties, LanguageRepository, out _, out _);
foreach (var propertyDataDto in propertyDataDtos)
Database.Insert(propertyDataDto);
// replace the property data
var deletePropertyDataSql = SqlContext.Sql().Delete<PropertyDataDto>().Where<PropertyDataDto>(x => x.VersionId == entity.VersionId);
Database.Execute(deletePropertyDataSql);
var propertyDataDtos = PropertyFactory.BuildDtos(entity.ContentType.Variations, entity.VersionId, 0, entity.Properties, LanguageRepository, out _, out _);
foreach (var propertyDataDto in propertyDataDtos)
Database.Insert(propertyDataDto);
SetEntityTags(entity, _tagRepository);
SetEntityTags(entity, _tagRepository);
PersistRelations(entity);
PersistRelations(entity);
}
OnUowRefreshedEntity(new ScopedEntityEventArgs(AmbientScope, entity));

2
src/Umbraco.Core/Services/IContentService.cs
View File

@@ -526,6 +526,6 @@ namespace Umbraco.Core.Services
OperationResult Rollback(int id, int versionId, string culture = "*", int userId = Constants.Security.SuperUserId);
#endregion
}
}

11
src/Umbraco.Core/Services/IContentServiceBase.cs
View File

@@ -1,9 +1,16 @@
namespace Umbraco.Core.Services
using Umbraco.Core.Models;
namespace Umbraco.Core.Services
{
/// <summary>
/// Placeholder for sharing logic between the content, media (and member) services
/// TODO: Start sharing the logic!
/// </summary>
public interface IContentServiceBase : IService
{ }
{
/// <summary>
/// Checks/fixes the data integrity of node paths/levels stored in the database
/// </summary>
ContentDataIntegrityReport CheckDataIntegrity(ContentDataIntegrityReportOptions options);
}
}

61
src/Umbraco.Core/Services/Implement/ContentService.cs
View File

@@ -601,23 +601,27 @@ namespace Umbraco.Core.Services.Implement
totalChildren = 0;
return Enumerable.Empty<IContent>();
}
return GetPagedDescendantsLocked(contentPath[0].Path, pageIndex, pageSize, out totalChildren, filter, ordering);
return GetPagedLocked(GetPagedDescendantQuery(contentPath[0].Path), pageIndex, pageSize, out totalChildren, filter, ordering);
}
return GetPagedDescendantsLocked(null, pageIndex, pageSize, out totalChildren, filter, ordering);
return GetPagedLocked(null, pageIndex, pageSize, out totalChildren, filter, ordering);
}
}
private IEnumerable<IContent> GetPagedDescendantsLocked(string contentPath, long pageIndex, int pageSize, out long totalChildren,
private IQuery<IContent> GetPagedDescendantQuery(string contentPath)
{
var query = Query<IContent>();
if (!contentPath.IsNullOrWhiteSpace())
query.Where(x => x.Path.SqlStartsWith($"{contentPath},", TextColumnType.NVarchar));
return query;
}
private IEnumerable<IContent> GetPagedLocked(IQuery<IContent> query, long pageIndex, int pageSize, out long totalChildren,
IQuery<IContent> filter, Ordering ordering)
{
if (pageIndex < 0) throw new ArgumentOutOfRangeException(nameof(pageIndex));
if (pageSize <= 0) throw new ArgumentOutOfRangeException(nameof(pageSize));
if (ordering == null) throw new ArgumentNullException(nameof(ordering));
var query = Query<IContent>();
if (!contentPath.IsNullOrWhiteSpace())
query.Where(x => x.Path.SqlStartsWith($"{contentPath},", TextColumnType.NVarchar));
return _documentRepository.GetPage(query, pageIndex, pageSize, out totalChildren, filter, ordering);
}
@@ -1866,7 +1870,7 @@ namespace Umbraco.Core.Services.Implement
public OperationResult MoveToRecycleBin(IContent content, int userId)
{
var evtMsgs = EventMessagesFactory.Get();
var moves = new List<Tuple<IContent, string>>();
var moves = new List<(IContent, string)>();
using (var scope = ScopeProvider.CreateScope())
{
@@ -1925,7 +1929,7 @@ namespace Umbraco.Core.Services.Implement
return;
}
var moves = new List<Tuple<IContent, string>>();
var moves = new List<(IContent, string)>();
using (var scope = ScopeProvider.CreateScope())
{
@@ -1978,7 +1982,7 @@ namespace Umbraco.Core.Services.Implement
// MUST be called from within WriteLock
// trash indicates whether we are trashing, un-trashing, or not changing anything
private void PerformMoveLocked(IContent content, int parentId, IContent parent, int userId,
ICollection<Tuple<IContent, string>> moves,
ICollection<(IContent, string)> moves,
bool? trash)
{
content.WriterId = userId;
@@ -1990,7 +1994,7 @@ namespace Umbraco.Core.Services.Implement
var paths = new Dictionary<int, string>();
moves.Add(Tuple.Create(content, content.Path)); // capture original path
moves.Add((content, content.Path)); // capture original path
//need to store the original path to lookup descendants based on it below
var originalPath = content.Path;
@@ -2007,20 +2011,24 @@ namespace Umbraco.Core.Services.Implement
paths[content.Id] = (parent == null ? (parentId == Constants.System.RecycleBinContent ? "-1,-20" : Constants.System.RootString) : parent.Path) + "," + content.Id;
const int pageSize = 500;
var total = long.MaxValue;
while (total > 0)
var query = GetPagedDescendantQuery(originalPath);
long total;
do
{
var descendants = GetPagedDescendantsLocked(originalPath, 0, pageSize, out total, null, Ordering.By("Path", Direction.Ascending));
// We always page a page 0 because for each page, we are moving the result so the resulting total will be reduced
var descendants = GetPagedLocked(query, 0, pageSize, out total, null, Ordering.By("Path", Direction.Ascending));
foreach (var descendant in descendants)
{
moves.Add(Tuple.Create(descendant, descendant.Path)); // capture original path
moves.Add((descendant, descendant.Path)); // capture original path
// update path and level since we do not update parentId
descendant.Path = paths[descendant.Id] = paths[descendant.ParentId] + "," + descendant.Id;
descendant.Level += levelDelta;
PerformMoveContentLocked(descendant, userId, trash);
}
}
} while (total > pageSize);
}
@@ -2375,6 +2383,25 @@ namespace Umbraco.Core.Services.Implement
return OperationResult.Succeed(evtMsgs);
}
public ContentDataIntegrityReport CheckDataIntegrity(ContentDataIntegrityReportOptions options)
{
using (var scope = ScopeProvider.CreateScope(autoComplete: true))
{
scope.WriteLock(Constants.Locks.ContentTree);
var report = _documentRepository.CheckDataIntegrity(options);
if (report.FixedIssues.Count > 0)
{
//The event args needs a content item so we'll make a fake one with enough properties to not cause a null ref
var root = new Content("root", -1, new ContentType(-1)) {Id = -1, Key = Guid.Empty};
scope.Events.Dispatch(TreeChanged, this, new TreeChange<IContent>.EventArgs(new TreeChange<IContent>(root, TreeChangeTypes.RefreshAll)));
}
return report;
}
}
#endregion
#region Internal Methods
@@ -2812,7 +2839,7 @@ namespace Umbraco.Core.Services.Implement
// which we need for many things like keeping caches in sync, but we can surely do this MUCH better.
var changes = new List<TreeChange<IContent>>();
var moves = new List<Tuple<IContent, string>>();
var moves = new List<(IContent, string)>();
var contentTypeIdsA = contentTypeIds.ToArray();
// using an immediate uow here because we keep making changes with

66
src/Umbraco.Core/Services/Implement/MediaService.cs
View File

@@ -530,23 +530,27 @@ namespace Umbraco.Core.Services.Implement
totalChildren = 0;
return Enumerable.Empty<IMedia>();
}
return GetPagedDescendantsLocked(mediaPath[0].Path, pageIndex, pageSize, out totalChildren, filter, ordering);
return GetPagedLocked(GetPagedDescendantQuery(mediaPath[0].Path), pageIndex, pageSize, out totalChildren, filter, ordering);
}
return GetPagedDescendantsLocked(null, pageIndex, pageSize, out totalChildren, filter, ordering);
return GetPagedLocked(GetPagedDescendantQuery(null), pageIndex, pageSize, out totalChildren, filter, ordering);
}
}
private IEnumerable<IMedia> GetPagedDescendantsLocked(string mediaPath, long pageIndex, int pageSize, out long totalChildren,
private IQuery<IMedia> GetPagedDescendantQuery(string mediaPath)
{
var query = Query<IMedia>();
if (!mediaPath.IsNullOrWhiteSpace())
query.Where(x => x.Path.SqlStartsWith(mediaPath + ",", TextColumnType.NVarchar));
return query;
}
private IEnumerable<IMedia> GetPagedLocked(IQuery<IMedia> query, long pageIndex, int pageSize, out long totalChildren,
IQuery<IMedia> filter, Ordering ordering)
{
if (pageIndex < 0) throw new ArgumentOutOfRangeException(nameof(pageIndex));
if (pageSize <= 0) throw new ArgumentOutOfRangeException(nameof(pageSize));
if (ordering == null) throw new ArgumentNullException(nameof(ordering));
var query = Query<IMedia>();
if (!mediaPath.IsNullOrWhiteSpace())
query.Where(x => x.Path.SqlStartsWith(mediaPath + ",", TextColumnType.NVarchar));
return _mediaRepository.GetPage(query, pageIndex, pageSize, out totalChildren, filter, ordering);
}
@@ -888,7 +892,7 @@ namespace Umbraco.Core.Services.Implement
public Attempt<OperationResult> MoveToRecycleBin(IMedia media, int userId = Constants.Security.SuperUserId)
{
var evtMsgs = EventMessagesFactory.Get();
var moves = new List<Tuple<IMedia, string>>();
var moves = new List<(IMedia, string)>();
using (var scope = ScopeProvider.CreateScope())
{
@@ -940,7 +944,7 @@ namespace Umbraco.Core.Services.Implement
return OperationResult.Attempt.Succeed(evtMsgs);
}
var moves = new List<Tuple<IMedia, string>>();
var moves = new List<(IMedia, string)>();
using (var scope = ScopeProvider.CreateScope())
{
@@ -979,7 +983,7 @@ namespace Umbraco.Core.Services.Implement
// MUST be called from within WriteLock
// trash indicates whether we are trashing, un-trashing, or not changing anything
private void PerformMoveLocked(IMedia media, int parentId, IMedia parent, int userId, ICollection<Tuple<IMedia, string>> moves, bool? trash)
private void PerformMoveLocked(IMedia media, int parentId, IMedia parent, int userId, ICollection<(IMedia, string)> moves, bool? trash)
{
media.ParentId = parentId;
@@ -989,7 +993,7 @@ namespace Umbraco.Core.Services.Implement
var paths = new Dictionary<int, string>();
moves.Add(Tuple.Create(media, media.Path)); // capture original path
moves.Add((media, media.Path)); // capture original path
//need to store the original path to lookup descendants based on it below
var originalPath = media.Path;
@@ -1006,21 +1010,25 @@ namespace Umbraco.Core.Services.Implement
paths[media.Id] = (parent == null ? (parentId == Constants.System.RecycleBinMedia ? "-1,-21" : Constants.System.RootString) : parent.Path) + "," + media.Id;
const int pageSize = 500;
var page = 0;
var total = long.MaxValue;
while (page * pageSize < total)
var query = GetPagedDescendantQuery(originalPath);
long total;
do
{
var descendants = GetPagedDescendantsLocked(originalPath, page++, pageSize, out total, null, Ordering.By("Path", Direction.Ascending));
// We always page a page 0 because for each page, we are moving the result so the resulting total will be reduced
var descendants = GetPagedLocked(query, 0, pageSize, out total, null, Ordering.By("Path", Direction.Ascending));
foreach (var descendant in descendants)
{
moves.Add(Tuple.Create(descendant, descendant.Path)); // capture original path
moves.Add((descendant, descendant.Path)); // capture original path
// update path and level since we do not update parentId
descendant.Path = paths[descendant.Id] = paths[descendant.ParentId] + "," + descendant.Id;
descendant.Level += levelDelta;
PerformMoveMediaLocked(descendant, userId, trash);
}
}
} while (total > pageSize);
}
private void PerformMoveMediaLocked(IMedia media, int userId, bool? trash)
@@ -1139,6 +1147,26 @@ namespace Umbraco.Core.Services.Implement
}
return true;
}
public ContentDataIntegrityReport CheckDataIntegrity(ContentDataIntegrityReportOptions options)
{
using (var scope = ScopeProvider.CreateScope(autoComplete: true))
{
scope.WriteLock(Constants.Locks.MediaTree);
var report = _mediaRepository.CheckDataIntegrity(options);
if (report.FixedIssues.Count > 0)
{
//The event args needs a content item so we'll make a fake one with enough properties to not cause a null ref
var root = new Models.Media("root", -1, new MediaType(-1)) { Id = -1, Key = Guid.Empty };
scope.Events.Dispatch(TreeChanged, this, new TreeChange<IMedia>.EventArgs(new TreeChange<IMedia>(root, TreeChangeTypes.RefreshAll)));
}
return report;
}
}
#endregion
@@ -1277,7 +1305,7 @@ namespace Umbraco.Core.Services.Implement
// which we need for many things like keeping caches in sync, but we can surely do this MUCH better.
var changes = new List<TreeChange<IMedia>>();
var moves = new List<Tuple<IMedia, string>>();
var moves = new List<(IMedia, string)>();
var mediaTypeIdsA = mediaTypeIds.ToArray();
using (var scope = ScopeProvider.CreateScope())
@@ -1358,5 +1386,7 @@ namespace Umbraco.Core.Services.Implement
}
#endregion
}
}

4
src/Umbraco.Core/Umbraco.Core.csproj
View File

@@ -128,9 +128,13 @@
</Compile>
-->
<Compile Include="AssemblyExtensions.cs" />
<Compile Include="Constants-SqlTemplates.cs" />
<Compile Include="Migrations\Upgrade\V_8_0_0\Models\ContentTypeDto80.cs" />
<Compile Include="Migrations\Upgrade\V_8_0_0\Models\PropertyDataDto80.cs" />
<Compile Include="Migrations\Upgrade\V_8_0_0\Models\PropertyTypeDto80.cs" />
<Compile Include="Models\ContentDataIntegrityReport.cs" />
<Compile Include="Models\ContentDataIntegrityReportEntry.cs" />
<Compile Include="Models\ContentDataIntegrityReportOptions.cs" />
<Compile Include="Models\InstallLog.cs" />
<Compile Include="Persistence\Repositories\IInstallationRepository.cs" />
<Compile Include="Persistence\Repositories\Implement\InstallationRepository.cs" />

124
src/Umbraco.Web/HealthCheck/Checks/Data/DatabaseIntegrityCheck.cs Normal file
View File

@@ -0,0 +1,124 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Serilog.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Services;
namespace Umbraco.Web.HealthCheck.Checks.Data
{
[HealthCheck(
"73DD0C1C-E0CA-4C31-9564-1DCA509788AF",
"Database data integrity check",
Description = "Checks for various data integrity issues in the Umbraco database.",
Group = "Data Integrity")]
public class DatabaseIntegrityCheck : HealthCheck
{
private readonly IContentService _contentService;
private readonly IMediaService _mediaService;
private const string _fixMediaPaths = "fixMediaPaths";
private const string _fixContentPaths = "fixContentPaths";
private const string _fixMediaPathsTitle = "Fix media paths";
private const string _fixContentPathsTitle = "Fix content paths";
public DatabaseIntegrityCheck(IContentService contentService, IMediaService mediaService)
{
_contentService = contentService;
_mediaService = mediaService;
}
/// <summary>
/// Get the status for this health check
/// </summary>
/// <returns></returns>
public override IEnumerable<HealthCheckStatus> GetStatus()
{
//return the statuses
return new[]
{
CheckDocuments(false),
CheckMedia(false)
};
}
private HealthCheckStatus CheckMedia(bool fix)
{
return CheckPaths(_fixMediaPaths, _fixMediaPathsTitle, Core.Constants.UdiEntityType.Media, fix,
() => _mediaService.CheckDataIntegrity(new ContentDataIntegrityReportOptions {FixIssues = fix}));
}
private HealthCheckStatus CheckDocuments(bool fix)
{
return CheckPaths(_fixContentPaths, _fixContentPathsTitle, Core.Constants.UdiEntityType.Document, fix,
() => _contentService.CheckDataIntegrity(new ContentDataIntegrityReportOptions {FixIssues = fix}));
}
private HealthCheckStatus CheckPaths(string actionAlias, string actionName, string entityType, bool detailedReport, Func<ContentDataIntegrityReport> doCheck)
{
var report = doCheck();
var actions = new List<HealthCheckAction>();
if (!report.Ok)
{
actions.Add(new HealthCheckAction(actionAlias, Id)
{
Name = actionName
});
}
return new HealthCheckStatus(GetReport(report, entityType, detailedReport))
{
ResultType = report.Ok ? StatusResultType.Success : StatusResultType.Error,
Actions = actions
};
}
private static string GetReport(ContentDataIntegrityReport report, string entityType, bool detailed)
{
var sb = new StringBuilder();
if (report.Ok)
{
sb.AppendLine($"<p>All {entityType} paths are valid</p>");
if (!detailed)
return sb.ToString();
}
else
{
sb.AppendLine($"<p>{report.DetectedIssues.Count} invalid {entityType} paths detected.</p>");
}
if (detailed && report.DetectedIssues.Count > 0)
{
sb.AppendLine("<ul>");
foreach (var issueGroup in report.DetectedIssues.GroupBy(x => x.Value.IssueType))
{
var countByGroup = issueGroup.Count();
var fixedByGroup = issueGroup.Count(x => x.Value.Fixed);
sb.AppendLine("<li>");
sb.AppendLine($"{countByGroup} issues of type <code>{issueGroup.Key}</code> ... {fixedByGroup} fixed");
sb.AppendLine("</li>");
}
sb.AppendLine("</ul>");
}
return sb.ToString();
}
public override HealthCheckStatus ExecuteAction(HealthCheckAction action)
{
switch (action.Alias)
{
case _fixContentPaths:
return CheckDocuments(true);
case _fixMediaPaths:
return CheckMedia(true);
default:
throw new InvalidOperationException("Action not supported");
}
}
}
}

29
src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs
View File

@@ -502,6 +502,14 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
/// <summary>
/// Validate the <see cref="ContentNodeKit"/> and try to create a parent <see cref="LinkedNode{ContentNode}"/>
/// </summary>
/// <param name="kit"></param>
/// <param name="parent"></param>
/// <returns>
/// Returns false if the parent was not found or if the kit validation failed
/// </returns>
private bool BuildKit(ContentNodeKit kit, out LinkedNode<ContentNode> parent)
{
// make sure parent exists
@@ -512,6 +520,15 @@ namespace Umbraco.Web.PublishedCache.NuCache
return false;
}
// We cannot continue if there's no value. This shouldn't happen but it can happen if the database umbracoNode.path
// data is invalid/corrupt. If that is the case, the parentId might be ok but not the Path which can result in null
// because the data sort operation is by path.
if (parent.Value == null)
{
_logger.Warn<ContentStore>($"Skip item id={kit.Node.Id}, no Data assigned for linked node with path {kit.Node.Path} and parent id {kit.Node.ParentContentId}. This can indicate data corruption for the Path value for node {kit.Node.Id}. See the Health Check dashboard in Settings to resolve data integrity issues.");
return false;
}
// make sure the kit is valid
if (kit.DraftData == null && kit.PublishedData == null)
{
@@ -800,7 +817,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
//this zero's out the branch (recursively), if we're in a new gen this will add a NULL placeholder for the gen
ClearBranchLocked(existing);
//TODO: This removes the current GEN from the tree - do we really want to do that?
//TODO: This removes the current GEN from the tree - do we really want to do that? (not sure if this is still an issue....)
RemoveTreeNodeLocked(existing);
}
@@ -865,6 +882,10 @@ namespace Umbraco.Web.PublishedCache.NuCache
private void ClearBranchLocked(ContentNode content)
{
// This should never be null, all code that calls this method is null checking but we've seen
// issues of null ref exceptions in issue reports so we'll double check here
if (content == null) throw new ArgumentNullException(nameof(content));
SetValueLocked(_contentNodes, content.Id, null);
if (_localDb != null) RegisterChange(content.Id, ContentNodeKit.Null);
@@ -1032,6 +1053,12 @@ namespace Umbraco.Web.PublishedCache.NuCache
var parent = parentLink.Value;
// We are doing a null check here but this should no longer be possible because we have a null check in BuildKit
// for the parent.Value property and we'll output a warning. However I'll leave this additional null check in place.
// see https://github.com/umbraco/Umbraco-CMS/issues/7868
if (parent == null)
throw new PanicException($"A null Value was returned on the {nameof(parentLink)} LinkedNode with id={content.ParentContentId}, potentially your database paths are corrupted.");
// if parent has no children, clone parent + add as first child
if (parent.FirstChildContentId < 0)
{

2
src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs
View File

@@ -11,7 +11,7 @@
{
public LinkedNode(TValue value, long gen, LinkedNode<TValue> next = null)
{
Value = value;
Value = value; // This is allowed to be null, we actually explicitly set this to null in ClearLocked
Gen = gen;
Next = next;
}

1
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -156,6 +156,7 @@
<Compile Include="Editors\MacrosController.cs" />
<Compile Include="Editors\RelationTypeController.cs" />
<Compile Include="Editors\TinyMceController.cs" />
<Compile Include="HealthCheck\Checks\Data\DatabaseIntegrityCheck.cs" />
<Compile Include="ImageCropperTemplateCoreExtensions.cs" />
<Compile Include="IUmbracoContextFactory.cs" />
<Compile Include="Install\ChangesMonitor.cs" />