();
+ if (!report.Ok)
+ {
+ actions.Add(new HealthCheckAction(actionAlias, Id)
+ {
+ Name = actionName
+ });
+ }
+
+ return new HealthCheckStatus(GetReport(report, entityType, detailedReport))
+ {
+ ResultType = report.Ok ? StatusResultType.Success : StatusResultType.Error,
+ Actions = actions
+ };
+ }
+
+ private static string GetReport(ContentDataIntegrityReport report, string entityType, bool detailed)
+ {
+ var sb = new StringBuilder();
+
+ if (report.Ok)
+ {
+ sb.AppendLine($"All {entityType} paths are valid
");
+
+ if (!detailed)
+ return sb.ToString();
+ }
+ else
+ {
+ sb.AppendLine($"{report.DetectedIssues.Count} invalid {entityType} paths detected.
");
+ }
+
+ if (detailed && report.DetectedIssues.Count > 0)
+ {
+ sb.AppendLine("");
+ foreach (var issueGroup in report.DetectedIssues.GroupBy(x => x.Value.IssueType))
+ {
+ var countByGroup = issueGroup.Count();
+ var fixedByGroup = issueGroup.Count(x => x.Value.Fixed);
+ sb.AppendLine("- ");
+ sb.AppendLine($"{countByGroup} issues of type
{issueGroup.Key} ... {fixedByGroup} fixed");
+ sb.AppendLine(" ");
+ }
+ sb.AppendLine("
");
+ }
+
+ return sb.ToString();
+ }
+
+ public override HealthCheckStatus ExecuteAction(HealthCheckAction action)
+ {
+ switch (action.Alias)
+ {
+ case _fixContentPaths:
+ return CheckDocuments(true);
+ case _fixMediaPaths:
+ return CheckMedia(true);
+ default:
+ throw new InvalidOperationException("Action not supported");
+ }
+ }
+ }
+}
diff --git a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs
index 34d21497a2..a3f918c92c 100644
--- a/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs
+++ b/src/Umbraco.Web/PublishedCache/NuCache/ContentStore.cs
@@ -502,6 +502,14 @@ namespace Umbraco.Web.PublishedCache.NuCache
}
}
+ ///
+ /// Validate the and try to create a parent
+ ///
+ ///
+ ///
+ ///
+ /// Returns false if the parent was not found or if the kit validation failed
+ ///
private bool BuildKit(ContentNodeKit kit, out LinkedNode parent)
{
// make sure parent exists
@@ -512,6 +520,15 @@ namespace Umbraco.Web.PublishedCache.NuCache
return false;
}
+ // We cannot continue if there's no value. This shouldn't happen but it can happen if the database umbracoNode.path
+ // data is invalid/corrupt. If that is the case, the parentId might be ok but not the Path which can result in null
+ // because the data sort operation is by path.
+ if (parent.Value == null)
+ {
+ _logger.Warn($"Skip item id={kit.Node.Id}, no Data assigned for linked node with path {kit.Node.Path} and parent id {kit.Node.ParentContentId}. This can indicate data corruption for the Path value for node {kit.Node.Id}. See the Health Check dashboard in Settings to resolve data integrity issues.");
+ return false;
+ }
+
// make sure the kit is valid
if (kit.DraftData == null && kit.PublishedData == null)
{
@@ -800,7 +817,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
{
//this zero's out the branch (recursively), if we're in a new gen this will add a NULL placeholder for the gen
ClearBranchLocked(existing);
- //TODO: This removes the current GEN from the tree - do we really want to do that?
+ //TODO: This removes the current GEN from the tree - do we really want to do that? (not sure if this is still an issue....)
RemoveTreeNodeLocked(existing);
}
@@ -865,6 +882,10 @@ namespace Umbraco.Web.PublishedCache.NuCache
private void ClearBranchLocked(ContentNode content)
{
+ // This should never be null, all code that calls this method is null checking but we've seen
+ // issues of null ref exceptions in issue reports so we'll double check here
+ if (content == null) throw new ArgumentNullException(nameof(content));
+
SetValueLocked(_contentNodes, content.Id, null);
if (_localDb != null) RegisterChange(content.Id, ContentNodeKit.Null);
@@ -1032,6 +1053,12 @@ namespace Umbraco.Web.PublishedCache.NuCache
var parent = parentLink.Value;
+ // We are doing a null check here but this should no longer be possible because we have a null check in BuildKit
+ // for the parent.Value property and we'll output a warning. However I'll leave this additional null check in place.
+ // see https://github.com/umbraco/Umbraco-CMS/issues/7868
+ if (parent == null)
+ throw new PanicException($"A null Value was returned on the {nameof(parentLink)} LinkedNode with id={content.ParentContentId}, potentially your database paths are corrupted.");
+
// if parent has no children, clone parent + add as first child
if (parent.FirstChildContentId < 0)
{
diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
index 19aab7ea65..694dac04df 100644
--- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
+++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/DatabaseDataSource.cs
@@ -20,6 +20,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
// provides efficient database access for NuCache
internal class DatabaseDataSource : IDataSource
{
+ private const int PageSize = 500;
+
// we want arrays, we want them all loaded, not an enumerable
private Sql ContentSourcesSelect(IScope scope, Func, Sql> joins = null)
@@ -79,33 +81,43 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder);
- return scope.Database.Query(sql).Select(CreateContentNodeKit);
+ // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
+ // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
+
+ foreach (var row in scope.Database.QueryPaged(PageSize, sql))
+ yield return CreateContentNodeKit(row);
}
public IEnumerable GetBranchContentSources(IScope scope, int id)
{
var syntax = scope.SqlContext.SqlSyntax;
- var sql = ContentSourcesSelect(scope, s => s
+ var sql = ContentSourcesSelect(scope,
+ s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
+ .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
+ .Where(x => x.NodeId == id, "x")
+ .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder);
- .InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
+ // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
+ // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
- .Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
- .Where(x => x.NodeId == id, "x")
- .OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder);
-
- return scope.Database.Query(sql).Select(CreateContentNodeKit);
+ foreach (var row in scope.Database.QueryPaged(PageSize, sql))
+ yield return CreateContentNodeKit(row);
}
public IEnumerable GetTypeContentSources(IScope scope, IEnumerable ids)
{
- if (!ids.Any()) return Enumerable.Empty();
+ if (!ids.Any()) yield break;
var sql = ContentSourcesSelect(scope)
.Where(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.WhereIn(x => x.ContentTypeId, ids)
.OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder);
- return scope.Database.Query(sql).Select(CreateContentNodeKit);
+ // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
+ // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
+
+ foreach (var row in scope.Database.QueryPaged(PageSize, sql))
+ yield return CreateContentNodeKit(row);
}
private Sql MediaSourcesSelect(IScope scope, Func, Sql> joins = null)
@@ -116,11 +128,8 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
.AndSelect(x => Alias(x.ContentTypeId, "ContentTypeId"))
-
.AndSelect(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
-
.AndSelect("nuEdit", x => Alias(x.Data, "EditData"))
-
.From();
if (joins != null)
@@ -128,9 +137,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
sql = sql
.InnerJoin().On((left, right) => left.NodeId == right.NodeId)
-
.InnerJoin().On((left, right) => left.NodeId == right.NodeId && right.Current)
-
.LeftJoin("nuEdit").On((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit");
return sql;
@@ -152,33 +159,43 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource
.Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder);
- return scope.Database.Query(sql).Select(CreateMediaNodeKit);
+ // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
+ // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
+
+ foreach (var row in scope.Database.QueryPaged(PageSize, sql))
+ yield return CreateMediaNodeKit(row);
}
public IEnumerable GetBranchMediaSources(IScope scope, int id)
{
var syntax = scope.SqlContext.SqlSyntax;
- var sql = MediaSourcesSelect(scope, s => s
-
- .InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
-
+ var sql = MediaSourcesSelect(scope,
+ s => s.InnerJoin("x").On((left, right) => left.NodeId == right.NodeId || SqlText(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
.Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.Where(x => x.NodeId == id, "x")
.OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder);
- return scope.Database.Query(sql).Select(CreateMediaNodeKit);
+ // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
+ // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
+
+ foreach (var row in scope.Database.QueryPaged(PageSize, sql))
+ yield return CreateMediaNodeKit(row);
}
public IEnumerable GetTypeMediaSources(IScope scope, IEnumerable ids)
{
- if (!ids.Any()) return Enumerable.Empty();
+ if (!ids.Any()) yield break;
var sql = MediaSourcesSelect(scope)
.Where(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.WhereIn(x => x.ContentTypeId, ids)
.OrderBy(x => x.Level, x => x.ParentId, x => x.SortOrder);
- return scope.Database.Query(sql).Select(CreateMediaNodeKit);
+ // We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
+ // We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
+
+ foreach (var row in scope.Database.QueryPaged(PageSize, sql))
+ yield return CreateMediaNodeKit(row);
}
private static ContentNodeKit CreateContentNodeKit(ContentSourceDto dto)
diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
index a33d9ee427..6866878484 100755
--- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
+++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
@@ -5,6 +5,7 @@ using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
+using System.Threading.Tasks;
using CSharpTest.Net.Collections;
using Newtonsoft.Json;
using Umbraco.Core;
@@ -866,12 +867,37 @@ namespace Umbraco.Web.PublishedCache.NuCache
//into a new DLL for the application which includes both content types and media types.
//Since the models in the cache are based on these actual classes, all of the objects in the cache need to be updated
//to use the newest version of the class.
- using (_contentStore.GetScopedWriteLock(_scopeProvider))
- using (_mediaStore.GetScopedWriteLock(_scopeProvider))
- {
- NotifyLocked(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out var draftChanged, out var publishedChanged);
- NotifyLocked(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out var anythingChanged);
- }
+
+ // NOTE: Ideally this can be run on background threads here which would prevent blocking the UI
+ // as is the case when saving a content type. Intially one would think that it won't be any different
+ // between running this here or in another background thread immediately after with regards to how the
+ // UI will respond because we already know between calling `WithSafeLiveFactoryReset` to reset the PureLive models
+ // and this code here, that many front-end requests could be attempted to be processed. If that is the case, those pages are going to get a
+ // model binding error and our ModelBindingExceptionFilter is going to to its magic to reload those pages so the end user is none the wiser.
+ // So whether or not this executes 'here' or on a background thread immediately wouldn't seem to make any difference except that we can return
+ // execution to the UI sooner.
+ // BUT!... there is a difference IIRC. There is still execution logic that continues after this call on this thread with the cache refreshers
+ // and those cache refreshers need to have the up-to-date data since other user cache refreshers will be expecting the data to be 'live'. If
+ // we ran this on a background thread then those cache refreshers are going to not get 'live' data when they query the content cache which
+ // they require.
+
+ // These can be run side by side in parallel.
+
+ Parallel.Invoke(
+ () =>
+ {
+ using (_contentStore.GetScopedWriteLock(_scopeProvider))
+ {
+ NotifyLocked(new[] { new ContentCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _, out _);
+ }
+ },
+ () =>
+ {
+ using (_mediaStore.GetScopedWriteLock(_scopeProvider))
+ {
+ NotifyLocked(new[] { new MediaCacheRefresher.JsonPayload(0, null, TreeChangeTypes.RefreshAll) }, out _);
+ }
+ });
}
((PublishedSnapshot)CurrentPublishedSnapshot)?.Resync();
diff --git a/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs b/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs
index d187996df8..94f83ac4e5 100644
--- a/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs
+++ b/src/Umbraco.Web/PublishedCache/NuCache/Snap/LinkedNode.cs
@@ -11,7 +11,7 @@
{
public LinkedNode(TValue value, long gen, LinkedNode next = null)
{
- Value = value;
+ Value = value; // This is allowed to be null, we actually explicitly set this to null in ClearLocked
Gen = gen;
Next = next;
}
diff --git a/src/Umbraco.Web/Search/ExamineComposer.cs b/src/Umbraco.Web/Search/ExamineComposer.cs
index b30f0cbe03..64eeb6978a 100644
--- a/src/Umbraco.Web/Search/ExamineComposer.cs
+++ b/src/Umbraco.Web/Search/ExamineComposer.cs
@@ -4,6 +4,7 @@ using Umbraco.Core;
using Umbraco.Core.Composing;
using Umbraco.Core.Models;
using Umbraco.Core.PropertyEditors;
+using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using Umbraco.Examine;
@@ -36,12 +37,14 @@ namespace Umbraco.Web.Search
factory.GetInstance(),
factory.GetInstance(),
factory.GetInstance(),
+ factory.GetInstance(),
true));
composition.RegisterUnique(factory =>
new ContentValueSetBuilder(
factory.GetInstance(),
factory.GetInstance(),
factory.GetInstance(),
+ factory.GetInstance(),
false));
composition.RegisterUnique, MediaValueSetBuilder>();
composition.RegisterUnique, MemberValueSetBuilder>();
diff --git a/src/Umbraco.Web/Security/BackOfficeSignInManager.cs b/src/Umbraco.Web/Security/BackOfficeSignInManager.cs
index 8e5e532731..5f1c1012f3 100644
--- a/src/Umbraco.Web/Security/BackOfficeSignInManager.cs
+++ b/src/Umbraco.Web/Security/BackOfficeSignInManager.cs
@@ -121,7 +121,9 @@ namespace Umbraco.Web.Security
{
_logger.WriteCore(TraceEventType.Information, 0,
$"Login attempt failed for username {userName} from IP address {_request.RemoteIpAddress}, no content and/or media start nodes could be found for any of the user's groups", null, null);
- return SignInStatus.Failure;
+
+ // We will say its a sucessful login which it is, but they have no node access
+ return SignInStatus.Success;
}
}
diff --git a/src/Umbraco.Web/Umbraco.Web.csproj b/src/Umbraco.Web/Umbraco.Web.csproj
index c3024f63ae..e39687bed8 100755
--- a/src/Umbraco.Web/Umbraco.Web.csproj
+++ b/src/Umbraco.Web/Umbraco.Web.csproj
@@ -156,6 +156,7 @@
+