yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Big refactor or PublishedSnapshotService to split up so that there's a service and repository responsible for the data querying and persistence

Browse Source
This commit is contained in:
Shannon
2020-12-09 22:43:49 +11:00
parent e3be4009c0
commit 4b85f8eb20
37 changed files with 1766 additions and 1491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/Umbraco.Core/PublishedCache/IPublishedSnapshotService.cs
View File

@@ -11,8 +11,6 @@ namespace Umbraco.Web.PublishedCache
/// </summary>
public interface IPublishedSnapshotService : IDisposable
{
#region PublishedSnapshot
/* Various places (such as Node) want to access the XML content, today as an XmlDocument
* but to migrate to a new cache, they're migrating to an XPathNavigator. Still, they need
* to find out how to get that navigator.
@@ -25,6 +23,8 @@ namespace Umbraco.Web.PublishedCache
*
*/
void LoadCachesOnStartup();
/// <summary>
/// Creates a published snapshot.
/// </summary>
@@ -47,20 +47,26 @@ namespace Umbraco.Web.PublishedCache
/// <returns>A value indicating whether the published snapshot has the proper environment to run.</returns>
bool EnsureEnvironment(out IEnumerable<string> errors);
#endregion
#region Rebuild
/// <summary>
/// Rebuilds internal caches (but does not reload).
/// </summary>
/// <param name="groupSize">The operation batch size to process the items</param>
/// <param name="contentTypeIds">If not null will process content for the matching content types, if empty will process all content</param>
/// <param name="mediaTypeIds">If not null will process content for the matching media types, if empty will process all media</param>
/// <param name="memberTypeIds">If not null will process content for the matching members types, if empty will process all members</param>
/// <remarks>
/// <para>Forces the snapshot service to rebuild its internal caches. For instance, some caches
/// may rely on a database table to store pre-serialized version of documents.</para>
/// <para>This does *not* reload the caches. Caches need to be reloaded, for instance via
/// <see cref="DistributedCache" /> RefreshAllPublishedSnapshot method.</para>
/// </remarks>
void Rebuild();
void Rebuild(
int groupSize = 5000,
IReadOnlyCollection<int> contentTypeIds = null,
IReadOnlyCollection<int> mediaTypeIds = null,
IReadOnlyCollection<int> memberTypeIds = null);
#endregion
@@ -84,11 +90,11 @@ namespace Umbraco.Web.PublishedCache
/// <returns>A preview token.</returns>
/// <remarks>
/// <para>Tells the caches that they should prepare any data that they would be keeping
/// in order to provide preview to a give user. In the Xml cache this means creating the Xml
/// in order to provide preview to a given user. In the Xml cache this means creating the Xml
/// file, though other caches may do things differently.</para>
/// <para>Does not handle the preview token storage (cookie, etc) that must be handled separately.</para>
/// </remarks>
string EnterPreview(IUser user, int contentId);
string EnterPreview(IUser user, int contentId); // TODO: Remove this, it is not needed and is legacy from the XML cache
/// <summary>
/// Refreshes preview for a specified content.
@@ -98,7 +104,7 @@ namespace Umbraco.Web.PublishedCache
/// <remarks>Tells the caches that they should update any data that they would be keeping
/// in order to provide preview to a given user. In the Xml cache this means updating the Xml
/// file, though other caches may do things differently.</remarks>
void RefreshPreview(string previewToken, int contentId);
void RefreshPreview(string previewToken, int contentId); // TODO: Remove this, it is not needed and is legacy from the XML cache
/// <summary>
/// Exits preview for a specified preview token.
@@ -110,7 +116,7 @@ namespace Umbraco.Web.PublishedCache
/// though other caches may do things differently.</para>
/// <para>Does not handle the preview token storage (cookie, etc) that must be handled separately.</para>
/// </remarks>
void ExitPreview(string previewToken);
void ExitPreview(string previewToken); // TODO: Remove this, it is not needed and is legacy from the XML cache
#endregion
@@ -162,14 +168,12 @@ namespace Umbraco.Web.PublishedCache
#endregion
#region Status
// TODO: This is weird, why is this is this a thing? Maybe IPublishedSnapshotStatus?
string GetStatus();
// TODO: This is weird, why is this is this a thing? Maybe IPublishedSnapshotStatus?
string StatusUrl { get; }
#endregion
void Collect();
}
}

43
src/Umbraco.Core/PublishedCache/PublishedSnapshotServiceBase.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using Umbraco.Core.Models.Membership;
using Umbraco.Core.Models.PublishedContent;
@@ -8,44 +8,83 @@ namespace Umbraco.Web.PublishedCache
{
public abstract class PublishedSnapshotServiceBase : IPublishedSnapshotService
{
/// <summary>
/// Initializes a new instance of the <see cref="PublishedSnapshotServiceBase"/> class.
/// </summary>
protected PublishedSnapshotServiceBase(IPublishedSnapshotAccessor publishedSnapshotAccessor, IVariationContextAccessor variationContextAccessor)
{
PublishedSnapshotAccessor = publishedSnapshotAccessor;
VariationContextAccessor = variationContextAccessor;
}
/// <inheritdoc/>
public IPublishedSnapshotAccessor PublishedSnapshotAccessor { get; }
/// <summary>
/// Gets the <see cref="IVariationContextAccessor"/>
/// </summary>
public IVariationContextAccessor VariationContextAccessor { get; }
// note: NOT setting _publishedSnapshotAccessor.PublishedSnapshot here because it is the
// responsibility of the caller to manage what the 'current' facade is
/// <inheritdoc/>
public abstract IPublishedSnapshot CreatePublishedSnapshot(string previewToken);
protected IPublishedSnapshot CurrentPublishedSnapshot => PublishedSnapshotAccessor.PublishedSnapshot;
/// <inheritdoc/>
public abstract bool EnsureEnvironment(out IEnumerable<string> errors);
/// <inheritdoc/>
public abstract string EnterPreview(IUser user, int contentId);
/// <inheritdoc/>
public abstract void RefreshPreview(string previewToken, int contentId);
/// <inheritdoc/>
public abstract void ExitPreview(string previewToken);
/// <inheritdoc/>
public abstract void Notify(ContentCacheRefresher.JsonPayload[] payloads, out bool draftChanged, out bool publishedChanged);
/// <inheritdoc/>
public abstract void Notify(MediaCacheRefresher.JsonPayload[] payloads, out bool anythingChanged);
/// <inheritdoc/>
public abstract void Notify(ContentTypeCacheRefresher.JsonPayload[] payloads);
/// <inheritdoc/>
public abstract void Notify(DataTypeCacheRefresher.JsonPayload[] payloads);
/// <inheritdoc/>
public abstract void Notify(DomainCacheRefresher.JsonPayload[] payloads);
public virtual void Rebuild()
// TODO: Why is this virtual?
/// <inheritdoc/>
public virtual void Rebuild(
int groupSize = 5000,
IReadOnlyCollection<int> contentTypeIds = null,
IReadOnlyCollection<int> mediaTypeIds = null,
IReadOnlyCollection<int> memberTypeIds = null)
{ }
/// <inheritdoc/>
public virtual void Dispose()
{ }
/// <inheritdoc/>
public abstract string GetStatus();
/// <inheritdoc/>
public virtual string StatusUrl => "views/dashboard/settings/publishedsnapshotcache.html";
/// <inheritdoc/>
public virtual void Collect()
{
}
public abstract void LoadCachesOnStartup();
}
}

2
src/Umbraco.Infrastructure/Persistence/Dtos/ContentNuDto.cs
View File

@@ -1,4 +1,4 @@
using System.Data;
using System.Data;
using NPoco;
using Umbraco.Core.Persistence.DatabaseAnnotations;

73
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/ContentRepositoryBase.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
@@ -767,8 +767,21 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
#region UnitOfWork Events
// TODO: The reason these events are in the repository is for legacy, the events should exist at the service
// level now since we can fire these events within the transaction... so move the events to service level
/*
* TODO: The reason these events are in the repository is for legacy, the events should exist at the service
* level now since we can fire these events within the transaction...
* The reason these events 'need' to fire in the transaction is to ensure data consistency with Nucache (currently
* the only thing that uses them). For example, if the transaction succeeds and NuCache listened to ContentService.Saved
* and then NuCache failed at persisting data after the trans completed, then NuCache would be out of sync. This way
* the entire trans is rolled back if NuCache files. That said, I'm unsure this is really required because there
* are other systems that rely on the "ed" (i.e. Saved) events like Examine which would be inconsistent if it failed
* too. I'm just not sure this is totally necessary especially.
* So these events can be moved to the service level. However, see the notes below, it seems the only event we
* really need is the ScopedEntityRefresh. The only tricky part with moving that to the service level is that the
* handlers of that event will need to deal with the data a little differently because it seems that the
* "Published" flag on the content item matters and this event is raised before that flag is switched. Weird.
* We have the ability with IContent to see if something "WasPublished", etc.. so i think we could still use that.
*/
public class ScopedEntityEventArgs : EventArgs
{
@@ -784,6 +797,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
public class ScopedVersionEventArgs : EventArgs
{
/// <summary>
/// Initializes a new instance of the <see cref="ScopedVersionEventArgs"/> class.
/// </summary>
public ScopedVersionEventArgs(IScope scope, int entityId, int versionId)
{
Scope = scope;
@@ -791,13 +807,43 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
VersionId = versionId;
}
/// <summary>
/// Gets the current <see cref="IScope"/>
/// </summary>
public IScope Scope { get; }
/// <summary>
/// Gets the entity id
/// </summary>
public int EntityId { get; }
/// <summary>
/// Gets the version id
/// </summary>
public int VersionId { get; }
}
/// <summary>
/// Occurs when an <see cref="TEntity"/> is created or updated from within the <see cref="IScope"/> (transaction)
/// </summary>
public static event TypedEventHandler<TRepository, ScopedEntityEventArgs> ScopedEntityRefresh;
/// <summary>
/// Occurs when an <see cref="TEntity"/> is being deleted from within the <see cref="IScope"/> (transaction)
/// </summary>
/// <remarks>
/// TODO: This doesn't seem to be necessary at all, the service "Deleting" events for this would work just fine
/// since they are raised before the item is actually deleted just like this event.
/// </remarks>
public static event TypedEventHandler<TRepository, ScopedEntityEventArgs> ScopeEntityRemove;
/// <summary>
/// Occurs when a version for an <see cref="TEntity"/> is being deleted from within the <see cref="IScope"/> (transaction)
/// </summary>
/// <remarks>
/// TODO: This doesn't seem to be necessary at all, the service "DeletingVersions" events for this would work just fine
/// since they are raised before the item is actually deleted just like this event.
/// </remarks>
public static event TypedEventHandler<TRepository, ScopedVersionEventArgs> ScopeVersionRemove;
// used by tests to clear events
@@ -808,20 +854,23 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
ScopeVersionRemove = null;
}
/// <summary>
/// Raises the <see cref="ScopedEntityRefresh"/> event
/// </summary>
protected void OnUowRefreshedEntity(ScopedEntityEventArgs args)
{
ScopedEntityRefresh.RaiseEvent(args, This);
}
=> ScopedEntityRefresh.RaiseEvent(args, This);
/// <summary>
/// Raises the <see cref="ScopeEntityRemove"/> event
/// </summary>
protected void OnUowRemovingEntity(ScopedEntityEventArgs args)
{
ScopeEntityRemove.RaiseEvent(args, This);
}
=> ScopeEntityRemove.RaiseEvent(args, This);
/// <summary>
/// Raises the <see cref="ScopeVersionRemove"/> event
/// </summary>
protected void OnUowRemovingVersion(ScopedVersionEventArgs args)
{
ScopeVersionRemove.RaiseEvent(args, This);
}
=> ScopeVersionRemove.RaiseEvent(args, This);
#endregion

27
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepository.cs
View File

@@ -1,15 +1,16 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using NPoco;
using Umbraco.Core.Cache;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Entities;
using Umbraco.Core.Persistence.Dtos;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Scoping;
using static Umbraco.Core.Persistence.SqlExtensionsStatics;
using Umbraco.Core.Persistence.SqlSyntax;
using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using static Umbraco.Core.Persistence.SqlExtensionsStatics;
namespace Umbraco.Core.Persistence.Repositories.Implement
{
@@ -20,19 +21,13 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
/// <para>Limited to objects that have a corresponding node (in umbracoNode table).</para>
/// <para>Returns <see cref="IEntitySlim"/> objects, i.e. lightweight representation of entities.</para>
/// </remarks>
internal class EntityRepository : IEntityRepository
internal class EntityRepository : RepositoryBase, IEntityRepository
{
private readonly IScopeAccessor _scopeAccessor;
public EntityRepository(IScopeAccessor scopeAccessor)
public EntityRepository(IScopeAccessor scopeAccessor, AppCaches appCaches)
: base(scopeAccessor, appCaches)
{
_scopeAccessor = scopeAccessor;
}
protected IUmbracoDatabase Database => _scopeAccessor.AmbientScope.Database;
protected Sql<ISqlContext> Sql() => _scopeAccessor.AmbientScope.SqlContext.Sql();
protected ISqlSyntaxProvider SqlSyntax => _scopeAccessor.AmbientScope.SqlContext.SqlSyntax;
#region Repository
public IEnumerable<IEntitySlim> GetPagedResultsByQuery(IQuery<IUmbracoEntity> query, Guid objectType, long pageIndex, int pageSize, out long totalRecords,
@@ -49,17 +44,17 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
var isMedia = objectTypes.Any(objectType => objectType == Constants.ObjectTypes.Media);
var isMember = objectTypes.Any(objectType => objectType == Constants.ObjectTypes.Member);
var sql = GetBaseWhere(isContent, isMedia, isMember, false, s =>
Sql<ISqlContext> sql = GetBaseWhere(isContent, isMedia, isMember, false, s =>
{
sqlCustomization?.Invoke(s);
if (filter != null)
{
foreach (var filterClause in filter.GetWhereClauses())
foreach (Tuple<string, object[]> filterClause in filter.GetWhereClauses())
{
s.Where(filterClause.Item1, filterClause.Item2);
}
}
}, objectTypes);
ordering = ordering ?? Ordering.ByDefault();

181
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs → src/Umbraco.Infrastructure/Persistence/Repositories/Implement/EntityRepositoryBase.cs
View File

@@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
using NPoco;
using Umbraco.Core.Cache;
using Umbraco.Core.Models.Entities;
using Umbraco.Core.Persistence.Querying;
@@ -9,53 +10,38 @@ using Umbraco.Core.Scoping;
namespace Umbraco.Core.Persistence.Repositories.Implement
{
/// <summary>
/// Provides a base class to all repositories.
/// Provides a base class to all <see cref="IEntity"/> based repositories.
/// </summary>
/// <typeparam name="TEntity">The type of the entity managed by this repository.</typeparam>
/// <typeparam name="TId">The type of the entity's unique identifier.</typeparam>
public abstract class RepositoryBase<TId, TEntity> : IReadWriteQueryRepository<TId, TEntity>
/// <typeparam name="TEntity">The type of the entity managed by this repository.</typeparam>
public abstract class EntityRepositoryBase<TId, TEntity> : RepositoryBase, IReadWriteQueryRepository<TId, TEntity>
where TEntity : class, IEntity
{
private IRepositoryCachePolicy<TEntity, TId> _cachePolicy;
protected RepositoryBase(IScopeAccessor scopeAccessor, AppCaches appCaches, ILogger<RepositoryBase<TId, TEntity>> logger)
{
ScopeAccessor = scopeAccessor ?? throw new ArgumentNullException(nameof(scopeAccessor));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
AppCaches = appCaches ?? throw new ArgumentNullException(nameof(appCaches));
}
protected ILogger<RepositoryBase<TId, TEntity>> Logger { get; }
protected AppCaches AppCaches { get; }
protected IAppPolicyCache GlobalIsolatedCache => AppCaches.IsolatedCaches.GetOrCreate<TEntity>();
protected IScopeAccessor ScopeAccessor { get; }
protected IScope AmbientScope
{
get
{
var scope = ScopeAccessor.AmbientScope;
if (scope == null)
throw new InvalidOperationException("Cannot run a repository without an ambient scope.");
return scope;
}
}
#region Static Queries
private IQuery<TEntity> _hasIdQuery;
private static RepositoryCachePolicyOptions s_defaultOptions;
#endregion
protected virtual TId GetEntityId(TEntity entity)
/// <summary>
/// Initializes a new instance of the <see cref="EntityRepositoryBase{TId, TEntity}"/> class.
/// </summary>
protected EntityRepositoryBase(IScopeAccessor scopeAccessor, AppCaches appCaches, ILogger<EntityRepositoryBase<TId, TEntity>> logger)
: base(scopeAccessor, appCaches)
{
return (TId) (object) entity.Id;
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <summary>
/// Gets the logger
/// </summary>
protected ILogger<EntityRepositoryBase<TId, TEntity>> Logger { get; }
/// <summary>
/// Gets the isolated cache for the <see cref="TEntity"/>
/// </summary>
protected IAppPolicyCache GlobalIsolatedCache => AppCaches.IsolatedCaches.GetOrCreate<TEntity>();
/// <summary>
/// Gets the isolated cache.
/// </summary>
@@ -78,30 +64,34 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
}
}
// ReSharper disable once StaticMemberInGenericType
private static RepositoryCachePolicyOptions _defaultOptions;
// ReSharper disable once InconsistentNaming
protected virtual RepositoryCachePolicyOptions DefaultOptions
{
get
{
return _defaultOptions ?? (_defaultOptions
/// <summary>
/// Gets the default <see cref="RepositoryCachePolicyOptions"/>
/// </summary>
protected virtual RepositoryCachePolicyOptions DefaultOptions => s_defaultOptions ?? (s_defaultOptions
= new RepositoryCachePolicyOptions(() =>
{
// get count of all entities of current type (TEntity) to ensure cached result is correct
// create query once if it is needed (no need for locking here) - query is static!
var query = _hasIdQuery ?? (_hasIdQuery = AmbientScope.SqlContext.Query<TEntity>().Where(x => x.Id != 0));
IQuery<TEntity> query = _hasIdQuery ?? (_hasIdQuery = AmbientScope.SqlContext.Query<TEntity>().Where(x => x.Id != 0));
return PerformCount(query);
}));
}
}
/// <summary>
/// Gets the node object type for the repository's entity
/// </summary>
protected abstract Guid NodeObjectTypeId { get; }
/// <summary>
/// Gets the repository cache policy
/// </summary>
protected IRepositoryCachePolicy<TEntity, TId> CachePolicy
{
get
{
if (AppCaches == AppCaches.NoCache)
{
return NoCacheRepositoryCachePolicy<TEntity, TId>.Instance;
}
// create the cache policy using IsolatedCache which is either global
// or scoped depending on the repository cache mode for the current scope
@@ -122,63 +112,98 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
}
}
/// <summary>
/// Get the entity id for the <see cref="TEntity"/>
/// </summary>
protected virtual TId GetEntityId(TEntity entity)
=> (TId)(object)entity.Id;
/// <summary>
/// Create the repository cache policy
/// </summary>
protected virtual IRepositoryCachePolicy<TEntity, TId> CreateCachePolicy()
{
return new DefaultRepositoryCachePolicy<TEntity, TId>(GlobalIsolatedCache, ScopeAccessor, DefaultOptions);
}
=> new DefaultRepositoryCachePolicy<TEntity, TId>(GlobalIsolatedCache, ScopeAccessor, DefaultOptions);
/// <summary>
/// Adds or Updates an entity of type TEntity
/// </summary>
/// <remarks>This method is backed by an <see cref="IAppPolicyCache"/> cache</remarks>
/// <param name="entity"></param>
public virtual void Save(TEntity entity)
{
if (entity.HasIdentity == false)
{
CachePolicy.Create(entity, PersistNewItem);
}
else
{
CachePolicy.Update(entity, PersistUpdatedItem);
}
}
/// <summary>
/// Deletes the passed in entity
/// </summary>
/// <param name="entity"></param>
public virtual void Delete(TEntity entity)
{
CachePolicy.Delete(entity, PersistDeletedItem);
}
=> CachePolicy.Delete(entity, PersistDeletedItem);
protected abstract TEntity PerformGet(TId id);
protected abstract IEnumerable<TEntity> PerformGetAll(params TId[] ids);
protected abstract IEnumerable<TEntity> PerformGetByQuery(IQuery<TEntity> query);
protected abstract bool PerformExists(TId id);
protected abstract int PerformCount(IQuery<TEntity> query);
protected abstract void PersistNewItem(TEntity item);
protected abstract void PersistUpdatedItem(TEntity item);
protected abstract void PersistDeletedItem(TEntity item);
protected abstract void PersistUpdatedItem(TEntity item);
protected abstract Sql<ISqlContext> GetBaseQuery(bool isCount); // TODO: obsolete, use QueryType instead everywhere
protected abstract string GetBaseWhereClause();
protected abstract IEnumerable<string> GetDeleteClauses();
protected virtual bool PerformExists(TId id)
{
var sql = GetBaseQuery(true);
sql.Where(GetBaseWhereClause(), new { id = id });
var count = Database.ExecuteScalar<int>(sql);
return count == 1;
}
protected virtual int PerformCount(IQuery<TEntity> query)
{
var sqlClause = GetBaseQuery(true);
var translator = new SqlTranslator<TEntity>(sqlClause, query);
var sql = translator.Translate();
return Database.ExecuteScalar<int>(sql);
}
protected virtual void PersistDeletedItem(TEntity entity)
{
var deletes = GetDeleteClauses();
foreach (var delete in deletes)
{
Database.Execute(delete, new { id = GetEntityId(entity) });
}
entity.DeleteDate = DateTime.Now;
}
/// <summary>
/// Gets an entity by the passed in Id utilizing the repository's cache policy
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public TEntity Get(TId id)
{
return CachePolicy.Get(id, PerformGet, PerformGetAll);
}
=> CachePolicy.Get(id, PerformGet, PerformGetAll);
/// <summary>
/// Gets all entities of type TEntity or a list according to the passed in Ids
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
public IEnumerable<TEntity> GetMany(params TId[] ids)
{
// ensure they are de-duplicated, easy win if people don't do this as this can cause many excess queries
ids = ids.Distinct()
// don't query by anything that is a default of T (like a zero)
// TODO: I think we should enabled this in case accidental calls are made to get all with invalid ids
// .Where(x => Equals(x, default(TId)) == false)
@@ -197,39 +222,27 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll));
}
return entities;
}
/// <summary>
/// Gets a list of entities by the passed in query
/// </summary>
/// <param name="query"></param>
/// <returns></returns>
public IEnumerable<TEntity> Get(IQuery<TEntity> query)
{
return PerformGetByQuery(query)
//ensure we don't include any null refs in the returned collection!
.WhereNotNull();
}
=> PerformGetByQuery(query)
.WhereNotNull(); // ensure we don't include any null refs in the returned collection!
/// <summary>
/// Returns a boolean indicating whether an entity with the passed Id exists
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public bool Exists(TId id)
{
return CachePolicy.Exists(id, PerformExists, PerformGetAll);
}
=> CachePolicy.Exists(id, PerformExists, PerformGetAll);
/// <summary>
/// Returns an integer with the count of entities found with the passed in query
/// </summary>
/// <param name="query"></param>
/// <returns></returns>
public int Count(IQuery<TEntity> query)
{
return PerformCount(query);
}
=> PerformCount(query);
}
}

61
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/NPocoRepositoryBase.cs
View File

@@ -1,7 +1,7 @@
using System;
using System;
using System.Collections.Generic;
using NPoco;
using Microsoft.Extensions.Logging;
using NPoco;
using Umbraco.Core.Cache;
using Umbraco.Core.Models.Entities;
using Umbraco.Core.Persistence.Querying;
@@ -13,9 +13,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
/// <summary>
/// Represent an abstract Repository for NPoco based repositories
/// </summary>
/// <typeparam name="TId"></typeparam>
/// <typeparam name="TEntity"></typeparam>
public abstract class NPocoRepositoryBase<TId, TEntity> : RepositoryBase<TId, TEntity>
public abstract class NPocoRepositoryBase<TId, TEntity> : EntityRepositoryBase<TId, TEntity>
where TEntity : class, IEntity
{
/// <summary>
@@ -24,58 +22,5 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected NPocoRepositoryBase(IScopeAccessor scopeAccessor, AppCaches cache, ILogger<NPocoRepositoryBase<TId, TEntity>> logger)
: base(scopeAccessor, cache, logger)
{ }
/// <summary>
/// Gets the repository's database.
/// </summary>
protected IUmbracoDatabase Database => AmbientScope.Database;
/// <summary>
/// Gets the Sql context.
/// </summary>
protected ISqlContext SqlContext=> AmbientScope.SqlContext;
protected Sql<ISqlContext> Sql() => SqlContext.Sql();
protected Sql<ISqlContext> Sql(string sql, params object[] args) => SqlContext.Sql(sql, args);
protected ISqlSyntaxProvider SqlSyntax => SqlContext.SqlSyntax;
protected IQuery<T> Query<T>() => SqlContext.Query<T>();
#region Abstract Methods
protected abstract Sql<ISqlContext> GetBaseQuery(bool isCount); // TODO: obsolete, use QueryType instead everywhere
protected abstract string GetBaseWhereClause();
protected abstract IEnumerable<string> GetDeleteClauses();
protected abstract Guid NodeObjectTypeId { get; }
protected abstract override void PersistNewItem(TEntity entity);
protected abstract override void PersistUpdatedItem(TEntity entity);
#endregion
protected override bool PerformExists(TId id)
{
var sql = GetBaseQuery(true);
sql.Where(GetBaseWhereClause(), new { id = id});
var count = Database.ExecuteScalar<int>(sql);
return count == 1;
}
protected override int PerformCount(IQuery<TEntity> query)
{
var sqlClause = GetBaseQuery(true);
var translator = new SqlTranslator<TEntity>(sqlClause, query);
var sql = translator.Translate();
return Database.ExecuteScalar<int>(sql);
}
protected override void PersistDeletedItem(TEntity entity)
{
var deletes = GetDeleteClauses();
foreach (var delete in deletes)
{
Database.Execute(delete, new { id = GetEntityId(entity) });
}
entity.DeleteDate = DateTime.Now;
}
}
}

81
src/Umbraco.Infrastructure/Persistence/Repositories/Implement/RepositoryBase.cs Normal file
View File

@@ -0,0 +1,81 @@
using System;
using NPoco;
using Umbraco.Core.Cache;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Persistence.SqlSyntax;
using Umbraco.Core.Scoping;
namespace Umbraco.Core.Persistence.Repositories.Implement
{
/// <summary>
/// Base repository class for all <see cref="IRepository"/> instances
/// </summary>
public abstract class RepositoryBase : IRepository
{
/// <summary>
/// Initializes a new instance of the <see cref="RepositoryBase"/> class.
/// </summary>
protected RepositoryBase(IScopeAccessor scopeAccessor, AppCaches appCaches)
{
ScopeAccessor = scopeAccessor ?? throw new ArgumentNullException(nameof(scopeAccessor));
AppCaches = appCaches ?? throw new ArgumentNullException(nameof(appCaches));
}
/// <summary>
/// Gets the <see cref="AppCaches"/>
/// </summary>
protected AppCaches AppCaches { get; }
/// <summary>
/// Gets the <see cref="IScopeAccessor"/>
/// </summary>
protected IScopeAccessor ScopeAccessor { get; }
/// <summary>
/// Gets the AmbientScope
/// </summary>
protected IScope AmbientScope
{
get
{
IScope scope = ScopeAccessor.AmbientScope;
if (scope == null)
{
throw new InvalidOperationException("Cannot run a repository without an ambient scope.");
}
return scope;
}
}
/// <summary>
/// Gets the repository's database.
/// </summary>
protected IUmbracoDatabase Database => AmbientScope.Database;
/// <summary>
/// Gets the Sql context.
/// </summary>
protected ISqlContext SqlContext => AmbientScope.SqlContext;
/// <summary>
/// Gets the <see cref="ISqlSyntaxProvider"/>
/// </summary>
protected ISqlSyntaxProvider SqlSyntax => SqlContext.SqlSyntax;
/// <summary>
/// Creates an<see cref="Sql{ISqlContext}"/> expression
/// </summary>
protected Sql<ISqlContext> Sql() => SqlContext.Sql();
/// <summary>
/// Creates a <see cref="Sql{ISqlContext}"/> expression
/// </summary>
protected Sql<ISqlContext> Sql(string sql, params object[] args) => SqlContext.Sql(sql, args);
/// <summary>
/// Creates a new query expression
/// </summary>
protected IQuery<T> Query<T>() => SqlContext.Query<T>();
}
}

23
src/Umbraco.Infrastructure/Services/Implement/ContentTypeServiceBaseOfTItemTService.cs
View File

@@ -1,4 +1,4 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging;
using Umbraco.Core.Events;
using Umbraco.Core.Models;
using Umbraco.Core.Scoping;
@@ -16,10 +16,23 @@ namespace Umbraco.Core.Services.Implement
protected abstract TService This { get; }
// that one must be dispatched
/// <summary>
/// Raised when a <see cref="TItem"/> is changed
/// </summary>
/// <remarks>
/// This event is dispatched after the trans is completed. Used by event refreshers.
/// </remarks>
public static event TypedEventHandler<TService, ContentTypeChange<TItem>.EventArgs> Changed;
// that one is always immediate (transactional)
/// <summary>
/// Occurs when an <see cref="TItem"/> is created or updated from within the <see cref="IScope"/> (transaction)
/// </summary>
/// <remarks>
/// The purpose of this event being raised within the transaction is so that listeners can perform database
/// operations from within the same transaction and guarantee data consistency so that if anything goes wrong
/// the entire transaction can be rolled back. This is used by Nucache.
/// TODO: See remarks in ContentRepositoryBase about these types of events. Not sure we need/want them.
/// </remarks>
public static event TypedEventHandler<TService, ContentTypeChange<TItem>.EventArgs> ScopedRefreshedEntity;
// used by tests to clear events
@@ -45,9 +58,11 @@ namespace Umbraco.Core.Services.Implement
scope.Events.Dispatch(Changed, This, args, nameof(Changed));
}
/// <summary>
/// Raises the <see cref="ScopedRefreshedEntity"/> event during the <see cref="IScope"/> (transaction)
/// </summary>
protected void OnUowRefreshedEntity(ContentTypeChange<TItem>.EventArgs args)
{
// that one is always immediate (not dispatched, transactional)
ScopedRefreshedEntity.RaiseEvent(args, This);
}

3
src/Umbraco.ModelsBuilder.Embedded/Compose/ModelsBuilderComposer.cs
View File

@@ -1,4 +1,4 @@
using System.Linq;
using System.Linq;
using Microsoft.Extensions.DependencyInjection;
using Umbraco.Core.Configuration;
using Umbraco.Core;
@@ -11,6 +11,7 @@ using Umbraco.Core.DependencyInjection;
namespace Umbraco.ModelsBuilder.Embedded.Compose
{
// TODO: We'll need to change this stuff to IUmbracoBuilder ext and control the order of things there
[ComposeBefore(typeof(IPublishedCacheComposer))]
public sealed class ModelsBuilderComposer : ICoreComposer
{

2
src/Umbraco.PublishedCache.NuCache/ContentCache.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;

2
src/Umbraco.PublishedCache.NuCache/DataSource/ContentNestedData.cs
View File

@@ -1,4 +1,4 @@
using Newtonsoft.Json;
using Newtonsoft.Json;
using System.Collections.Generic;
using Umbraco.Core.Serialization;

325
src/Umbraco.PublishedCache.NuCache/DataSource/DatabaseDataSource.cs
View File

@@ -1,325 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using NPoco;
using Umbraco.Core;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.Dtos;
using Umbraco.Core.Scoping;
using Umbraco.Core.Serialization;
using static Umbraco.Core.Persistence.SqlExtensionsStatics;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
// TODO: use SqlTemplate for these queries else it's going to be horribly slow!
// provides efficient database access for NuCache
internal class DatabaseDataSource : IDataSource
{
private const int PageSize = 500;
private readonly ILogger<DatabaseDataSource> _logger;
public DatabaseDataSource(ILogger<DatabaseDataSource> logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
// we want arrays, we want them all loaded, not an enumerable
private Sql<ISqlContext> ContentSourcesSelect(IScope scope, Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
{
var sql = scope.SqlContext.Sql()
.Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
.AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
.AndSelect<DocumentDto>(x => Alias(x.Published, "Published"), x => Alias(x.Edited, "Edited"))
.AndSelect<ContentVersionDto>(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
.AndSelect<DocumentVersionDto>(x => Alias(x.TemplateId, "EditTemplateId"))
.AndSelect<ContentVersionDto>("pcver", x => Alias(x.Id, "PublishedVersionId"), x => Alias(x.Text, "PubName"), x => Alias(x.VersionDate, "PubVersionDate"), x => Alias(x.UserId, "PubWriterId"))
.AndSelect<DocumentVersionDto>("pdver", x => Alias(x.TemplateId, "PubTemplateId"))
.AndSelect<ContentNuDto>("nuEdit", x => Alias(x.Data, "EditData"))
.AndSelect<ContentNuDto>("nuPub", x => Alias(x.Data, "PubData"))
.From<NodeDto>();
if (joins != null)
sql = joins(sql);
sql = sql
.InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<DocumentDto>().On<NodeDto, DocumentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
.InnerJoin<DocumentVersionDto>().On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id)
.LeftJoin<ContentVersionDto>(j =>
j.InnerJoin<DocumentVersionDto>("pdver").On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id && right.Published, "pcver", "pdver"), "pcver")
.On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId, aliasRight: "pcver")
.LeftJoin<ContentNuDto>("nuEdit").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit")
.LeftJoin<ContentNuDto>("nuPub").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && right.Published, aliasRight: "nuPub");
return sql;
}
public ContentNodeKit GetContentSource(IScope scope, int id)
{
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && x.NodeId == id && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto);
}
public IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope)
{
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateContentNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id)
{
var syntax = scope.SqlContext.SqlSyntax;
var sql = ContentSourcesSelect(scope,
s => s.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateContentNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids)
{
if (!ids.Any()) yield break;
var sql = ContentSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateContentNodeKit(row);
}
private Sql<ISqlContext> MediaSourcesSelect(IScope scope, Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
{
var sql = scope.SqlContext.Sql()
.Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
.AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
.AndSelect<ContentVersionDto>(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
.AndSelect<ContentNuDto>("nuEdit", x => Alias(x.Data, "EditData"))
.From<NodeDto>();
if (joins != null)
sql = joins(sql);
sql = sql
.InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
.LeftJoin<ContentNuDto>("nuEdit").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit");
return sql;
}
public ContentNodeKit GetMediaSource(IScope scope, int id)
{
var sql = MediaSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && x.NodeId == id && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = scope.Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto);
}
public IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope)
{
var sql = MediaSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id)
{
var syntax = scope.SqlContext.SqlSyntax;
var sql = MediaSourcesSelect(scope,
s => s.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
}
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids)
{
if (!ids.Any()) yield break;
var sql = MediaSourcesSelect(scope)
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in scope.Database.QueryPaged<ContentSourceDto>(PageSize, sql))
yield return CreateMediaNodeKit(row);
}
private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto)
{
ContentData d = null;
ContentData p = null;
if (dto.Edited)
{
if (dto.EditData == null)
{
if (Debugger.IsAttached)
throw new InvalidOperationException("Missing cmsContentNu edited content for node " + dto.Id + ", consider rebuilding.");
_logger.LogWarning("Missing cmsContentNu edited content for node {NodeId}, consider rebuilding.", dto.Id);
}
else
{
var nested = DeserializeNestedData(dto.EditData);
d = new ContentData
{
Name = dto.EditName,
Published = false,
TemplateId = dto.EditTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
WriterId = dto.EditWriterId,
Properties = nested.PropertyData,
CultureInfos = nested.CultureData,
UrlSegment = nested.UrlSegment
};
}
}
if (dto.Published)
{
if (dto.PubData == null)
{
if (Debugger.IsAttached)
throw new InvalidOperationException("Missing cmsContentNu published content for node " + dto.Id + ", consider rebuilding.");
_logger.LogWarning("Missing cmsContentNu published content for node {NodeId}, consider rebuilding.", dto.Id);
}
else
{
var nested = DeserializeNestedData(dto.PubData);
p = new ContentData
{
Name = dto.PubName,
UrlSegment = nested.UrlSegment,
Published = true,
TemplateId = dto.PubTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.PubVersionDate,
WriterId = dto.PubWriterId,
Properties = nested.PropertyData,
CultureInfos = nested.CultureData
};
}
}
var n = new ContentNode(dto.Id, dto.Uid,
dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId);
var s = new ContentNodeKit
{
Node = n,
ContentTypeId = dto.ContentTypeId,
DraftData = d,
PublishedData = p
};
return s;
}
private static ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto)
{
if (dto.EditData == null)
throw new InvalidOperationException("No data for media " + dto.Id);
var nested = DeserializeNestedData(dto.EditData);
var p = new ContentData
{
Name = dto.EditName,
Published = true,
TemplateId = -1,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
WriterId = dto.CreatorId, // what-else?
Properties = nested.PropertyData,
CultureInfos = nested.CultureData
};
var n = new ContentNode(dto.Id, dto.Uid,
dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId);
var s = new ContentNodeKit
{
Node = n,
ContentTypeId = dto.ContentTypeId,
PublishedData = p
};
return s;
}
private static ContentNestedData DeserializeNestedData(string data)
{
// by default JsonConvert will deserialize our numeric values as Int64
// which is bad, because they were Int32 in the database - take care
var settings = new JsonSerializerSettings
{
Converters = new List<JsonConverter> { new ForceInt32Converter() }
};
return JsonConvert.DeserializeObject<ContentNestedData>(data, settings);
}
}
}

77
src/Umbraco.PublishedCache.NuCache/DataSource/IDataSource.cs
View File

@@ -1,77 +0,0 @@
using System.Collections.Generic;
using Umbraco.Core.Scoping;
namespace Umbraco.Web.PublishedCache.NuCache.DataSource
{
/// <summary>
/// Defines a data source for NuCache.
/// </summary>
internal interface IDataSource
{
//TODO: For these required sort orders, would sorting on Path 'just work'?
ContentNodeKit GetContentSource(IScope scope, int id);
/// <summary>
/// Returns all content ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope);
/// <summary>
/// Returns branch for content ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id);
/// <summary>
/// Returns content by Ids ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids);
ContentNodeKit GetMediaSource(IScope scope, int id);
/// <summary>
/// Returns all media ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope);
/// <summary>
/// Returns branch for media ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id); // must order by level, sortOrder
/// <summary>
/// Returns media by Ids ordered by level + sortOrder
/// </summary>
/// <param name="scope"></param>
/// <returns></returns>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids);
}
}

2
src/Umbraco.PublishedCache.NuCache/DataSource/PropertyData.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.ComponentModel;
using Newtonsoft.Json;

18
src/Umbraco.PublishedCache.NuCache/NuCacheComponent.cs
View File

@@ -1,18 +0,0 @@
using Umbraco.Core.Composing;
namespace Umbraco.Web.PublishedCache.NuCache
{
public sealed class NuCacheComponent : IComponent
{
public NuCacheComponent(IPublishedSnapshotService service)
{
// nothing - this just ensures that the service is created at boot time
}
public void Initialize()
{ }
public void Terminate()
{ }
}
}

24
src/Umbraco.PublishedCache.NuCache/NuCacheComposer.cs
View File

@@ -1,23 +1,26 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection;
using Umbraco.Core;
using Umbraco.Core.DependencyInjection;
using Umbraco.Core.Composing;
using Umbraco.Core.DependencyInjection;
using Umbraco.Core.Models;
using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Infrastructure.PublishedCache;
using Umbraco.Web.PublishedCache.NuCache.DataSource;
using Umbraco.Infrastructure.PublishedCache.Persistence;
namespace Umbraco.Web.PublishedCache.NuCache
{
public class NuCacheComposer : ComponentComposer<NuCacheComponent>, IPublishedCacheComposer
// TODO: We'll need to change this stuff to IUmbracoBuilder ext and control the order of things there,
// see comment in ModelsBuilderComposer which requires this weird IPublishedCacheComposer
public class NuCacheComposer : IComposer, IPublishedCacheComposer
{
public override void Compose(IUmbracoBuilder builder)
/// <inheritdoc/>
public void Compose(IUmbracoBuilder builder)
{
base.Compose(builder);
// register the NuCache database data source
builder.Services.AddTransient<IDataSource, DatabaseDataSource>();
builder.Services.AddSingleton<INuCacheContentRepository, NuCacheContentRepository>();
builder.Services.AddSingleton<INuCacheContentService, NuCacheContentService>();
builder.Services.AddSingleton<PublishedSnapshotServiceEventHandler>();
// register the NuCache published snapshot service
// must register default options, required in the service ctor
@@ -26,15 +29,16 @@ namespace Umbraco.Web.PublishedCache.NuCache
// replace this service since we want to improve the content/media
// mapping lookups if we are using nucache.
// TODO: Gotta wonder how much this does actually improve perf? It's a lot of weird code to make this happen so hope it's worth it
builder.Services.AddUnique<IIdKeyMap>(factory =>
{
var idkSvc = new IdKeyMap(factory.GetRequiredService<IScopeProvider>());
var publishedSnapshotService = factory.GetRequiredService<IPublishedSnapshotService>() as PublishedSnapshotService;
if (publishedSnapshotService != null)
if (factory.GetRequiredService<IPublishedSnapshotService>() is PublishedSnapshotService publishedSnapshotService)
{
idkSvc.SetMapper(UmbracoObjectTypes.Document, id => publishedSnapshotService.GetDocumentUid(id), uid => publishedSnapshotService.GetDocumentId(uid));
idkSvc.SetMapper(UmbracoObjectTypes.Media, id => publishedSnapshotService.GetMediaUid(id), uid => publishedSnapshotService.GetMediaId(uid));
}
return idkSvc;
});

46
src/Umbraco.PublishedCache.NuCache/Persistence/INuCacheContentRepository.cs Normal file
View File

@@ -0,0 +1,46 @@
using System.Collections.Generic;
using Umbraco.Core.Models;
using Umbraco.Web.PublishedCache.NuCache;
namespace Umbraco.Infrastructure.PublishedCache.Persistence
{
public interface INuCacheContentRepository
{
void DeleteContentItem(IContentBase item);
IEnumerable<ContentNodeKit> GetAllContentSources();
IEnumerable<ContentNodeKit> GetAllMediaSources();
IEnumerable<ContentNodeKit> GetBranchContentSources(int id);
IEnumerable<ContentNodeKit> GetBranchMediaSources(int id);
ContentNodeKit GetContentSource(int id);
ContentNodeKit GetMediaSource(int id);
IEnumerable<ContentNodeKit> GetTypeContentSources(IEnumerable<int> ids);
IEnumerable<ContentNodeKit> GetTypeMediaSources(IEnumerable<int> ids);
/// <summary>
/// Refreshes the nucache database row for the <see cref="IContent"/>
/// </summary>
void RefreshContent(IContent content);
/// <summary>
/// Refreshes the nucache database row for the <see cref="IContentBase"/> (used for media/members)
/// </summary>
void RefreshEntity(IContentBase content);
/// <summary>
/// Rebuilds the caches for content, media and/or members based on the content type ids specified
/// </summary>
/// <param name="groupSize">The operation batch size to process the items</param>
/// <param name="contentTypeIds">If not null will process content for the matching content types, if empty will process all content</param>
/// <param name="mediaTypeIds">If not null will process content for the matching media types, if empty will process all media</param>
/// <param name="memberTypeIds">If not null will process content for the matching members types, if empty will process all members</param>
void Rebuild(
int groupSize = 5000,
IReadOnlyCollection<int> contentTypeIds = null,
IReadOnlyCollection<int> mediaTypeIds = null,
IReadOnlyCollection<int> memberTypeIds = null);
bool VerifyContentDbCache();
bool VerifyMediaDbCache();
bool VerifyMemberDbCache();
}
}

96
src/Umbraco.PublishedCache.NuCache/Persistence/INuCacheContentService.cs Normal file
View File

@@ -0,0 +1,96 @@
using System.Collections.Generic;
using Umbraco.Core.Models;
using Umbraco.Web.PublishedCache.NuCache;
namespace Umbraco.Infrastructure.PublishedCache.Persistence
{
/// <summary>
/// Defines a data source for NuCache.
/// </summary>
public interface INuCacheContentService
{
// TODO: For these required sort orders, would sorting on Path 'just work'?
ContentNodeKit GetContentSource(int id);
/// <summary>
/// Returns all content ordered by level + sortOrder
/// </summary>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetAllContentSources();
/// <summary>
/// Returns branch for content ordered by level + sortOrder
/// </summary>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetBranchContentSources(int id);
/// <summary>
/// Returns content by Ids ordered by level + sortOrder
/// </summary>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetTypeContentSources(IEnumerable<int> ids);
ContentNodeKit GetMediaSource(int id);
/// <summary>
/// Returns all media ordered by level + sortOrder
/// </summary>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetAllMediaSources();
/// <summary>
/// Returns branch for media ordered by level + sortOrder
/// </summary>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetBranchMediaSources(int id); // must order by level, sortOrder
/// <summary>
/// Returns media by Ids ordered by level + sortOrder
/// </summary>
/// <remarks>
/// MUST be ordered by level + parentId + sortOrder!
/// </remarks>
IEnumerable<ContentNodeKit> GetTypeMediaSources(IEnumerable<int> ids);
void DeleteContentItem(IContentBase item);
/// <summary>
/// Refreshes the nucache database row for the <see cref="IContent"/>
/// </summary>
void RefreshContent(IContent content);
/// <summary>
/// Refreshes the nucache database row for the <see cref="IContentBase"/> (used for media/members)
/// </summary>
void RefreshEntity(IContentBase content);
/// <summary>
/// Rebuilds the caches for content, media and/or members based on the content type ids specified
/// </summary>
/// <param name="groupSize">The operation batch size to process the items</param>
/// <param name="contentTypeIds">If not null will process content for the matching content types, if empty will process all content</param>
/// <param name="mediaTypeIds">If not null will process content for the matching media types, if empty will process all media</param>
/// <param name="memberTypeIds">If not null will process content for the matching members types, if empty will process all members</param>
void Rebuild(
int groupSize = 5000,
IReadOnlyCollection<int> contentTypeIds = null,
IReadOnlyCollection<int> mediaTypeIds = null,
IReadOnlyCollection<int> memberTypeIds = null);
bool VerifyContentDbCache();
bool VerifyMediaDbCache();
bool VerifyMemberDbCache();
}
}

735
src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentRepository.cs Normal file
View File

@@ -0,0 +1,735 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using NPoco;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.Dtos;
using Umbraco.Core.Persistence.Querying;
using Umbraco.Core.Persistence.Repositories;
using Umbraco.Core.Persistence.Repositories.Implement;
using Umbraco.Core.Scoping;
using Umbraco.Core.Serialization;
using Umbraco.Core.Services;
using Umbraco.Core.Strings;
using Umbraco.Web.PublishedCache.NuCache;
using Umbraco.Web.PublishedCache.NuCache.DataSource;
using static Umbraco.Core.Persistence.SqlExtensionsStatics;
namespace Umbraco.Infrastructure.PublishedCache.Persistence
{
public class NuCacheContentRepository : RepositoryBase, INuCacheContentRepository
{
private const int PageSize = 500;
private readonly ILogger<NuCacheContentRepository> _logger;
private readonly IMemberRepository _memberRepository;
private readonly IDocumentRepository _documentRepository;
private readonly IMediaRepository _mediaRepository;
private readonly IShortStringHelper _shortStringHelper;
private readonly UrlSegmentProviderCollection _urlSegmentProviders;
/// <summary>
/// Initializes a new instance of the <see cref="NuCacheContentRepository"/> class.
/// </summary>
public NuCacheContentRepository(
IScopeAccessor scopeAccessor,
AppCaches appCaches,
ILogger<NuCacheContentRepository> logger,
IMemberRepository memberRepository,
IDocumentRepository documentRepository,
IMediaRepository mediaRepository,
IShortStringHelper shortStringHelper,
UrlSegmentProviderCollection urlSegmentProviders)
: base(scopeAccessor, appCaches)
{
_logger = logger;
_memberRepository = memberRepository;
_documentRepository = documentRepository;
_mediaRepository = mediaRepository;
_shortStringHelper = shortStringHelper;
_urlSegmentProviders = urlSegmentProviders;
}
public void DeleteContentItem(IContentBase item)
=> Database.Execute("DELETE FROM cmsContentNu WHERE nodeId=@id", new { id = item.Id });
public void RefreshContent(IContent content)
{
// always refresh the edited data
OnRepositoryRefreshed(content, false);
if (content.PublishedState == PublishedState.Unpublishing)
{
// if unpublishing, remove published data from table
Database.Execute("DELETE FROM cmsContentNu WHERE nodeId=@id AND published=1", new { id = content.Id });
}
else if (content.PublishedState == PublishedState.Publishing)
{
// if publishing, refresh the published data
OnRepositoryRefreshed(content, true);
}
}
public void RefreshEntity(IContentBase content)
=> OnRepositoryRefreshed(content, false);
private void OnRepositoryRefreshed(IContentBase content, bool published)
{
// use a custom SQL to update row version on each update
// db.InsertOrUpdate(dto);
ContentNuDto dto = GetDto(content, published);
Database.InsertOrUpdate(
dto,
"SET data=@data, rv=rv+1 WHERE nodeId=@id AND published=@published",
new
{
data = dto.Data,
id = dto.NodeId,
published = dto.Published
});
}
public void Rebuild(
int groupSize = 5000,
IReadOnlyCollection<int> contentTypeIds = null,
IReadOnlyCollection<int> mediaTypeIds = null,
IReadOnlyCollection<int> memberTypeIds = null)
{
if (contentTypeIds != null)
{
RebuildContentDbCache(groupSize, contentTypeIds);
}
if (mediaTypeIds != null)
{
RebuildContentDbCache(groupSize, mediaTypeIds);
}
if (memberTypeIds != null)
{
RebuildContentDbCache(groupSize, memberTypeIds);
}
}
// assumes content tree lock
private void RebuildContentDbCache(int groupSize, IReadOnlyCollection<int> contentTypeIds)
{
Guid contentObjectType = Constants.ObjectTypes.Document;
// remove all - if anything fails the transaction will rollback
if (contentTypeIds == null || contentTypeIds.Count == 0)
{
// must support SQL-CE
Database.Execute(
@"DELETE FROM cmsContentNu
WHERE cmsContentNu.nodeId IN (
SELECT id FROM umbracoNode WHERE umbracoNode.nodeObjectType=@objType
)",
new { objType = contentObjectType });
}
else
{
// assume number of ctypes won't blow IN(...)
// must support SQL-CE
Database.Execute(
$@"DELETE FROM cmsContentNu
WHERE cmsContentNu.nodeId IN (
SELECT id FROM umbracoNode
JOIN {Constants.DatabaseSchema.Tables.Content} ON {Constants.DatabaseSchema.Tables.Content}.nodeId=umbracoNode.id
WHERE umbracoNode.nodeObjectType=@objType
AND {Constants.DatabaseSchema.Tables.Content}.contentTypeId IN (@ctypes)
)",
new { objType = contentObjectType, ctypes = contentTypeIds });
}
// insert back - if anything fails the transaction will rollback
IQuery<IContent> query = SqlContext.Query<IContent>();
if (contentTypeIds != null && contentTypeIds.Count > 0)
{
query = query.WhereIn(x => x.ContentTypeId, contentTypeIds); // assume number of ctypes won't blow IN(...)
}
long pageIndex = 0;
long processed = 0;
long total;
do
{
// the tree is locked, counting and comparing to total is safe
IEnumerable<IContent> descendants = _documentRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path"));
var items = new List<ContentNuDto>();
var count = 0;
foreach (IContent c in descendants)
{
// always the edited version
items.Add(GetDto(c, false));
// and also the published version if it makes any sense
if (c.Published)
{
items.Add(GetDto(c, true));
}
count++;
}
Database.BulkInsertRecords(items);
processed += count;
} while (processed < total);
}
// assumes media tree lock
private void RebuildMediaDbCache(int groupSize, IReadOnlyCollection<int> contentTypeIds)
{
var mediaObjectType = Constants.ObjectTypes.Media;
// remove all - if anything fails the transaction will rollback
if (contentTypeIds == null || contentTypeIds.Count == 0)
{
// must support SQL-CE
Database.Execute(
@"DELETE FROM cmsContentNu
WHERE cmsContentNu.nodeId IN (
SELECT id FROM umbracoNode WHERE umbracoNode.nodeObjectType=@objType
)",
new { objType = mediaObjectType });
}
else
{
// assume number of ctypes won't blow IN(...)
// must support SQL-CE
Database.Execute(
$@"DELETE FROM cmsContentNu
WHERE cmsContentNu.nodeId IN (
SELECT id FROM umbracoNode
JOIN {Constants.DatabaseSchema.Tables.Content} ON {Constants.DatabaseSchema.Tables.Content}.nodeId=umbracoNode.id
WHERE umbracoNode.nodeObjectType=@objType
AND {Constants.DatabaseSchema.Tables.Content}.contentTypeId IN (@ctypes)
)",
new { objType = mediaObjectType, ctypes = contentTypeIds });
}
// insert back - if anything fails the transaction will rollback
var query = SqlContext.Query<IMedia>();
if (contentTypeIds != null && contentTypeIds.Count > 0)
{
query = query.WhereIn(x => x.ContentTypeId, contentTypeIds); // assume number of ctypes won't blow IN(...)
}
long pageIndex = 0;
long processed = 0;
long total;
do
{
// the tree is locked, counting and comparing to total is safe
var descendants = _mediaRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path"));
var items = descendants.Select(m => GetDto(m, false)).ToList();
Database.BulkInsertRecords(items);
processed += items.Count;
} while (processed < total);
}
// assumes member tree lock
private void RebuildMemberDbCache(int groupSize, IReadOnlyCollection<int> contentTypeIds)
{
Guid memberObjectType = Constants.ObjectTypes.Member;
// remove all - if anything fails the transaction will rollback
if (contentTypeIds == null || contentTypeIds.Count == 0)
{
// must support SQL-CE
Database.Execute(
@"DELETE FROM cmsContentNu
WHERE cmsContentNu.nodeId IN (
SELECT id FROM umbracoNode WHERE umbracoNode.nodeObjectType=@objType
)",
new { objType = memberObjectType });
}
else
{
// assume number of ctypes won't blow IN(...)
// must support SQL-CE
Database.Execute(
$@"DELETE FROM cmsContentNu
WHERE cmsContentNu.nodeId IN (
SELECT id FROM umbracoNode
JOIN {Constants.DatabaseSchema.Tables.Content} ON {Constants.DatabaseSchema.Tables.Content}.nodeId=umbracoNode.id
WHERE umbracoNode.nodeObjectType=@objType
AND {Constants.DatabaseSchema.Tables.Content}.contentTypeId IN (@ctypes)
)",
new { objType = memberObjectType, ctypes = contentTypeIds });
}
// insert back - if anything fails the transaction will rollback
IQuery<IMember> query = SqlContext.Query<IMember>();
if (contentTypeIds != null && contentTypeIds.Count > 0)
{
query = query.WhereIn(x => x.ContentTypeId, contentTypeIds); // assume number of ctypes won't blow IN(...)
}
long pageIndex = 0;
long processed = 0;
long total;
do
{
IEnumerable<IMember> descendants = _memberRepository.GetPage(query, pageIndex++, groupSize, out total, null, Ordering.By("Path"));
ContentNuDto[] items = descendants.Select(m => GetDto(m, false)).ToArray();
Database.BulkInsertRecords(items);
processed += items.Length;
} while (processed < total);
}
// assumes content tree lock
public bool VerifyContentDbCache()
{
// every document should have a corresponding row for edited properties
// and if published, may have a corresponding row for published properties
Guid contentObjectType = Constants.ObjectTypes.Document;
var count = Database.ExecuteScalar<int>(
$@"SELECT COUNT(*)
FROM umbracoNode
JOIN {Constants.DatabaseSchema.Tables.Document} ON umbracoNode.id={Constants.DatabaseSchema.Tables.Document}.nodeId
LEFT JOIN cmsContentNu nuEdited ON (umbracoNode.id=nuEdited.nodeId AND nuEdited.published=0)
LEFT JOIN cmsContentNu nuPublished ON (umbracoNode.id=nuPublished.nodeId AND nuPublished.published=1)
WHERE umbracoNode.nodeObjectType=@objType
AND nuEdited.nodeId IS NULL OR ({Constants.DatabaseSchema.Tables.Document}.published=1 AND nuPublished.nodeId IS NULL);",
new { objType = contentObjectType });
return count == 0;
}
// assumes media tree lock
public bool VerifyMediaDbCache()
{
// every media item should have a corresponding row for edited properties
Guid mediaObjectType = Constants.ObjectTypes.Media;
var count = Database.ExecuteScalar<int>(
@"SELECT COUNT(*)
FROM umbracoNode
LEFT JOIN cmsContentNu ON (umbracoNode.id=cmsContentNu.nodeId AND cmsContentNu.published=0)
WHERE umbracoNode.nodeObjectType=@objType
AND cmsContentNu.nodeId IS NULL
", new { objType = mediaObjectType });
return count == 0;
}
// assumes member tree lock
public bool VerifyMemberDbCache()
{
// every member item should have a corresponding row for edited properties
var memberObjectType = Constants.ObjectTypes.Member;
var count = Database.ExecuteScalar<int>(
@"SELECT COUNT(*)
FROM umbracoNode
LEFT JOIN cmsContentNu ON (umbracoNode.id=cmsContentNu.nodeId AND cmsContentNu.published=0)
WHERE umbracoNode.nodeObjectType=@objType
AND cmsContentNu.nodeId IS NULL
", new { objType = memberObjectType });
return count == 0;
}
private ContentNuDto GetDto(IContentBase content, bool published)
{
// should inject these in ctor
// BUT for the time being we decide not to support ConvertDbToXml/String
// var propertyEditorResolver = PropertyEditorResolver.Current;
// var dataTypeService = ApplicationContext.Current.Services.DataTypeService;
var propertyData = new Dictionary<string, PropertyData[]>();
foreach (IProperty prop in content.Properties)
{
var pdatas = new List<PropertyData>();
foreach (IPropertyValue pvalue in prop.Values)
{
// sanitize - properties should be ok but ... never knows
if (!prop.PropertyType.SupportsVariation(pvalue.Culture, pvalue.Segment))
{
continue;
}
// note: at service level, invariant is 'null', but here invariant becomes 'string.Empty'
var value = published ? pvalue.PublishedValue : pvalue.EditedValue;
if (value != null)
{
pdatas.Add(new PropertyData { Culture = pvalue.Culture ?? string.Empty, Segment = pvalue.Segment ?? string.Empty, Value = value });
}
}
propertyData[prop.Alias] = pdatas.ToArray();
}
var cultureData = new Dictionary<string, CultureVariation>();
// sanitize - names should be ok but ... never knows
if (content.ContentType.VariesByCulture())
{
ContentCultureInfosCollection infos = content is IContent document
? published
? document.PublishCultureInfos
: document.CultureInfos
: content.CultureInfos;
// ReSharper disable once UseDeconstruction
foreach (ContentCultureInfos cultureInfo in infos)
{
var cultureIsDraft = !published && content is IContent d && d.IsCultureEdited(cultureInfo.Culture);
cultureData[cultureInfo.Culture] = new CultureVariation
{
Name = cultureInfo.Name,
UrlSegment = content.GetUrlSegment(_shortStringHelper, _urlSegmentProviders, cultureInfo.Culture),
Date = content.GetUpdateDate(cultureInfo.Culture) ?? DateTime.MinValue,
IsDraft = cultureIsDraft
};
}
}
// the dictionary that will be serialized
var nestedData = new ContentNestedData
{
PropertyData = propertyData,
CultureData = cultureData,
UrlSegment = content.GetUrlSegment(_shortStringHelper, _urlSegmentProviders)
};
var dto = new ContentNuDto
{
NodeId = content.Id,
Published = published,
// note that numeric values (which are Int32) are serialized without their
// type (eg "value":1234) and JsonConvert by default deserializes them as Int64
Data = JsonConvert.SerializeObject(nestedData)
};
return dto;
}
// we want arrays, we want them all loaded, not an enumerable
private Sql<ISqlContext> ContentSourcesSelect(Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
{
var sql = Sql()
.Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
.AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
.AndSelect<DocumentDto>(x => Alias(x.Published, "Published"), x => Alias(x.Edited, "Edited"))
.AndSelect<ContentVersionDto>(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
.AndSelect<DocumentVersionDto>(x => Alias(x.TemplateId, "EditTemplateId"))
.AndSelect<ContentVersionDto>("pcver", x => Alias(x.Id, "PublishedVersionId"), x => Alias(x.Text, "PubName"), x => Alias(x.VersionDate, "PubVersionDate"), x => Alias(x.UserId, "PubWriterId"))
.AndSelect<DocumentVersionDto>("pdver", x => Alias(x.TemplateId, "PubTemplateId"))
.AndSelect<ContentNuDto>("nuEdit", x => Alias(x.Data, "EditData"))
.AndSelect<ContentNuDto>("nuPub", x => Alias(x.Data, "PubData"))
.From<NodeDto>();
if (joins != null)
{
sql = joins(sql);
}
sql = sql
.InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<DocumentDto>().On<NodeDto, DocumentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
.InnerJoin<DocumentVersionDto>().On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id)
.LeftJoin<ContentVersionDto>(j =>
j.InnerJoin<DocumentVersionDto>("pdver").On<ContentVersionDto, DocumentVersionDto>((left, right) => left.Id == right.Id && right.Published, "pcver", "pdver"), "pcver")
.On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId, aliasRight: "pcver")
.LeftJoin<ContentNuDto>("nuEdit").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit")
.LeftJoin<ContentNuDto>("nuPub").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && right.Published, aliasRight: "nuPub");
return sql;
}
public ContentNodeKit GetContentSource(int id)
{
var sql = ContentSourcesSelect()
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && x.NodeId == id && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateContentNodeKit(dto);
}
public IEnumerable<ContentNodeKit> GetAllContentSources()
{
var sql = ContentSourcesSelect()
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateContentNodeKit(row);
}
}
public IEnumerable<ContentNodeKit> GetBranchContentSources(int id)
{
var syntax = SqlSyntax;
var sql = ContentSourcesSelect(
s => s.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateContentNodeKit(row);
}
}
public IEnumerable<ContentNodeKit> GetTypeContentSources(IEnumerable<int> ids)
{
if (!ids.Any())
yield break;
var sql = ContentSourcesSelect()
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Document && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateContentNodeKit(row);
}
}
private Sql<ISqlContext> MediaSourcesSelect(Func<Sql<ISqlContext>, Sql<ISqlContext>> joins = null)
{
var sql = Sql()
.Select<NodeDto>(x => Alias(x.NodeId, "Id"), x => Alias(x.UniqueId, "Uid"),
x => Alias(x.Level, "Level"), x => Alias(x.Path, "Path"), x => Alias(x.SortOrder, "SortOrder"), x => Alias(x.ParentId, "ParentId"),
x => Alias(x.CreateDate, "CreateDate"), x => Alias(x.UserId, "CreatorId"))
.AndSelect<ContentDto>(x => Alias(x.ContentTypeId, "ContentTypeId"))
.AndSelect<ContentVersionDto>(x => Alias(x.Id, "VersionId"), x => Alias(x.Text, "EditName"), x => Alias(x.VersionDate, "EditVersionDate"), x => Alias(x.UserId, "EditWriterId"))
.AndSelect<ContentNuDto>("nuEdit", x => Alias(x.Data, "EditData"))
.From<NodeDto>();
if (joins != null)
{
sql = joins(sql);
}
sql = sql
.InnerJoin<ContentDto>().On<NodeDto, ContentDto>((left, right) => left.NodeId == right.NodeId)
.InnerJoin<ContentVersionDto>().On<NodeDto, ContentVersionDto>((left, right) => left.NodeId == right.NodeId && right.Current)
.LeftJoin<ContentNuDto>("nuEdit").On<NodeDto, ContentNuDto>((left, right) => left.NodeId == right.NodeId && !right.Published, aliasRight: "nuEdit");
return sql;
}
public ContentNodeKit GetMediaSource(int id)
{
var sql = MediaSourcesSelect()
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && x.NodeId == id && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
var dto = Database.Fetch<ContentSourceDto>(sql).FirstOrDefault();
return dto == null ? new ContentNodeKit() : CreateMediaNodeKit(dto);
}
public IEnumerable<ContentNodeKit> GetAllMediaSources()
{
var sql = MediaSourcesSelect()
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateMediaNodeKit(row);
}
}
public IEnumerable<ContentNodeKit> GetBranchMediaSources(int id)
{
var syntax = SqlSyntax;
var sql = MediaSourcesSelect(
s => s.InnerJoin<NodeDto>("x").On<NodeDto, NodeDto>((left, right) => left.NodeId == right.NodeId || SqlText<bool>(left.Path, right.Path, (lp, rp) => $"({lp} LIKE {syntax.GetConcat(rp, "',%'")})"), aliasRight: "x"))
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.Where<NodeDto>(x => x.NodeId == id, "x")
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateMediaNodeKit(row);
}
}
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IEnumerable<int> ids)
{
if (!ids.Any())
{
yield break;
}
var sql = MediaSourcesSelect()
.Where<NodeDto>(x => x.NodeObjectType == Constants.ObjectTypes.Media && !x.Trashed)
.WhereIn<ContentDto>(x => x.ContentTypeId, ids)
.OrderBy<NodeDto>(x => x.Level, x => x.ParentId, x => x.SortOrder);
// We need to page here. We don't want to iterate over every single row in one connection cuz this can cause an SQL Timeout.
// We also want to read with a db reader and not load everything into memory, QueryPaged lets us do that.
foreach (var row in Database.QueryPaged<ContentSourceDto>(PageSize, sql))
{
yield return CreateMediaNodeKit(row);
}
}
private ContentNodeKit CreateContentNodeKit(ContentSourceDto dto)
{
ContentData d = null;
ContentData p = null;
if (dto.Edited)
{
if (dto.EditData == null)
{
if (Debugger.IsAttached)
{
throw new InvalidOperationException("Missing cmsContentNu edited content for node " + dto.Id + ", consider rebuilding.");
}
_logger.LogWarning("Missing cmsContentNu edited content for node {NodeId}, consider rebuilding.", dto.Id);
}
else
{
var nested = DeserializeNestedData(dto.EditData);
d = new ContentData
{
Name = dto.EditName,
Published = false,
TemplateId = dto.EditTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
WriterId = dto.EditWriterId,
Properties = nested.PropertyData,
CultureInfos = nested.CultureData,
UrlSegment = nested.UrlSegment
};
}
}
if (dto.Published)
{
if (dto.PubData == null)
{
if (Debugger.IsAttached)
{
throw new InvalidOperationException("Missing cmsContentNu published content for node " + dto.Id + ", consider rebuilding.");
}
_logger.LogWarning("Missing cmsContentNu published content for node {NodeId}, consider rebuilding.", dto.Id);
}
else
{
var nested = DeserializeNestedData(dto.PubData);
p = new ContentData
{
Name = dto.PubName,
UrlSegment = nested.UrlSegment,
Published = true,
TemplateId = dto.PubTemplateId,
VersionId = dto.VersionId,
VersionDate = dto.PubVersionDate,
WriterId = dto.PubWriterId,
Properties = nested.PropertyData,
CultureInfos = nested.CultureData
};
}
}
var n = new ContentNode(dto.Id, dto.Uid,
dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId);
var s = new ContentNodeKit
{
Node = n,
ContentTypeId = dto.ContentTypeId,
DraftData = d,
PublishedData = p
};
return s;
}
private static ContentNodeKit CreateMediaNodeKit(ContentSourceDto dto)
{
if (dto.EditData == null)
throw new InvalidOperationException("No data for media " + dto.Id);
var nested = DeserializeNestedData(dto.EditData);
var p = new ContentData
{
Name = dto.EditName,
Published = true,
TemplateId = -1,
VersionId = dto.VersionId,
VersionDate = dto.EditVersionDate,
WriterId = dto.CreatorId, // what-else?
Properties = nested.PropertyData,
CultureInfos = nested.CultureData
};
var n = new ContentNode(dto.Id, dto.Uid,
dto.Level, dto.Path, dto.SortOrder, dto.ParentId, dto.CreateDate, dto.CreatorId);
var s = new ContentNodeKit
{
Node = n,
ContentTypeId = dto.ContentTypeId,
PublishedData = p
};
return s;
}
private static ContentNestedData DeserializeNestedData(string data)
{
// by default JsonConvert will deserialize our numeric values as Int64
// which is bad, because they were Int32 in the database - take care
var settings = new JsonSerializerSettings
{
Converters = new List<JsonConverter> { new ForceInt32Converter() }
};
return JsonConvert.DeserializeObject<ContentNestedData>(data, settings);
}
}
}

107
src/Umbraco.PublishedCache.NuCache/Persistence/NuCacheContentService.cs Normal file
View File

@@ -0,0 +1,107 @@
using System.Collections.Generic;
using Microsoft.Extensions.Logging;
using Umbraco.Core;
using Umbraco.Core.Events;
using Umbraco.Core.Models;
using Umbraco.Core.Scoping;
using Umbraco.Core.Services.Implement;
using Umbraco.Web.PublishedCache.NuCache;
namespace Umbraco.Infrastructure.PublishedCache.Persistence
{
public class NuCacheContentService : RepositoryService, INuCacheContentService
{
private readonly INuCacheContentRepository _repository;
public NuCacheContentService(INuCacheContentRepository repository, IScopeProvider provider, ILoggerFactory loggerFactory, IEventMessagesFactory eventMessagesFactory)
: base(provider, loggerFactory, eventMessagesFactory)
{
_repository = repository;
}
/// <inheritdoc/>
public IEnumerable<ContentNodeKit> GetAllContentSources()
=> _repository.GetAllContentSources();
/// <inheritdoc/>
public IEnumerable<ContentNodeKit> GetAllMediaSources()
=> _repository.GetAllMediaSources();
/// <inheritdoc/>
public IEnumerable<ContentNodeKit> GetBranchContentSources(int id)
=> _repository.GetBranchContentSources(id);
/// <inheritdoc/>
public IEnumerable<ContentNodeKit> GetBranchMediaSources(int id)
=> _repository.GetBranchMediaSources(id);
/// <inheritdoc/>
public ContentNodeKit GetContentSource(int id)
=> _repository.GetContentSource(id);
/// <inheritdoc/>
public ContentNodeKit GetMediaSource(int id)
=> _repository.GetMediaSource(id);
/// <inheritdoc/>
public IEnumerable<ContentNodeKit> GetTypeContentSources(IEnumerable<int> ids)
=> _repository.GetTypeContentSources(ids);
/// <inheritdoc/>
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IEnumerable<int> ids)
=> _repository.GetTypeContentSources(ids);
/// <inheritdoc/>
public void DeleteContentItem(IContentBase item)
=> _repository.DeleteContentItem(item);
/// <inheritdoc/>
public void RefreshContent(IContent content)
=> _repository.RefreshContent(content);
/// <inheritdoc/>
public void RefreshEntity(IContentBase content)
=> _repository.RefreshEntity(content);
/// <inheritdoc/>
public void Rebuild(
int groupSize = 5000,
IReadOnlyCollection<int> contentTypeIds = null,
IReadOnlyCollection<int> mediaTypeIds = null,
IReadOnlyCollection<int> memberTypeIds = null)
{
using (IScope scope = ScopeProvider.CreateScope(repositoryCacheMode: RepositoryCacheMode.Scoped))
{
if (contentTypeIds != null)
{
scope.ReadLock(Constants.Locks.ContentTree);
}
if (mediaTypeIds != null)
{
scope.ReadLock(Constants.Locks.MediaTree);
}
if (memberTypeIds != null)
{
scope.ReadLock(Constants.Locks.MemberTree);
}
_repository.Rebuild(groupSize, contentTypeIds, mediaTypeIds, memberTypeIds);
scope.Complete();
}
}
/// <inheritdoc/>
public bool VerifyContentDbCache()
=> _repository.VerifyContentDbCache();
/// <inheritdoc/>
public bool VerifyMediaDbCache()
=> _repository.VerifyMediaDbCache();
/// <inheritdoc/>
public bool VerifyMemberDbCache()
=> _repository.VerifyMemberDbCache();
}
}

896
src/Umbraco.PublishedCache.NuCache/PublishedSnapshotService.cs
View File

File diff suppressed because it is too large Load Diff

187
src/Umbraco.PublishedCache.NuCache/PublishedSnapshotServiceEventHandler.cs Normal file
View File

@@ -0,0 +1,187 @@
using System;
using System.Linq;
using Umbraco.Core;
using Umbraco.Core.Models;
using Umbraco.Core.Persistence;
using Umbraco.Core.Persistence.Repositories.Implement;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Services.Implement;
using Umbraco.Infrastructure.PublishedCache.Persistence;
namespace Umbraco.Web.PublishedCache.NuCache
{
public class PublishedSnapshotServiceEventHandler : IDisposable
{
private readonly IRuntimeState _runtime;
private bool _disposedValue;
private readonly IPublishedSnapshotService _publishedSnapshotService;
private readonly INuCacheContentService _publishedContentService;
public PublishedSnapshotServiceEventHandler(
IRuntimeState runtime,
IPublishedSnapshotService publishedSnapshotService,
INuCacheContentService publishedContentService)
{
_runtime = runtime;
_publishedSnapshotService = publishedSnapshotService;
_publishedContentService = publishedContentService;
}
public bool Start()
{
// however, the cache is NOT available until we are configured, because loading
// content (and content types) from database cannot be consistent (see notes in "Handle
// Notifications" region), so
// - notifications will be ignored
// - trying to obtain a published snapshot from the service will throw
if (_runtime.Level != RuntimeLevel.Run)
{
return false;
}
// this initializes the caches.
// TODO: This is still temporal coupling (i.e. Initialize)
_publishedSnapshotService.LoadCachesOnStartup();
// we always want to handle repository events, configured or not
// assuming no repository event will trigger before the whole db is ready
// (ideally we'd have Upgrading.App vs Upgrading.Data application states...)
InitializeRepositoryEvents();
return true;
}
private void InitializeRepositoryEvents()
{
// TODO: The reason these events are in the repository is for legacy, the events should exist at the service
// level now since we can fire these events within the transaction... so move the events to service level
// plug repository event handlers
// these trigger within the transaction to ensure consistency
// and are used to maintain the central, database-level XML cache
DocumentRepository.ScopeEntityRemove += OnContentRemovingEntity;
DocumentRepository.ScopedEntityRefresh += DocumentRepository_ScopedEntityRefresh;
MediaRepository.ScopeEntityRemove += OnMediaRemovingEntity;
MediaRepository.ScopedEntityRefresh += MediaRepository_ScopedEntityRefresh;
MemberRepository.ScopeEntityRemove += OnMemberRemovingEntity;
MemberRepository.ScopedEntityRefresh += MemberRepository_ScopedEntityRefresh;
// plug
ContentTypeService.ScopedRefreshedEntity += OnContentTypeRefreshedEntity;
MediaTypeService.ScopedRefreshedEntity += OnMediaTypeRefreshedEntity;
MemberTypeService.ScopedRefreshedEntity += OnMemberTypeRefreshedEntity;
// TODO: This should be a cache refresher call!
LocalizationService.SavedLanguage += OnLanguageSaved;
}
private void TearDownRepositoryEvents()
{
DocumentRepository.ScopeEntityRemove -= OnContentRemovingEntity;
DocumentRepository.ScopedEntityRefresh -= DocumentRepository_ScopedEntityRefresh;
MediaRepository.ScopeEntityRemove -= OnMediaRemovingEntity;
MediaRepository.ScopedEntityRefresh -= MediaRepository_ScopedEntityRefresh;
MemberRepository.ScopeEntityRemove -= OnMemberRemovingEntity;
MemberRepository.ScopedEntityRefresh -= MemberRepository_ScopedEntityRefresh;
ContentTypeService.ScopedRefreshedEntity -= OnContentTypeRefreshedEntity;
MediaTypeService.ScopedRefreshedEntity -= OnMediaTypeRefreshedEntity;
MemberTypeService.ScopedRefreshedEntity -= OnMemberTypeRefreshedEntity;
LocalizationService.SavedLanguage -= OnLanguageSaved; // TODO: Shouldn't this be a cache refresher event?
}
// note: if the service is not ready, ie _isReady is false, then we still handle repository events,
// because we can, we do not need a working published snapshot to do it - the only reason why it could cause an
// issue is if the database table is not ready, but that should be prevented by migrations.
// we need them to be "repository" events ie to trigger from within the repository transaction,
// because they need to be consistent with the content that is being refreshed/removed - and that
// should be guaranteed by a DB transaction
private void OnContentRemovingEntity(DocumentRepository sender, DocumentRepository.ScopedEntityEventArgs args)
=> _publishedContentService.DeleteContentItem(args.Entity);
private void OnMediaRemovingEntity(MediaRepository sender, MediaRepository.ScopedEntityEventArgs args)
=> _publishedContentService.DeleteContentItem(args.Entity);
private void OnMemberRemovingEntity(MemberRepository sender, MemberRepository.ScopedEntityEventArgs args)
=> _publishedContentService.DeleteContentItem(args.Entity);
private void MemberRepository_ScopedEntityRefresh(MemberRepository sender, ContentRepositoryBase<int, IMember, MemberRepository>.ScopedEntityEventArgs e)
=> _publishedContentService.RefreshEntity(e.Entity);
private void MediaRepository_ScopedEntityRefresh(MediaRepository sender, ContentRepositoryBase<int, IMedia, MediaRepository>.ScopedEntityEventArgs e)
=> _publishedContentService.RefreshEntity(e.Entity);
private void DocumentRepository_ScopedEntityRefresh(DocumentRepository sender, ContentRepositoryBase<int, IContent, DocumentRepository>.ScopedEntityEventArgs e)
=> _publishedContentService.RefreshContent(e.Entity);
private void OnContentTypeRefreshedEntity(IContentTypeService sender, ContentTypeChange<IContentType>.EventArgs args)
{
const ContentTypeChangeTypes types // only for those that have been refreshed
= ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther;
var contentTypeIds = args.Changes.Where(x => x.ChangeTypes.HasTypesAny(types)).Select(x => x.Item.Id).ToArray();
if (contentTypeIds.Any())
{
_publishedSnapshotService.Rebuild(contentTypeIds: contentTypeIds);
}
}
private void OnMediaTypeRefreshedEntity(IMediaTypeService sender, ContentTypeChange<IMediaType>.EventArgs args)
{
const ContentTypeChangeTypes types // only for those that have been refreshed
= ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther;
var mediaTypeIds = args.Changes.Where(x => x.ChangeTypes.HasTypesAny(types)).Select(x => x.Item.Id).ToArray();
if (mediaTypeIds.Any())
{
_publishedSnapshotService.Rebuild(mediaTypeIds: mediaTypeIds);
}
}
private void OnMemberTypeRefreshedEntity(IMemberTypeService sender, ContentTypeChange<IMemberType>.EventArgs args)
{
const ContentTypeChangeTypes types // only for those that have been refreshed
= ContentTypeChangeTypes.RefreshMain | ContentTypeChangeTypes.RefreshOther;
var memberTypeIds = args.Changes.Where(x => x.ChangeTypes.HasTypesAny(types)).Select(x => x.Item.Id).ToArray();
if (memberTypeIds.Any())
{
_publishedSnapshotService.Rebuild(memberTypeIds: memberTypeIds);
}
}
/// <summary>
/// If a <see cref="ILanguage"/> is ever saved with a different culture, we need to rebuild all of the content nucache table
/// </summary>
private void OnLanguageSaved(ILocalizationService sender, Core.Events.SaveEventArgs<ILanguage> e)
{
// TODO: This should be a cache refresher call!
// culture changed on an existing language
var cultureChanged = e.SavedEntities.Any(x => !x.WasPropertyDirty(nameof(ILanguage.Id)) && x.WasPropertyDirty(nameof(ILanguage.IsoCode)));
if (cultureChanged)
{
// Rebuild all content types
_publishedSnapshotService.Rebuild(contentTypeIds: Array.Empty<int>());
}
}
protected virtual void Dispose(bool disposing)
{
if (!_disposedValue)
{
if (disposing)
{
TearDownRepositoryEvents();
}
_disposedValue = true;
}
}
public void Dispose()
{
// Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

4
src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/Repositories/DocumentRepositoryTest.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Extensions.Logging;
@@ -114,7 +114,7 @@ namespace Umbraco.Tests.Integration.Umbraco.Infrastructure.Persistence.Repositor
var languageRepository = new LanguageRepository(scopeAccessor, appCaches, LoggerFactory.CreateLogger<LanguageRepository>(), globalSettings);
contentTypeRepository = new ContentTypeRepository(scopeAccessor, appCaches, LoggerFactory.CreateLogger<ContentTypeRepository>(), commonRepository, languageRepository, ShortStringHelper);
var relationTypeRepository = new RelationTypeRepository(scopeAccessor, AppCaches.Disabled, LoggerFactory.CreateLogger<RelationTypeRepository>());
var entityRepository = new EntityRepository(scopeAccessor);
var entityRepository = new EntityRepository(scopeAccessor, AppCaches.Disabled);
var relationRepository = new RelationRepository(scopeAccessor, LoggerFactory.CreateLogger<RelationRepository>(), relationTypeRepository, entityRepository);
var propertyEditors = new Lazy<PropertyEditorCollection>(() => new PropertyEditorCollection(new DataEditorCollection(Enumerable.Empty<IDataEditor>())));
var dataValueReferences = new DataValueReferenceFactoryCollection(Enumerable.Empty<IDataValueReferenceFactory>());

5
src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/Repositories/EntityRepositoryTest.cs
View File

@@ -1,7 +1,8 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Models;
using Umbraco.Core.Models.Entities;
using Umbraco.Core.Persistence.Repositories.Implement;
@@ -19,7 +20,7 @@ namespace Umbraco.Tests.Integration.Umbraco.Infrastructure.Persistence.Repositor
{
private EntityRepository CreateRepository(IScopeAccessor scopeAccessor)
{
var entityRepository = new EntityRepository(scopeAccessor);
var entityRepository = new EntityRepository(scopeAccessor, AppCaches.Disabled);
return entityRepository;
}

4
src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/Repositories/MediaRepositoryTest.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Linq;
using Microsoft.Extensions.Logging;
using Moq;
@@ -53,7 +53,7 @@ namespace Umbraco.Tests.Integration.Umbraco.Infrastructure.Persistence.Repositor
mediaTypeRepository = new MediaTypeRepository(scopeAccessor, appCaches, LoggerFactory.CreateLogger<MediaTypeRepository>(), commonRepository, languageRepository, ShortStringHelper);
var tagRepository = new TagRepository(scopeAccessor, appCaches, LoggerFactory.CreateLogger<TagRepository>());
var relationTypeRepository = new RelationTypeRepository(scopeAccessor, AppCaches.Disabled, LoggerFactory.CreateLogger<RelationTypeRepository>());
var entityRepository = new EntityRepository(scopeAccessor);
var entityRepository = new EntityRepository(scopeAccessor, AppCaches.Disabled);
var relationRepository = new RelationRepository(scopeAccessor, LoggerFactory.CreateLogger<RelationRepository>(), relationTypeRepository, entityRepository);
var propertyEditors = new Lazy<PropertyEditorCollection>(() => new PropertyEditorCollection(new DataEditorCollection(Enumerable.Empty<IDataEditor>())));
var mediaUrlGenerators = new MediaUrlGeneratorCollection(Enumerable.Empty<IMediaUrlGenerator>());

4
src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/Repositories/RelationRepositoryTest.cs
View File

@@ -19,7 +19,7 @@ using Umbraco.Tests.Testing;
namespace Umbraco.Tests.Integration.Umbraco.Infrastructure.Persistence.Repositories
{
[TestFixture]
[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest, Boot = true)]
[UmbracoTest(Database = UmbracoTestOptions.Database.NewSchemaPerTest)]
public class RelationRepositoryTest : UmbracoIntegrationTest
{
private RelationType _relateContent;
@@ -432,7 +432,7 @@ namespace Umbraco.Tests.Integration.Umbraco.Infrastructure.Persistence.Repositor
{
var accessor = (IScopeAccessor)ScopeProvider;
var relationTypeRepository = new RelationTypeRepository(accessor, AppCaches.Disabled, Mock.Of<ILogger<RelationTypeRepository>>());
var entityRepository = new EntityRepository(accessor);
var entityRepository = new EntityRepository(accessor, AppCaches.Disabled);
var relationRepository = new RelationRepository(accessor, Mock.Of<ILogger<RelationRepository>>(), relationTypeRepository, entityRepository);
relationTypeRepository.Save(_relateContent);

4
src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Persistence/Repositories/TemplateRepositoryTest.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -262,7 +262,7 @@ namespace Umbraco.Tests.Integration.Umbraco.Infrastructure.Persistence.Repositor
var languageRepository = new LanguageRepository(scopeAccessor, AppCaches.Disabled, LoggerFactory.CreateLogger<LanguageRepository>(), Microsoft.Extensions.Options.Options.Create(globalSettings));
var contentTypeRepository = new ContentTypeRepository(scopeAccessor, AppCaches.Disabled, LoggerFactory.CreateLogger<ContentTypeRepository>(), commonRepository, languageRepository, ShortStringHelper);
var relationTypeRepository = new RelationTypeRepository(scopeAccessor, AppCaches.Disabled, LoggerFactory.CreateLogger<RelationTypeRepository>());
var entityRepository = new EntityRepository(scopeAccessor);
var entityRepository = new EntityRepository(scopeAccessor, AppCaches.Disabled);
var relationRepository = new RelationRepository(scopeAccessor, LoggerFactory.CreateLogger<RelationRepository>(), relationTypeRepository, entityRepository);
var propertyEditors = new Lazy<PropertyEditorCollection>(() => new PropertyEditorCollection(new DataEditorCollection(Enumerable.Empty<IDataEditor>())));
var dataValueReferences = new DataValueReferenceFactoryCollection(Enumerable.Empty<IDataValueReferenceFactory>());

7
src/Umbraco.Tests.Integration/Umbraco.Infrastructure/Services/EntityServiceTests.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
@@ -42,10 +42,9 @@ namespace Umbraco.Tests.Integration.Umbraco.Infrastructure.Services
[SetUp]
public void SetupTestData()
{
// This is super nasty, but this lets us initialize the cache while it is empty.
var publishedSnapshotService = GetRequiredService<IPublishedSnapshotService>() as PublishedSnapshotService;
publishedSnapshotService?.OnApplicationInit(null, EventArgs.Empty);
// var publishedSnapshotService = GetRequiredService<IPublishedSnapshotService>() as PublishedSnapshotService;
// publishedSnapshotService?.OnApplicationInit(null, EventArgs.Empty);
if (_langFr == null && _langEs == null)
{

17
src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
@@ -33,6 +33,7 @@ using Umbraco.Web.PublishedCache.NuCache.DataSource;
using Current = Umbraco.Web.Composing.Current;
using Umbraco.Core.Serialization;
using Umbraco.Net;
using Umbraco.Infrastructure.PublishedCache.Persistence;
namespace Umbraco.Tests.PublishedContent
{
@@ -148,11 +149,9 @@ namespace Umbraco.Tests.PublishedContent
// at last, create the complete NuCache snapshot service!
var options = new PublishedSnapshotServiceOptions { IgnoreLocalDb = true };
var lifetime = new Mock<IUmbracoApplicationLifetime>();
_snapshotService = new PublishedSnapshotService(options,
_snapshotService = new PublishedSnapshotService(
options,
null,
lifetime.Object,
runtime,
serviceContext,
contentTypeFactory,
_snapshotAccessor,
@@ -160,23 +159,17 @@ namespace Umbraco.Tests.PublishedContent
Mock.Of<IProfilingLogger>(),
NullLoggerFactory.Instance,
scopeProvider.Object,
Mock.Of<IDocumentRepository>(),
Mock.Of<IMediaRepository>(),
Mock.Of<IMemberRepository>(),
new TestDefaultCultureAccessor(),
_source,
new TestDefaultCultureAccessor(),
Options.Create(globalSettings),
Mock.Of<IEntityXmlSerializer>(),
PublishedModelFactory,
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }),
hostingEnvironment,
Mock.Of<IShortStringHelper>(),
TestHelper.IOHelper,
Options.Create(nuCacheSettings));
// invariant is the current default
_variationAccesor.VariationContext = new VariationContext();
lifetime.Raise(e => e.ApplicationInit += null, EventArgs.Empty);
Mock.Get(factory).Setup(x => x.GetService(typeof(IVariationContextAccessor))).Returns(_variationAccesor);
}

17
src/Umbraco.Tests/PublishedContent/NuCacheTests.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
@@ -189,11 +189,9 @@ namespace Umbraco.Tests.PublishedContent
// at last, create the complete NuCache snapshot service!
var options = new PublishedSnapshotServiceOptions { IgnoreLocalDb = true };
var lifetime = new Mock<IUmbracoApplicationLifetime>();
_snapshotService = new PublishedSnapshotService(options,
_snapshotService = new PublishedSnapshotService(
options,
null,
lifetime.Object,
runtime,
serviceContext,
contentTypeFactory,
new TestPublishedSnapshotAccessor(),
@@ -201,22 +199,15 @@ namespace Umbraco.Tests.PublishedContent
Mock.Of<IProfilingLogger>(),
NullLoggerFactory.Instance,
scopeProvider,
Mock.Of<IDocumentRepository>(),
Mock.Of<IMediaRepository>(),
Mock.Of<IMemberRepository>(),
new TestDefaultCultureAccessor(),
dataSource,
new TestDefaultCultureAccessor(),
Microsoft.Extensions.Options.Options.Create(globalSettings),
Mock.Of<IEntityXmlSerializer>(),
publishedModelFactory,
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(TestHelper.ShortStringHelper) }),
TestHelper.GetHostingEnvironment(),
Mock.Of<IShortStringHelper>(),
TestHelper.IOHelper,
Microsoft.Extensions.Options.Options.Create(nuCacheSettings));
lifetime.Raise(e => e.ApplicationInit += null, EventArgs.Empty);
// invariant is the current default
_variationAccesor.VariationContext = new VariationContext();

15
src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Web.Routing;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
@@ -17,6 +17,7 @@ using Umbraco.Core.Services;
using Umbraco.Core.Services.Implement;
using Umbraco.Core.Strings;
using Umbraco.Core.Sync;
using Umbraco.Infrastructure.PublishedCache.Persistence;
using Umbraco.Net;
using Umbraco.Tests.Common;
using Umbraco.Tests.TestHelpers;
@@ -71,7 +72,7 @@ namespace Umbraco.Tests.Scoping
protected override IPublishedSnapshotService CreatePublishedSnapshotService(GlobalSettings globalSettings = null)
{
var options = new PublishedSnapshotServiceOptions { IgnoreLocalDb = true };
var publishedSnapshotAccessor = new UmbracoContextPublishedSnapshotAccessor(Umbraco.Web.Composing.Current.UmbracoContextAccessor);
var publishedSnapshotAccessor = new UmbracoContextPublishedSnapshotAccessor(Current.UmbracoContextAccessor);
var runtimeStateMock = new Mock<IRuntimeState>();
runtimeStateMock.Setup(x => x.Level).Returns(() => RuntimeLevel.Run);
@@ -85,27 +86,23 @@ namespace Umbraco.Tests.Scoping
var nuCacheSettings = new NuCacheSettings();
var lifetime = new Mock<IUmbracoApplicationLifetime>();
var repository = new NuCacheContentRepository(ScopeProvider, AppCaches.Disabled, Mock.Of<ILogger<NuCacheContentRepository>>(), memberRepository, documentRepository, mediaRepository, Mock.Of<IShortStringHelper>(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(ShortStringHelper) }));
var snapshotService = new PublishedSnapshotService(
options,
null,
lifetime.Object,
runtimeStateMock.Object,
ServiceContext,
contentTypeFactory,
publishedSnapshotAccessor,
Mock.Of<IVariationContextAccessor>(),
ProfilingLogger,
base.ProfilingLogger,
NullLoggerFactory.Instance,
ScopeProvider,
documentRepository, mediaRepository, memberRepository,
new NuCacheContentService(repository, ScopeProvider, NullLoggerFactory.Instance, Mock.Of<IEventMessagesFactory>()),
DefaultCultureAccessor,
new DatabaseDataSource(Mock.Of<ILogger<DatabaseDataSource>>()),
Microsoft.Extensions.Options.Options.Create(globalSettings ?? new GlobalSettings()),
Factory.GetRequiredService<IEntityXmlSerializer>(),
new NoopPublishedModelFactory(),
new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider(ShortStringHelper) }),
hostingEnvironment,
Mock.Of<IShortStringHelper>(),
IOHelper,
Microsoft.Extensions.Options.Options.Create(nuCacheSettings));

50
src/Umbraco.Tests/Testing/Objects/TestDataSource.cs
View File

@@ -1,16 +1,17 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using Umbraco.Core.Models;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.Scoping;
using Umbraco.Infrastructure.PublishedCache.Persistence;
using Umbraco.Web;
using Umbraco.Web.PublishedCache.NuCache;
using Umbraco.Web.PublishedCache.NuCache.DataSource;
namespace Umbraco.Tests.Testing.Objects
{
internal class TestDataSource : IDataSource
internal class TestDataSource : INuCacheContentService
{
private IPublishedModelFactory PublishedModelFactory { get; } = new NoopPublishedModelFactory();
@@ -19,27 +20,23 @@ namespace Umbraco.Tests.Testing.Objects
: this((IEnumerable<ContentNodeKit>) kits)
{ }
public TestDataSource(IEnumerable<ContentNodeKit> kits)
{
Kits = kits.ToDictionary(x => x.Node.Id, x => x);
}
public TestDataSource(IEnumerable<ContentNodeKit> kits) => Kits = kits.ToDictionary(x => x.Node.Id, x => x);
public Dictionary<int, ContentNodeKit> Kits { get; }
// note: it is important to clone the returned kits, as the inner
// ContentNode is directly reused and modified by the snapshot service
public ContentNodeKit GetContentSource(int id)
=> Kits.TryGetValue(id, out ContentNodeKit kit) ? kit.Clone(PublishedModelFactory) : default;
public ContentNodeKit GetContentSource(IScope scope, int id)
=> Kits.TryGetValue(id, out var kit) ? kit.Clone(PublishedModelFactory) : default;
public IEnumerable<ContentNodeKit> GetAllContentSources(IScope scope)
public IEnumerable<ContentNodeKit> GetAllContentSources()
=> Kits.Values
.OrderBy(x => x.Node.Level)
.ThenBy(x => x.Node.ParentContentId)
.ThenBy(x => x.Node.SortOrder)
.Select(x => x.Clone(PublishedModelFactory));
public IEnumerable<ContentNodeKit> GetBranchContentSources(IScope scope, int id)
public IEnumerable<ContentNodeKit> GetBranchContentSources(int id)
=> Kits.Values
.Where(x => x.Node.Path.EndsWith("," + id) || x.Node.Path.Contains("," + id + ","))
.OrderBy(x => x.Node.Level)
@@ -47,7 +44,7 @@ namespace Umbraco.Tests.Testing.Objects
.ThenBy(x => x.Node.SortOrder)
.Select(x => x.Clone(PublishedModelFactory));
public IEnumerable<ContentNodeKit> GetTypeContentSources(IScope scope, IEnumerable<int> ids)
public IEnumerable<ContentNodeKit> GetTypeContentSources(IEnumerable<int> ids)
=> Kits.Values
.Where(x => ids.Contains(x.ContentTypeId))
.OrderBy(x => x.Node.Level)
@@ -55,24 +52,19 @@ namespace Umbraco.Tests.Testing.Objects
.ThenBy(x => x.Node.SortOrder)
.Select(x => x.Clone(PublishedModelFactory));
public ContentNodeKit GetMediaSource(IScope scope, int id)
{
return default;
}
public ContentNodeKit GetMediaSource(int id) => default;
public IEnumerable<ContentNodeKit> GetAllMediaSources(IScope scope)
{
return Enumerable.Empty<ContentNodeKit>();
}
public IEnumerable<ContentNodeKit> GetAllMediaSources() => Enumerable.Empty<ContentNodeKit>();
public IEnumerable<ContentNodeKit> GetBranchMediaSources(IScope scope, int id)
{
return Enumerable.Empty<ContentNodeKit>();
}
public IEnumerable<ContentNodeKit> GetBranchMediaSources(int id) => Enumerable.Empty<ContentNodeKit>();
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IScope scope, IEnumerable<int> ids)
{
return Enumerable.Empty<ContentNodeKit>();
}
public IEnumerable<ContentNodeKit> GetTypeMediaSources(IEnumerable<int> ids) => Enumerable.Empty<ContentNodeKit>();
public void DeleteContentItem(IContentBase item) => throw new NotImplementedException();
public void RefreshContent(IContent content) => throw new NotImplementedException();
public void RefreshEntity(IContentBase content) => throw new NotImplementedException();
public bool VerifyContentDbCache() => throw new NotImplementedException();
public bool VerifyMediaDbCache() => throw new NotImplementedException();
public bool VerifyMemberDbCache() => throw new NotImplementedException();
public void Rebuild(int groupSize = 5000, IReadOnlyCollection<int> contentTypeIds = null, IReadOnlyCollection<int> mediaTypeIds = null, IReadOnlyCollection<int> memberTypeIds = null) => throw new NotImplementedException();
}
}

12
src/Umbraco.Web.Common/Extensions/ApplicationBuilderExtensions.cs
View File

@@ -11,6 +11,7 @@ using Umbraco.Core;
using Umbraco.Core.Hosting;
using Umbraco.Infrastructure.Logging.Serilog.Enrichers;
using Umbraco.Web.Common.Middleware;
using Umbraco.Web.PublishedCache.NuCache;
namespace Umbraco.Extensions
{
@@ -36,6 +37,7 @@ namespace Umbraco.Extensions
// We need to add this before UseRouting so that the UmbracoContext and other middlewares are executed
// before endpoint routing middleware.
app.UseUmbracoRouting();
app.UseUmbracoContentCache();
app.UseStatusCodePages();
@@ -176,6 +178,16 @@ namespace Umbraco.Extensions
return app;
}
/// <summary>
/// Enables the Umbraco content cache
/// </summary>
public static IApplicationBuilder UseUmbracoContentCache(this IApplicationBuilder app)
{
PublishedSnapshotServiceEventHandler publishedContentEvents = app.ApplicationServices.GetRequiredService<PublishedSnapshotServiceEventHandler>();
publishedContentEvents.Start();
return app;
}
/// <summary>
/// Ensures the runtime is shutdown when the application is shutting down
/// </summary>

2
src/Umbraco.Web.Common/Middleware/UmbracoRequestMiddleware.cs
View File

@@ -55,7 +55,7 @@ namespace Umbraco.Web.Common.Middleware
return;
}
_backofficeSecurityFactory.EnsureBackOfficeSecurity(); // Needs to be before UmbracoContext
_backofficeSecurityFactory.EnsureBackOfficeSecurity(); // Needs to be before UmbracoContext, TODO: Why?
UmbracoContextReference umbracoContextReference = _umbracoContextFactory.EnsureUmbracoContext();
try

1
src/Umbraco.Web.Common/Umbraco.Web.Common.csproj
View File

@@ -17,6 +17,7 @@
<ItemGroup>
<ProjectReference Include="..\Umbraco.Core\Umbraco.Core.csproj" />
<ProjectReference Include="..\Umbraco.Infrastructure\Umbraco.Infrastructure.csproj" />
<ProjectReference Include="..\Umbraco.PublishedCache.NuCache\Umbraco.PublishedCache.NuCache.csproj" />
</ItemGroup>
<ItemGroup>

2
src/Umbraco.Web.Website/Routing/UmbracoRouteValueTransformer.cs
View File

@@ -190,7 +190,7 @@ namespace Umbraco.Web.Website.Routing
{
ControllerActionDescriptor descriptor = _actionDescriptorCollectionProvider.ActionDescriptors.Items
.Cast<ControllerActionDescriptor>()
.First(x =>
.FirstOrDefault(x =>
x.ControllerName.Equals(controllerName));
return descriptor?.ControllerTypeInfo;