diff --git a/src/Umbraco.Core/Models/CacheInstruction.cs b/src/Umbraco.Core/Models/CacheInstruction.cs new file mode 100644 index 0000000000..000788c8a0 --- /dev/null +++ b/src/Umbraco.Core/Models/CacheInstruction.cs @@ -0,0 +1,51 @@ +using System; +using System.Runtime.Serialization; + +namespace Umbraco.Cms.Core.Models +{ + /// + /// Represents a cache instruction. + /// + [Serializable] + [DataContract(IsReference = true)] + public class CacheInstruction + { + /// + /// Initializes a new instance of the class. + /// + public CacheInstruction(int id, DateTime utcStamp, string instructions, string originIdentity, int instructionCount) + { + Id = id; + UtcStamp = utcStamp; + Instructions = instructions; + OriginIdentity = originIdentity; + InstructionCount = instructionCount; + } + + /// + /// Cache instruction Id. + /// + public int Id { get; private set; } + + /// + /// Cache instruction created date. + /// + public DateTime UtcStamp { get; private set; } + + /// + /// Serialized instructions. + /// + public string Instructions { get; private set; } + + /// + /// Identity of server originating the instruction. + /// + public string OriginIdentity { get; private set; } + + /// + /// Count of instructions. + /// + public int InstructionCount { get; private set; } + + } +} diff --git a/src/Umbraco.Core/Persistence/Repositories/ICacheInstructionRepository.cs b/src/Umbraco.Core/Persistence/Repositories/ICacheInstructionRepository.cs new file mode 100644 index 0000000000..0381c74a03 --- /dev/null +++ b/src/Umbraco.Core/Persistence/Repositories/ICacheInstructionRepository.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using Umbraco.Cms.Core.Models; + +namespace Umbraco.Cms.Core.Persistence.Repositories +{ + /// + /// Represents a repository for entities. + /// + public interface ICacheInstructionRepository : IRepository + { + /// + /// Gets the count of pending cache instruction records. + /// + int CountAll(); + + /// + /// Gets the count of pending cache instructions. + /// + int CountPendingInstructions(int lastId); + + /// + /// Gets the most recent cache instruction record Id. + /// + /// + int GetMaxId(); + + /// + /// Checks to see if a single cache instruction by Id exists. + /// + bool Exists(int id); + + /// + /// Adds a new cache instruction record. + /// + void Add(CacheInstruction cacheInstruction); + + /// + /// Gets a collection of cache instructions created later than the provided Id. + /// + /// Last id processed. + /// The maximum number of instructions to retrieve. + IEnumerable GetInstructions(int lastId, int maxNumberToRetrieve); + + /// + /// Deletes cache instructions older than the provided date. + /// + void DeleteInstructionsOlderThan(DateTime pruneDate); + } +} diff --git a/src/Umbraco.Core/Services/CacheInstructionServiceInitializationResult.cs b/src/Umbraco.Core/Services/CacheInstructionServiceInitializationResult.cs new file mode 100644 index 0000000000..e7cea8ef33 --- /dev/null +++ b/src/Umbraco.Core/Services/CacheInstructionServiceInitializationResult.cs @@ -0,0 +1,31 @@ +namespace Umbraco.Cms.Core.Services +{ + /// + /// Defines a result object for the operation. + /// + public class CacheInstructionServiceInitializationResult + { + private CacheInstructionServiceInitializationResult() + { + } + + public bool Initialized { get; private set; } + + public bool ColdBootRequired { get; private set; } + + public int MaxId { get; private set; } + + public int LastId { get; private set; } + + public static CacheInstructionServiceInitializationResult AsUninitialized() => new CacheInstructionServiceInitializationResult { Initialized = false }; + + public static CacheInstructionServiceInitializationResult AsInitialized(bool coldBootRequired, int maxId, int lastId) => + new CacheInstructionServiceInitializationResult + { + Initialized = true, + ColdBootRequired = coldBootRequired, + MaxId = maxId, + LastId = lastId, + }; + } +} diff --git a/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs b/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs new file mode 100644 index 0000000000..79d8ec1bbb --- /dev/null +++ b/src/Umbraco.Core/Services/CacheInstructionServiceProcessInstructionsResult.cs @@ -0,0 +1,24 @@ +using System; + +namespace Umbraco.Cms.Core.Services +{ + /// + /// Defines a result object for the operation. + /// + public class CacheInstructionServiceProcessInstructionsResult + { + private CacheInstructionServiceProcessInstructionsResult() + { + } + + public int LastId { get; private set; } + + public bool InstructionsWerePruned { get; private set; } + + public static CacheInstructionServiceProcessInstructionsResult AsCompleted(int lastId) => + new CacheInstructionServiceProcessInstructionsResult { LastId = lastId }; + + public static CacheInstructionServiceProcessInstructionsResult AsCompletedAndPruned(int lastId) => + new CacheInstructionServiceProcessInstructionsResult { LastId = lastId, InstructionsWerePruned = true }; + }; +} diff --git a/src/Umbraco.Core/Services/ICacheInstructionService.cs b/src/Umbraco.Core/Services/ICacheInstructionService.cs new file mode 100644 index 0000000000..4589053fa8 --- /dev/null +++ b/src/Umbraco.Core/Services/ICacheInstructionService.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using Umbraco.Cms.Core.Sync; + +namespace Umbraco.Cms.Core.Services +{ + public interface ICacheInstructionService + { + /// + /// Ensures that the cache instruction service is initialized and can be used for syncing messages. + /// + CacheInstructionServiceInitializationResult EnsureInitialized(bool released, int lastId); + + /// + /// Creates a cache instruction record from a set of individual instructions and saves it. + /// + void DeliverInstructions(IEnumerable instructions, string localIdentity); + + /// + /// Creates one or more cache instruction records base on the configured batch size from a set of individual instructions and saves them. + /// + void DeliverInstructionsInBatches(IEnumerable instructions, string localIdentity); + + /// + /// Processes and then prunes pending database cache instructions. + /// + /// Flag indicating if process is shutting now and operations should exit. + /// Local local identity of the executing AppDomain. + /// Date of last prune operation. + CacheInstructionServiceProcessInstructionsResult ProcessInstructions(bool released, string localIdentity, DateTime lastPruned); + } +} diff --git a/src/Umbraco.Infrastructure/Sync/RefreshInstruction.cs b/src/Umbraco.Core/Sync/RefreshInstruction.cs similarity index 74% rename from src/Umbraco.Infrastructure/Sync/RefreshInstruction.cs rename to src/Umbraco.Core/Sync/RefreshInstruction.cs index ec6ffd0ed8..c01d758583 100644 --- a/src/Umbraco.Infrastructure/Sync/RefreshInstruction.cs +++ b/src/Umbraco.Core/Sync/RefreshInstruction.cs @@ -1,11 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; -using Newtonsoft.Json; using Umbraco.Cms.Core.Cache; -using Umbraco.Cms.Core.Sync; +using Umbraco.Cms.Core.Serialization; -namespace Umbraco.Cms.Infrastructure.Sync +namespace Umbraco.Cms.Core.Sync { [Serializable] public class RefreshInstruction @@ -17,15 +16,27 @@ namespace Umbraco.Cms.Infrastructure.Sync // need this public, parameter-less constructor so the web service messenger // can de-serialize the instructions it receives - public RefreshInstruction() - { - //set default - this value is not used for reading after it's been deserialized, it's only used for persisting the instruction to the db - JsonIdCount = 1; - } - // need this public one so it can be de-serialized - used by the Json thing - // otherwise, should use GetInstructions(...) + /// + /// Initializes a new instance of the class. + /// + /// + /// Need this public, parameter-less constructor so the web service messenger can de-serialize the instructions it receives. + /// + public RefreshInstruction() => + + // Set default - this value is not used for reading after it's been deserialized, it's only used for persisting the instruction to the db + JsonIdCount = 1; + + /// + /// Initializes a new instance of the class. + /// + /// + /// Need this public one so it can be de-serialized - used by the Json thing + /// otherwise, should use GetInstructions(...) + /// public RefreshInstruction(Guid refresherId, RefreshMethodType refreshType, Guid guidId, int intId, string jsonIds, string jsonPayload) + : this() { RefresherId = refresherId; RefreshType = refreshType; @@ -33,36 +44,24 @@ namespace Umbraco.Cms.Infrastructure.Sync IntId = intId; JsonIds = jsonIds; JsonPayload = jsonPayload; - //set default - this value is not used for reading after it's been deserialized, it's only used for persisting the instruction to the db - JsonIdCount = 1; } private RefreshInstruction(ICacheRefresher refresher, RefreshMethodType refreshType) + : this() { RefresherId = refresher.RefresherUniqueId; RefreshType = refreshType; - //set default - this value is not used for reading after it's been deserialized, it's only used for persisting the instruction to the db - JsonIdCount = 1; } private RefreshInstruction(ICacheRefresher refresher, RefreshMethodType refreshType, Guid guidId) - : this(refresher, refreshType) - { - GuidId = guidId; - } + : this(refresher, refreshType) => GuidId = guidId; private RefreshInstruction(ICacheRefresher refresher, RefreshMethodType refreshType, int intId) - : this(refresher, refreshType) - { - IntId = intId; - } + : this(refresher, refreshType) => IntId = intId; /// /// A private constructor to create a new instance /// - /// - /// - /// /// /// When the refresh method is we know how many Ids are being refreshed so we know the instruction /// count which will be taken into account when we store this count in the database. @@ -73,13 +72,18 @@ namespace Umbraco.Cms.Infrastructure.Sync JsonIdCount = idCount; if (refreshType == RefreshMethodType.RefreshByJson) + { JsonPayload = json; + } else + { JsonIds = json; + } } public static IEnumerable GetInstructions( ICacheRefresher refresher, + IJsonSerializer jsonSerializer, MessageType messageType, IEnumerable ids, Type idType, @@ -95,20 +99,27 @@ namespace Umbraco.Cms.Infrastructure.Sync case MessageType.RefreshById: if (idType == null) + { throw new InvalidOperationException("Cannot refresh by id if idType is null."); + } + if (idType == typeof(int)) { - // bulk of ints is supported + // Bulk of ints is supported var intIds = ids.Cast().ToArray(); - return new[] { new RefreshInstruction(refresher, RefreshMethodType.RefreshByIds, JsonConvert.SerializeObject(intIds), intIds.Length) }; + return new[] { new RefreshInstruction(refresher, RefreshMethodType.RefreshByIds, jsonSerializer.Serialize(intIds), intIds.Length) }; } - // else must be guids, bulk of guids is not supported, iterate + + // Else must be guids, bulk of guids is not supported, so iterate. return ids.Select(x => new RefreshInstruction(refresher, RefreshMethodType.RefreshByGuid, (Guid) x)); case MessageType.RemoveById: if (idType == null) + { throw new InvalidOperationException("Cannot remove by id if idType is null."); - // must be ints, bulk-remove is not supported, iterate + } + + // Must be ints, bulk-remove is not supported, so iterate. return ids.Select(x => new RefreshInstruction(refresher, RefreshMethodType.RemoveById, (int) x)); //return new[] { new RefreshInstruction(refresher, RefreshMethodType.RemoveByIds, JsonConvert.SerializeObject(ids.Cast().ToArray())) }; @@ -145,10 +156,10 @@ namespace Umbraco.Cms.Infrastructure.Sync public string JsonIds { get; set; } /// - /// Gets or sets the number of Ids contained in the JsonIds json value + /// Gets or sets the number of Ids contained in the JsonIds json value. /// /// - /// This is used to determine the instruction count per row + /// This is used to determine the instruction count per row. /// public int JsonIdCount { get; set; } @@ -157,21 +168,31 @@ namespace Umbraco.Cms.Infrastructure.Sync /// public string JsonPayload { get; set; } - protected bool Equals(RefreshInstruction other) - { - return RefreshType == other.RefreshType + protected bool Equals(RefreshInstruction other) => + RefreshType == other.RefreshType && RefresherId.Equals(other.RefresherId) && GuidId.Equals(other.GuidId) && IntId == other.IntId && string.Equals(JsonIds, other.JsonIds) && string.Equals(JsonPayload, other.JsonPayload); - } public override bool Equals(object other) { - if (ReferenceEquals(null, other)) return false; - if (ReferenceEquals(this, other)) return true; - if (other.GetType() != this.GetType()) return false; + if (other is null) + { + return false; + } + + if (ReferenceEquals(this, other)) + { + return true; + } + + if (other.GetType() != GetType()) + { + return false; + } + return Equals((RefreshInstruction) other); } @@ -189,14 +210,8 @@ namespace Umbraco.Cms.Infrastructure.Sync } } - public static bool operator ==(RefreshInstruction left, RefreshInstruction right) - { - return Equals(left, right); - } + public static bool operator ==(RefreshInstruction left, RefreshInstruction right) => Equals(left, right); - public static bool operator !=(RefreshInstruction left, RefreshInstruction right) - { - return Equals(left, right) == false; - } + public static bool operator !=(RefreshInstruction left, RefreshInstruction right) => Equals(left, right) == false; } } diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Repositories.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Repositories.cs index 8292fd2ecb..9b57e8586c 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Repositories.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Repositories.cs @@ -18,6 +18,7 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection // repositories builder.Services.AddUnique(); builder.Services.AddUnique(); + builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(); diff --git a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Services.cs b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Services.cs index 97e32451a5..30e8ae37f8 100644 --- a/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Services.cs +++ b/src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Services.cs @@ -37,6 +37,7 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(); + builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(); builder.Services.AddUnique(); diff --git a/src/Umbraco.Infrastructure/Persistence/Factories/CacheInstructionFactory.cs b/src/Umbraco.Infrastructure/Persistence/Factories/CacheInstructionFactory.cs new file mode 100644 index 0000000000..1a38348acf --- /dev/null +++ b/src/Umbraco.Infrastructure/Persistence/Factories/CacheInstructionFactory.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using System.Linq; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Infrastructure.Persistence.Dtos; + +namespace Umbraco.Cms.Infrastructure.Persistence.Factories +{ + internal static class CacheInstructionFactory + { + public static IEnumerable BuildEntities(IEnumerable dtos) => dtos.Select(BuildEntity).ToList(); + + public static CacheInstruction BuildEntity(CacheInstructionDto dto) => + new CacheInstruction(dto.Id, dto.UtcStamp, dto.Instructions, dto.OriginIdentity, dto.InstructionCount); + + public static CacheInstructionDto BuildDto(CacheInstruction entity) => + new CacheInstructionDto + { + Id = entity.Id, + UtcStamp = entity.UtcStamp, + Instructions = entity.Instructions, + OriginIdentity = entity.OriginIdentity, + InstructionCount = entity.InstructionCount, + }; + } +} diff --git a/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/CacheInstructionRepository.cs b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/CacheInstructionRepository.cs new file mode 100644 index 0000000000..f5452b53c0 --- /dev/null +++ b/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/CacheInstructionRepository.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NPoco; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Persistence.Repositories; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Infrastructure.Persistence.Dtos; +using Umbraco.Cms.Infrastructure.Persistence.Factories; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement +{ + /// + /// Represents the NPoco implementation of . + /// + internal class CacheInstructionRepository : ICacheInstructionRepository + { + private readonly IScopeAccessor _scopeAccessor; + + public CacheInstructionRepository(IScopeAccessor scopeAccessor) => _scopeAccessor = scopeAccessor; + + /// + private IScope AmbientScope => _scopeAccessor.AmbientScope; + + /// + public int CountAll() + { + Sql sql = AmbientScope.SqlContext.Sql().Select("COUNT(*)") + .From(); + + return AmbientScope.Database.ExecuteScalar(sql); + } + + /// + public int CountPendingInstructions(int lastId) => + AmbientScope.Database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new { lastId }); + + /// + public int GetMaxId() => + AmbientScope.Database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); + + /// + public bool Exists(int id) => AmbientScope.Database.Exists(id); + + /// + public void Add(CacheInstruction cacheInstruction) + { + CacheInstructionDto dto = CacheInstructionFactory.BuildDto(cacheInstruction); + AmbientScope.Database.Insert(dto); + } + + /// + public IEnumerable GetInstructions(int lastId, int maxNumberToRetrieve) + { + Sql sql = AmbientScope.SqlContext.Sql().SelectAll() + .From() + .Where(dto => dto.Id > lastId) + .OrderBy(dto => dto.Id); + Sql topSql = sql.SelectTop(maxNumberToRetrieve); + return AmbientScope.Database.Fetch(topSql).Select(CacheInstructionFactory.BuildEntity); + } + + /// + public void DeleteInstructionsOlderThan(DateTime pruneDate) + { + // Using 2 queries is faster than convoluted joins. + var maxId = AmbientScope.Database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction;"); + Sql deleteSql = new Sql().Append(@"DELETE FROM umbracoCacheInstruction WHERE utcStamp < @pruneDate AND id < @maxId", new { pruneDate, maxId }); + AmbientScope.Database.Execute(deleteSql); + } + } +} diff --git a/src/Umbraco.Infrastructure/Services/Implement/AuditService.cs b/src/Umbraco.Infrastructure/Services/Implement/AuditService.cs index cc8110f38e..05c489102f 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/AuditService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/AuditService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.Logging; @@ -11,7 +11,7 @@ using Umbraco.Cms.Infrastructure.Persistence.Dtos; namespace Umbraco.Cms.Core.Services.Implement { - public sealed class AuditService : ScopeRepositoryService, IAuditService + public sealed class AuditService : RepositoryService, IAuditService { private readonly Lazy _isAvailable; private readonly IAuditRepository _auditRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs b/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs new file mode 100644 index 0000000000..6ca01ccb50 --- /dev/null +++ b/src/Umbraco.Infrastructure/Services/Implement/CacheInstructionService.cs @@ -0,0 +1,501 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Umbraco.Cms.Core.Cache; +using Umbraco.Cms.Core.Configuration.Models; +using Umbraco.Cms.Core.Events; +using Umbraco.Cms.Core.Logging; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Persistence.Repositories; +using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Sync; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Core.Services.Implement +{ + /// + /// Implements providing a service for retrieving and saving cache instructions. + /// + public class CacheInstructionService : RepositoryService, ICacheInstructionService + { + private readonly IServerRoleAccessor _serverRoleAccessor; + private readonly CacheRefresherCollection _cacheRefreshers; + private readonly ICacheInstructionRepository _cacheInstructionRepository; + private readonly IProfilingLogger _profilingLogger; + private readonly ILogger _logger; + private readonly GlobalSettings _globalSettings; + + private readonly object _locko = new object(); + + /// + /// Initializes a new instance of the class. + /// + public CacheInstructionService( + IScopeProvider provider, + ILoggerFactory loggerFactory, + IEventMessagesFactory eventMessagesFactory, + IServerRoleAccessor serverRoleAccessor, + CacheRefresherCollection cacheRefreshers, + ICacheInstructionRepository cacheInstructionRepository, + IProfilingLogger profilingLogger, + ILogger logger, + IOptions globalSettings) + : base(provider, loggerFactory, eventMessagesFactory) + { + _serverRoleAccessor = serverRoleAccessor; + _cacheRefreshers = cacheRefreshers; + _cacheInstructionRepository = cacheInstructionRepository; + _profilingLogger = profilingLogger; + _logger = logger; + _globalSettings = globalSettings.Value; + } + + /// + public CacheInstructionServiceInitializationResult EnsureInitialized(bool released, int lastId) + { + using (IScope scope = ScopeProvider.CreateScope(autoComplete: true)) + { + lastId = EnsureInstructions(lastId); // reset _lastId if instructions are missing + return Initialize(released, lastId); // boot + } + } + + /// + /// Ensure that the last instruction that was processed is still in the database. + /// + /// + /// If the last instruction is not in the database anymore, then the messenger + /// should not try to process any instructions, because some instructions might be lost, + /// and it should instead cold-boot. + /// However, if the last synced instruction id is '0' and there are '0' records, then this indicates + /// that it's a fresh site and no user actions have taken place, in this circumstance we do not want to cold + /// boot. See: http://issues.umbraco.org/issue/U4-8627 + /// + private int EnsureInstructions(int lastId) + { + if (lastId == 0) + { + var count = _cacheInstructionRepository.CountAll(); + + // If there are instructions but we haven't synced, then a cold boot is necessary. + if (count > 0) + { + lastId = -1; + } + } + else + { + // If the last synced instruction is not found in the db, then a cold boot is necessary. + if (!_cacheInstructionRepository.Exists(lastId)) + { + lastId = -1; + } + } + + return lastId; + } + + /// + /// Initializes a server that has never synchronized before. + /// + /// + /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. + /// Callers MUST ensure thread-safety. + /// + private CacheInstructionServiceInitializationResult Initialize(bool released, int lastId) + { + lock (_locko) + { + if (released) + { + return CacheInstructionServiceInitializationResult.AsUninitialized(); + } + + var coldboot = false; + + // Never synced before. + if (lastId < 0) + { + // We haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new + // server and it will need to rebuild it's own caches, e.g. Lucene or the XML cache file. + _logger.LogWarning("No last synced Id found, this generally means this is a new server/install." + + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" + + " the database and maintain cache updates based on that Id."); + + coldboot = true; + } + else + { + // Check for how many instructions there are to process. Each row contains a count of the number of instructions contained in each + // row so we will sum these numbers to get the actual count. + var count = _cacheInstructionRepository.CountPendingInstructions(lastId); + if (count > _globalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount) + { + // Too many instructions, proceed to cold boot + _logger.LogWarning( + "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." + + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" + + " to the latest found in the database and maintain cache updates based on that Id.", + count, _globalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount); + + coldboot = true; + } + } + + // If cold boot is required, go get the last id in the db and store it. + // Note: do it BEFORE initializing otherwise some instructions might get lost. + // When doing it before, some instructions might run twice - not an issue. + var maxId = coldboot + ? _cacheInstructionRepository.GetMaxId() + : 0; + + return CacheInstructionServiceInitializationResult.AsInitialized(coldboot, maxId, lastId); + } + } + + /// + public void DeliverInstructions(IEnumerable instructions, string localIdentity) + { + CacheInstruction entity = CreateCacheInstruction(instructions, localIdentity); + + using (IScope scope = ScopeProvider.CreateScope()) + { + _cacheInstructionRepository.Add(entity); + } + } + + /// + public void DeliverInstructionsInBatches(IEnumerable instructions, string localIdentity) + { + // Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount. + using (IScope scope = ScopeProvider.CreateScope()) + { + foreach (IEnumerable instructionsBatch in instructions.InGroupsOf(_globalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount)) + { + CacheInstruction entity = CreateCacheInstruction(instructionsBatch, localIdentity); + _cacheInstructionRepository.Add(entity); + } + + scope.Complete(); + } + } + + private CacheInstruction CreateCacheInstruction(IEnumerable instructions, string localIdentity) => + new CacheInstruction(0, DateTime.UtcNow, JsonConvert.SerializeObject(instructions, Formatting.None), localIdentity, instructions.Sum(x => x.JsonIdCount)); + + /// + public CacheInstructionServiceProcessInstructionsResult ProcessInstructions(bool released, string localIdentity, DateTime lastPruned) + { + using (_profilingLogger.DebugDuration("Syncing from database...")) + using (IScope scope = ScopeProvider.CreateScope()) + { + ProcessDatabaseInstructions(released, localIdentity, out int lastId); + + // Check for pruning throttling. + if (released || (DateTime.UtcNow - lastPruned) <= _globalSettings.DatabaseServerMessenger.TimeBetweenPruneOperations) + { + scope.Complete(); + return CacheInstructionServiceProcessInstructionsResult.AsCompleted(lastId); + } + + switch (_serverRoleAccessor.CurrentServerRole) + { + case ServerRole.Single: + case ServerRole.Master: + PruneOldInstructions(); + break; + } + + scope.Complete(); + return CacheInstructionServiceProcessInstructionsResult.AsCompletedAndPruned(lastId); + } + } + + /// + /// Process instructions from the database. + /// + /// + /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. + /// + private void ProcessDatabaseInstructions(bool released, string localIdentity, out int lastId) + { + // NOTE: + // We 'could' recurse to ensure that no remaining instructions are pending in the table before proceeding but I don't think that + // would be a good idea since instructions could keep getting added and then all other threads will probably get stuck from serving requests + // (depending on what the cache refreshers are doing). I think it's best we do the one time check, process them and continue, if there are + // pending requests after being processed, they'll just be processed on the next poll. + // + // TODO: not true if we're running on a background thread, assuming we can? + + // Only retrieve the top 100 (just in case there are tons). + // Even though MaxProcessingInstructionCount is by default 1000 we still don't want to process that many + // rows in one request thread since each row can contain a ton of instructions (until 7.5.5 in which case + // a row can only contain MaxProcessingInstructionCount). + const int MaxInstructionsToRetrieve = 100; + + // Only process instructions coming from a remote server, and ignore instructions coming from + // the local server as they've already been processed. We should NOT assume that the sequence of + // instructions in the database makes any sense whatsoever, because it's all async. + + // Tracks which ones have already been processed to avoid duplicates + var processed = new HashSet(); + + // It would have been nice to do this in a Query instead of Fetch using a data reader to save + // some memory however we cannot do that because inside of this loop the cache refreshers are also + // performing some lookups which cannot be done with an active reader open. + lastId = 0; + foreach (CacheInstruction instruction in _cacheInstructionRepository.GetInstructions(lastId, MaxInstructionsToRetrieve)) + { + // If this flag gets set it means we're shutting down! In this case, we need to exit asap and cannot + // continue processing anything otherwise we'll hold up the app domain shutdown. + if (released) + { + break; + } + + if (instruction.OriginIdentity == localIdentity) + { + // Just skip that local one but update lastId nevertheless. + lastId = instruction.Id; + continue; + } + + // Deserialize remote instructions & skip if it fails. + if (!TryDeserializeInstructions(instruction, out JArray jsonInstructions)) + { + lastId = instruction.Id; // skip + continue; + } + + List instructionBatch = GetAllInstructions(jsonInstructions); + + // Process as per-normal. + var success = ProcessDatabaseInstructions(instructionBatch, instruction, processed, released, ref lastId); + + // If they couldn't be all processed (i.e. we're shutting down) then exit. + if (success == false) + { + _logger.LogInformation("The current batch of instructions was not processed, app is shutting down"); + break; + } + } + } + + /// + /// Attempts to deserialize the instructions to a JArray. + /// + private bool TryDeserializeInstructions(CacheInstruction instruction, out JArray jsonInstructions) + { + try + { + jsonInstructions = JsonConvert.DeserializeObject(instruction.Instructions); + return true; + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').", + instruction.Id, + instruction.Instructions); + jsonInstructions = null; + return false; + } + } + + /// + /// Parses out the individual instructions to be processed. + /// + private static List GetAllInstructions(IEnumerable jsonInstructions) + { + var result = new List(); + foreach (JToken jsonItem in jsonInstructions) + { + // Could be a JObject in which case we can convert to a RefreshInstruction. + // Otherwise it could be another JArray - in which case we'll iterate that. + if (jsonItem is JObject jsonObj) + { + RefreshInstruction instruction = jsonObj.ToObject(); + result.Add(instruction); + } + else + { + var jsonInnerArray = (JArray)jsonItem; + result.AddRange(GetAllInstructions(jsonInnerArray)); // recurse + } + } + + return result; + } + + /// + /// Processes the instruction batch and checks for errors. + /// + /// + /// Tracks which instructions have already been processed to avoid duplicates + /// + /// Returns true if all instructions in the batch were processed, otherwise false if they could not be due to the app being shut down + /// + private bool ProcessDatabaseInstructions(IReadOnlyCollection instructionBatch, CacheInstruction instruction, HashSet processed, bool released, ref int lastId) + { + // Execute remote instructions & update lastId. + try + { + var result = NotifyRefreshers(instructionBatch, processed, released); + if (result) + { + // If all instructions were processed, set the last id. + lastId = instruction.Id; + } + + return result; + } + //catch (ThreadAbortException ex) + //{ + // //This will occur if the instructions processing is taking too long since this is occurring on a request thread. + // // Or possibly if IIS terminates the appdomain. In any case, we should deal with this differently perhaps... + //} + catch (Exception ex) + { + _logger.LogError( + ex, + "DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored", + instruction.Id, + instruction.Instructions); + + // We cannot throw here because this invalid instruction will just keep getting processed over and over and errors + // will be thrown over and over. The only thing we can do is ignore and move on. + lastId = instruction.Id; + return false; + } + } + + /// + /// Executes the instructions against the cache refresher instances. + /// + /// + /// Returns true if all instructions were processed, otherwise false if the processing was interrupted (i.e. by app shutdown). + /// + private bool NotifyRefreshers(IEnumerable instructions, HashSet processed, bool released) + { + foreach (RefreshInstruction instruction in instructions) + { + // Check if the app is shutting down, we need to exit if this happens. + if (released) + { + return false; + } + + // This has already been processed. + if (processed.Contains(instruction)) + { + continue; + } + + switch (instruction.RefreshType) + { + case RefreshMethodType.RefreshAll: + RefreshAll(instruction.RefresherId); + break; + case RefreshMethodType.RefreshByGuid: + RefreshByGuid(instruction.RefresherId, instruction.GuidId); + break; + case RefreshMethodType.RefreshById: + RefreshById(instruction.RefresherId, instruction.IntId); + break; + case RefreshMethodType.RefreshByIds: + RefreshByIds(instruction.RefresherId, instruction.JsonIds); + break; + case RefreshMethodType.RefreshByJson: + RefreshByJson(instruction.RefresherId, instruction.JsonPayload); + break; + case RefreshMethodType.RemoveById: + RemoveById(instruction.RefresherId, instruction.IntId); + break; + } + + processed.Add(instruction); + } + + return true; + } + + private void RefreshAll(Guid uniqueIdentifier) + { + ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + refresher.RefreshAll(); + } + + private void RefreshByGuid(Guid uniqueIdentifier, Guid id) + { + ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + refresher.Refresh(id); + } + + private void RefreshById(Guid uniqueIdentifier, int id) + { + ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + refresher.Refresh(id); + } + + private void RefreshByIds(Guid uniqueIdentifier, string jsonIds) + { + ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + foreach (var id in JsonConvert.DeserializeObject(jsonIds)) + { + refresher.Refresh(id); + } + } + + private void RefreshByJson(Guid uniqueIdentifier, string jsonPayload) + { + IJsonCacheRefresher refresher = GetJsonRefresher(uniqueIdentifier); + refresher.Refresh(jsonPayload); + } + + private void RemoveById(Guid uniqueIdentifier, int id) + { + ICacheRefresher refresher = GetRefresher(uniqueIdentifier); + refresher.Remove(id); + } + + private ICacheRefresher GetRefresher(Guid id) + { + ICacheRefresher refresher = _cacheRefreshers[id]; + if (refresher == null) + { + throw new InvalidOperationException("Cache refresher with ID \"" + id + "\" does not exist."); + } + + return refresher; + } + + private IJsonCacheRefresher GetJsonRefresher(Guid id) => GetJsonRefresher(GetRefresher(id)); + + private static IJsonCacheRefresher GetJsonRefresher(ICacheRefresher refresher) + { + if (refresher is not IJsonCacheRefresher jsonRefresher) + { + throw new InvalidOperationException("Cache refresher with ID \"" + refresher.RefresherUniqueId + "\" does not implement " + typeof(IJsonCacheRefresher) + "."); + } + + return jsonRefresher; + } + + /// + /// Remove old instructions from the database + /// + /// + /// Always leave the last (most recent) record in the db table, this is so that not all instructions are removed which would cause + /// the site to cold boot if there's been no instruction activity for more than TimeToRetainInstructions. + /// See: http://issues.umbraco.org/issue/U4-7643#comment=67-25085 + /// + private void PruneOldInstructions() + { + DateTime pruneDate = DateTime.UtcNow - _globalSettings.DatabaseServerMessenger.TimeToRetainInstructions; + _cacheInstructionRepository.DeleteInstructionsOlderThan(pruneDate); + } + } +} diff --git a/src/Umbraco.Infrastructure/Services/Implement/ConsentService.cs b/src/Umbraco.Infrastructure/Services/Implement/ConsentService.cs index aebef075a5..b00a2579fd 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/ConsentService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/ConsentService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Events; @@ -11,7 +11,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Implements . /// - internal class ConsentService : ScopeRepositoryService, IConsentService + internal class ConsentService : RepositoryService, IConsentService { private readonly IConsentRepository _consentRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/ContentTypeServiceBase.cs b/src/Umbraco.Infrastructure/Services/Implement/ContentTypeServiceBase.cs index 3886050432..d850e0c21a 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/ContentTypeServiceBase.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/ContentTypeServiceBase.cs @@ -1,10 +1,10 @@ -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using Umbraco.Cms.Core.Events; using Umbraco.Cms.Core.Scoping; namespace Umbraco.Cms.Core.Services.Implement { - public abstract class ContentTypeServiceBase : ScopeRepositoryService + public abstract class ContentTypeServiceBase : RepositoryService { protected ContentTypeServiceBase(IScopeProvider provider, ILoggerFactory loggerFactory, IEventMessagesFactory eventMessagesFactory) : base(provider, loggerFactory, eventMessagesFactory) diff --git a/src/Umbraco.Infrastructure/Services/Implement/DataTypeService.cs b/src/Umbraco.Infrastructure/Services/Implement/DataTypeService.cs index dacaa7e228..2284483c2d 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/DataTypeService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/DataTypeService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.Logging; @@ -19,7 +19,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Represents the DataType Service, which is an easy access to operations involving /// - public class DataTypeService : ScopeRepositoryService, IDataTypeService + public class DataTypeService : RepositoryService, IDataTypeService { private readonly IDataTypeRepository _dataTypeRepository; private readonly IDataTypeContainerRepository _dataTypeContainerRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/DomainService.cs b/src/Umbraco.Infrastructure/Services/Implement/DomainService.cs index ce0d2ee0ed..2b7d964a13 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/DomainService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/DomainService.cs @@ -7,7 +7,7 @@ using Umbraco.Cms.Core.Scoping; namespace Umbraco.Cms.Core.Services.Implement { - public class DomainService : ScopeRepositoryService, IDomainService + public class DomainService : RepositoryService, IDomainService { private readonly IDomainRepository _domainRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/EntityService.cs b/src/Umbraco.Infrastructure/Services/Implement/EntityService.cs index b7ff4f4d5f..0cbcc8d729 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/EntityService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/EntityService.cs @@ -15,7 +15,7 @@ using Umbraco.Extensions; namespace Umbraco.Cms.Core.Services.Implement { - public class EntityService : ScopeRepositoryService, IEntityService + public class EntityService : RepositoryService, IEntityService { private readonly IEntityRepository _entityRepository; private readonly Dictionary _objectTypes; diff --git a/src/Umbraco.Infrastructure/Services/Implement/ExternalLoginService.cs b/src/Umbraco.Infrastructure/Services/Implement/ExternalLoginService.cs index bbec75338b..662259a093 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/ExternalLoginService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/ExternalLoginService.cs @@ -8,7 +8,7 @@ using Umbraco.Cms.Core.Scoping; namespace Umbraco.Cms.Core.Services.Implement { - public class ExternalLoginService : ScopeRepositoryService, IExternalLoginService + public class ExternalLoginService : RepositoryService, IExternalLoginService { private readonly IExternalLoginRepository _externalLoginRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/FileService.cs b/src/Umbraco.Infrastructure/Services/Implement/FileService.cs index 364454f876..67e0607292 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/FileService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/FileService.cs @@ -19,7 +19,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Represents the File Service, which is an easy access to operations involving objects like Scripts, Stylesheets and Templates /// - public class FileService : ScopeRepositoryService, IFileService + public class FileService : RepositoryService, IFileService { private readonly IStylesheetRepository _stylesheetRepository; private readonly IScriptRepository _scriptRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/LocalizationService.cs b/src/Umbraco.Infrastructure/Services/Implement/LocalizationService.cs index a88c883eef..abdda2e68c 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/LocalizationService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/LocalizationService.cs @@ -13,7 +13,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Represents the Localization Service, which is an easy access to operations involving and /// - public class LocalizationService : ScopeRepositoryService, ILocalizationService + public class LocalizationService : RepositoryService, ILocalizationService { private readonly IDictionaryRepository _dictionaryRepository; private readonly ILanguageRepository _languageRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/MacroService.cs b/src/Umbraco.Infrastructure/Services/Implement/MacroService.cs index 92308e671f..c42d29b3c0 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/MacroService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/MacroService.cs @@ -12,7 +12,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Represents the Macro Service, which is an easy access to operations involving /// - public class MacroService : ScopeRepositoryService, IMacroService + public class MacroService : RepositoryService, IMacroService { private readonly IMacroRepository _macroRepository; private readonly IAuditRepository _auditRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/MediaService.cs b/src/Umbraco.Infrastructure/Services/Implement/MediaService.cs index 60061ed9bf..04e4008da2 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/MediaService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/MediaService.cs @@ -20,7 +20,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Represents the Media Service, which is an easy access to operations involving /// - public class MediaService : ScopeRepositoryService, IMediaService + public class MediaService : RepositoryService, IMediaService { private readonly IMediaRepository _mediaRepository; private readonly IMediaTypeRepository _mediaTypeRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/MemberService.cs b/src/Umbraco.Infrastructure/Services/Implement/MemberService.cs index 96ba494790..38b70af19c 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/MemberService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/MemberService.cs @@ -16,7 +16,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Represents the MemberService. /// - public class MemberService : ScopeRepositoryService, IMemberService + public class MemberService : RepositoryService, IMemberService { private readonly IMemberRepository _memberRepository; private readonly IMemberTypeRepository _memberTypeRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/PublicAccessService.cs b/src/Umbraco.Infrastructure/Services/Implement/PublicAccessService.cs index 4c8615f442..72e7873a7c 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/PublicAccessService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/PublicAccessService.cs @@ -10,7 +10,7 @@ using Umbraco.Extensions; namespace Umbraco.Cms.Core.Services.Implement { - public class PublicAccessService : ScopeRepositoryService, IPublicAccessService + public class PublicAccessService : RepositoryService, IPublicAccessService { private readonly IPublicAccessRepository _publicAccessRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/RedirectUrlService.cs b/src/Umbraco.Infrastructure/Services/Implement/RedirectUrlService.cs index d2ea10df49..a3faf64081 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/RedirectUrlService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/RedirectUrlService.cs @@ -8,7 +8,7 @@ using Umbraco.Cms.Core.Scoping; namespace Umbraco.Cms.Core.Services.Implement { - internal class RedirectUrlService : ScopeRepositoryService, IRedirectUrlService + internal class RedirectUrlService : RepositoryService, IRedirectUrlService { private readonly IRedirectUrlRepository _redirectUrlRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/RelationService.cs b/src/Umbraco.Infrastructure/Services/Implement/RelationService.cs index b83d3f286c..19fb68ae8c 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/RelationService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/RelationService.cs @@ -11,7 +11,7 @@ using Umbraco.Extensions; namespace Umbraco.Cms.Core.Services.Implement { - public class RelationService : ScopeRepositoryService, IRelationService + public class RelationService : RepositoryService, IRelationService { private readonly IEntityService _entityService; private readonly IRelationRepository _relationRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/ScopeRepositoryService.cs b/src/Umbraco.Infrastructure/Services/Implement/ScopeRepositoryService.cs deleted file mode 100644 index ca8e074b04..0000000000 --- a/src/Umbraco.Infrastructure/Services/Implement/ScopeRepositoryService.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Microsoft.Extensions.Logging; -using Umbraco.Cms.Core.Events; -using Umbraco.Cms.Core.Scoping; - -namespace Umbraco.Cms.Core.Services.Implement -{ - // TODO: that one does not add anything = kill - public abstract class ScopeRepositoryService : RepositoryService - { - protected ScopeRepositoryService(IScopeProvider provider, ILoggerFactory loggerFactory, IEventMessagesFactory eventMessagesFactory) - : base(provider, loggerFactory, eventMessagesFactory) - { } - } -} diff --git a/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs b/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs index 75466c2013..9c03f9aabc 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/ServerRegistrationService.cs @@ -16,7 +16,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Manages server registrations in the database. /// - public sealed class ServerRegistrationService : ScopeRepositoryService, IServerRegistrationService + public sealed class ServerRegistrationService : RepositoryService, IServerRegistrationService { private readonly IServerRegistrationRepository _serverRegistrationRepository; private readonly IHostingEnvironment _hostingEnvironment; diff --git a/src/Umbraco.Infrastructure/Services/Implement/TagService.cs b/src/Umbraco.Infrastructure/Services/Implement/TagService.cs index 2d2cf082c6..907af05ab2 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/TagService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/TagService.cs @@ -14,7 +14,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// If there is unpublished content with tags, those tags will not be contained /// - public class TagService : ScopeRepositoryService, ITagService + public class TagService : RepositoryService, ITagService { private readonly ITagRepository _tagRepository; diff --git a/src/Umbraco.Infrastructure/Services/Implement/UserService.cs b/src/Umbraco.Infrastructure/Services/Implement/UserService.cs index 751581e068..d35bcbfa50 100644 --- a/src/Umbraco.Infrastructure/Services/Implement/UserService.cs +++ b/src/Umbraco.Infrastructure/Services/Implement/UserService.cs @@ -21,7 +21,7 @@ namespace Umbraco.Cms.Core.Services.Implement /// /// Represents the UserService, which is an easy access to operations involving , and eventually Backoffice Users. /// - public class UserService : ScopeRepositoryService, IUserService + public class UserService : RepositoryService, IUserService { private readonly IUserRepository _userRepository; private readonly IUserGroupRepository _userGroupRepository; diff --git a/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs index 96bdeea82d..940ebfe0cd 100644 --- a/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs @@ -3,17 +3,14 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Newtonsoft.Json; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Runtime; -using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Serialization; +using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Sync; using Umbraco.Cms.Core.Web; -using Umbraco.Cms.Infrastructure.Persistence.Dtos; -using Umbraco.Extensions; namespace Umbraco.Cms.Infrastructure.Sync { @@ -30,17 +27,15 @@ namespace Umbraco.Cms.Infrastructure.Sync /// public BatchedDatabaseServerMessenger( IMainDom mainDom, - IScopeProvider scopeProvider, - IProfilingLogger proflog, ILogger logger, - IServerRoleAccessor serverRegistrar, DatabaseServerMessengerCallbacks callbacks, IHostingEnvironment hostingEnvironment, - CacheRefresherCollection cacheRefreshers, + ICacheInstructionService cacheInstructionService, + IJsonSerializer jsonSerializer, IRequestCache requestCache, IRequestAccessor requestAccessor, IOptions globalSettings) - : base(mainDom, scopeProvider, proflog, logger, serverRegistrar, true, callbacks, hostingEnvironment, cacheRefreshers, globalSettings) + : base(mainDom, logger, true, callbacks, hostingEnvironment, cacheInstructionService, jsonSerializer, globalSettings) { _requestCache = requestCache; _requestAccessor = requestAccessor; @@ -71,28 +66,7 @@ namespace Umbraco.Cms.Infrastructure.Sync RefreshInstruction[] instructions = batch.SelectMany(x => x.Instructions).ToArray(); batch.Clear(); - // Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount - using (IScope scope = ScopeProvider.CreateScope()) - { - foreach (IEnumerable instructionsBatch in instructions.InGroupsOf(GlobalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount)) - { - WriteInstructions(scope, instructionsBatch); - } - - scope.Complete(); - } - } - - private void WriteInstructions(IScope scope, IEnumerable instructions) - { - var dto = new CacheInstructionDto - { - UtcStamp = DateTime.UtcNow, - Instructions = JsonConvert.SerializeObject(instructions, Formatting.None), - OriginIdentity = LocalIdentity, - InstructionCount = instructions.Sum(x => x.JsonIdCount) - }; - scope.Database.Insert(dto); + CacheInstructionService.DeliverInstructionsInBatches(instructions, LocalIdentity); } private ICollection GetBatch(bool create) @@ -104,7 +78,7 @@ namespace Umbraco.Cms.Infrastructure.Sync return null; } - // no thread-safety here because it'll run in only 1 thread (request) at a time + // No thread-safety here because it'll run in only 1 thread (request) at a time. var batch = (ICollection)_requestCache.Get(key); if (batch == null && create) { @@ -123,27 +97,17 @@ namespace Umbraco.Cms.Infrastructure.Sync string json = null) { ICollection batch = GetBatch(true); - IEnumerable instructions = RefreshInstruction.GetInstructions(refresher, messageType, ids, idType, json); + IEnumerable instructions = RefreshInstruction.GetInstructions(refresher, JsonSerializer, messageType, ids, idType, json); - // batch if we can, else write to DB immediately + // Batch if we can, else write to DB immediately. if (batch == null) { - // only write the json blob with a maximum count of the MaxProcessingInstructionCount - using (IScope scope = ScopeProvider.CreateScope()) - { - foreach (IEnumerable maxBatch in instructions.InGroupsOf(GlobalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount)) - { - WriteInstructions(scope, maxBatch); - } - - scope.Complete(); - } + CacheInstructionService.DeliverInstructionsInBatches(instructions, LocalIdentity); } else { batch.Add(new RefreshInstructionEnvelope(refresher, instructions)); } - } } } diff --git a/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs index 92bd732246..7d1ef9f360 100644 --- a/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs @@ -7,19 +7,14 @@ using System.Linq; using System.Threading; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using NPoco; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Hosting; -using Umbraco.Cms.Core.Logging; using Umbraco.Cms.Core.Runtime; -using Umbraco.Cms.Core.Scoping; +using Umbraco.Cms.Core.Serialization; +using Umbraco.Cms.Core.Services; using Umbraco.Cms.Core.Sync; -using Umbraco.Cms.Infrastructure.Persistence; -using Umbraco.Cms.Infrastructure.Persistence.Dtos; using Umbraco.Extensions; namespace Umbraco.Cms.Infrastructure.Sync @@ -29,8 +24,6 @@ namespace Umbraco.Cms.Infrastructure.Sync /// public abstract class DatabaseServerMessenger : ServerMessengerBase { - // TODO: This class needs to be split into a service/repo for DB access - /* * this messenger writes ALL instructions to the database, * but only processes instructions coming from remote servers, @@ -40,10 +33,7 @@ namespace Umbraco.Cms.Infrastructure.Sync private readonly IMainDom _mainDom; private readonly ManualResetEvent _syncIdle; private readonly object _locko = new object(); - private readonly IProfilingLogger _profilingLogger; - private readonly IServerRoleAccessor _serverRegistrar; private readonly IHostingEnvironment _hostingEnvironment; - private readonly CacheRefresherCollection _cacheRefreshers; private readonly Lazy _distCacheFilePath; private int _lastId = -1; @@ -58,33 +48,29 @@ namespace Umbraco.Cms.Infrastructure.Sync /// protected DatabaseServerMessenger( IMainDom mainDom, - IScopeProvider scopeProvider, - IProfilingLogger proflog, ILogger logger, - IServerRoleAccessor serverRegistrar, bool distributedEnabled, DatabaseServerMessengerCallbacks callbacks, IHostingEnvironment hostingEnvironment, - CacheRefresherCollection cacheRefreshers, + ICacheInstructionService cacheInstructionService, + IJsonSerializer jsonSerializer, IOptions globalSettings) : base(distributedEnabled) { - ScopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider)); _mainDom = mainDom; - _profilingLogger = proflog ?? throw new ArgumentNullException(nameof(proflog)); - _serverRegistrar = serverRegistrar; _hostingEnvironment = hostingEnvironment; - _cacheRefreshers = cacheRefreshers; Logger = logger; Callbacks = callbacks ?? throw new ArgumentNullException(nameof(callbacks)); + CacheInstructionService = cacheInstructionService; + JsonSerializer = jsonSerializer; GlobalSettings = globalSettings.Value; _lastPruned = _lastSync = DateTime.UtcNow; _syncIdle = new ManualResetEvent(true); _distCacheFilePath = new Lazy(() => GetDistCacheFilePath(hostingEnvironment)); - // See notes on LocalIdentity + // See notes on _localIdentity LocalIdentity = NetworkHelper.MachineName // eg DOMAIN\SERVER - + "/" + _hostingEnvironment.ApplicationId // eg /LM/S3SVC/11/ROOT + + "/" + hostingEnvironment.ApplicationId // eg /LM/S3SVC/11/ROOT + " [P" + Process.GetCurrentProcess().Id // eg 1234 + "/D" + AppDomain.CurrentDomain.Id // eg 22 + "] " + Guid.NewGuid().ToString("N").ToUpper(); // make it truly unique @@ -92,21 +78,33 @@ namespace Umbraco.Cms.Infrastructure.Sync _initialized = new Lazy(EnsureInitialized); } + private string DistCacheFilePath => _distCacheFilePath.Value; + public DatabaseServerMessengerCallbacks Callbacks { get; } public GlobalSettings GlobalSettings { get; } protected ILogger Logger { get; } - protected IScopeProvider ScopeProvider { get; } + protected ICacheInstructionService CacheInstructionService { get; } - protected Sql Sql() => ScopeProvider.SqlContext.Sql(); + protected IJsonSerializer JsonSerializer { get; } - private string DistCacheFilePath => _distCacheFilePath.Value; + /// + /// Gets the unique local identity of the executing AppDomain. + /// + /// + /// It is not only about the "server" (machine name and appDomainappId), but also about + /// an AppDomain, within a Process, on that server - because two AppDomains running at the same + /// time on the same server (eg during a restart) are, practically, a LB setup. + /// Practically, all we really need is the guid, the other infos are here for information + /// and debugging purposes. + /// + protected string LocalIdentity { get; } #region Messenger - // we don't care if there's servers listed or not, + // we don't care if there are servers listed or not, // if distributed call is enabled we will make the call protected override bool RequiresDistributed(ICacheRefresher refresher, MessageType dispatchType) => _initialized.Value && DistributedEnabled; @@ -119,26 +117,14 @@ namespace Umbraco.Cms.Infrastructure.Sync { var idsA = ids?.ToArray(); - if (GetArrayType(idsA, out var idType) == false) + if (GetArrayType(idsA, out Type idType) == false) { throw new ArgumentException("All items must be of the same type, either int or Guid.", nameof(ids)); } - var instructions = RefreshInstruction.GetInstructions(refresher, messageType, idsA, idType, json); + IEnumerable instructions = RefreshInstruction.GetInstructions(refresher, JsonSerializer, messageType, idsA, idType, json); - var dto = new CacheInstructionDto - { - UtcStamp = DateTime.UtcNow, - Instructions = JsonConvert.SerializeObject(instructions, Formatting.None), - OriginIdentity = LocalIdentity, - InstructionCount = instructions.Sum(x => x.JsonIdCount) - }; - - using (var scope = ScopeProvider.CreateScope()) - { - scope.Database.Insert(dto); - scope.Complete(); - } + CacheInstructionService.DeliverInstructions(instructions, LocalIdentity); } #endregion @@ -151,7 +137,7 @@ namespace Umbraco.Cms.Infrastructure.Sync private bool EnsureInitialized() { // weight:10, must release *before* the published snapshot service, because once released - // the service will *not* be able to properly handle our notifications anymore + // the service will *not* be able to properly handle our notifications anymore. const int weight = 10; var registered = _mainDom.Register( @@ -162,11 +148,11 @@ namespace Umbraco.Cms.Infrastructure.Sync _released = true; // no more syncs } - // wait a max of 5 seconds and then return, so that we don't block + // Wait a max of 5 seconds and then return, so that we don't block // the entire MainDom callbacks chain and prevent the AppDomain from // properly releasing MainDom - a timeout here means that one refresher // is taking too much time processing, however when it's done we will - // not update lastId and stop everything + // not update lastId and stop everything. var idle = _syncIdle.WaitOne(5000); if (idle == false) { @@ -182,86 +168,28 @@ namespace Umbraco.Cms.Infrastructure.Sync ReadLastSynced(); // get _lastId - using (var scope = ScopeProvider.CreateScope(autoComplete: true)) + CacheInstructionServiceInitializationResult result = CacheInstructionService.EnsureInitialized(_released, _lastId); + + if (result.ColdBootRequired) { - EnsureInstructions(scope.Database); // reset _lastId if instructions are missing - return Initialize(scope.Database); // boot + // If there is a max currently, or if we've never synced. + if (result.MaxId > 0 || result.LastId < 0) + { + SaveLastSynced(result.MaxId); + } + + // Execute initializing callbacks. + if (Callbacks.InitializingCallbacks != null) + { + foreach (Action callback in Callbacks.InitializingCallbacks) + { + callback(); + } + } } - } - /// - /// Initializes a server that has never synchronized before. - /// - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// Callers MUST ensure thread-safety. - /// - private bool Initialize(IUmbracoDatabase database) - { - lock (_locko) - { - if (_released) - { - return false; - } - - var coldboot = false; - - // never synced before - if (_lastId < 0) - { - // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new - // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. - Logger.LogWarning("No last synced Id found, this generally means this is a new server/install." - + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" - + " the database and maintain cache updates based on that Id."); - - coldboot = true; - } - else - { - // check for how many instructions there are to process, each row contains a count of the number of instructions contained in each - // row so we will sum these numbers to get the actual count. - var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new { lastId = _lastId }); - if (count > GlobalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount) - { - // too many instructions, proceed to cold boot - Logger.LogWarning( - "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." - + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" - + " to the latest found in the database and maintain cache updates based on that Id.", - count, GlobalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount); - - coldboot = true; - } - } - - if (coldboot) - { - // go get the last id in the db and store it - // note: do it BEFORE initializing otherwise some instructions might get lost - // when doing it before, some instructions might run twice - not an issue - var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); - - // if there is a max currently, or if we've never synced - if (maxId > 0 || _lastId < 0) - { - SaveLastSynced(maxId); - } - - // execute initializing callbacks - if (Callbacks.InitializingCallbacks != null) - { - foreach (var callback in Callbacks.InitializingCallbacks) - { - callback(); - } - } - } - - return true; - } - } + return result.Initialized; + } /// /// Synchronize the server (throttled). @@ -299,29 +227,15 @@ namespace Umbraco.Cms.Infrastructure.Sync try { - using (_profilingLogger.DebugDuration("Syncing from database...")) - using (var scope = ScopeProvider.CreateScope()) + CacheInstructionServiceProcessInstructionsResult result = CacheInstructionService.ProcessInstructions(_released, LocalIdentity, _lastPruned); + if (result.InstructionsWerePruned) { - ProcessDatabaseInstructions(scope.Database); - - // Check for pruning throttling - if (_released || (DateTime.UtcNow - _lastPruned) <= GlobalSettings.DatabaseServerMessenger.TimeBetweenPruneOperations) - { - scope.Complete(); - return; - } - _lastPruned = _lastSync; + } - switch (_serverRegistrar.CurrentServerRole) - { - case ServerRole.Single: - case ServerRole.Master: - PruneOldInstructions(scope.Database); - break; - } - - scope.Complete(); + if (result.LastId > 0) + { + SaveLastSynced(result.LastId); } } finally @@ -334,206 +248,8 @@ namespace Umbraco.Cms.Infrastructure.Sync _syncIdle.Set(); } - } - - /// - /// Process instructions from the database. - /// - /// - /// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded. - /// - private void ProcessDatabaseInstructions(IUmbracoDatabase database) - { - // NOTE - // we 'could' recurse to ensure that no remaining instructions are pending in the table before proceeding but I don't think that - // would be a good idea since instructions could keep getting added and then all other threads will probably get stuck from serving requests - // (depending on what the cache refreshers are doing). I think it's best we do the one time check, process them and continue, if there are - // pending requests after being processed, they'll just be processed on the next poll. - // - // TODO: not true if we're running on a background thread, assuming we can? - - var sql = Sql().SelectAll() - .From() - .Where(dto => dto.Id > _lastId) - .OrderBy(dto => dto.Id); - - // only retrieve the top 100 (just in case there's tons) - // even though MaxProcessingInstructionCount is by default 1000 we still don't want to process that many - // rows in one request thread since each row can contain a ton of instructions (until 7.5.5 in which case - // a row can only contain MaxProcessingInstructionCount) - var topSql = sql.SelectTop(100); - - // only process instructions coming from a remote server, and ignore instructions coming from - // the local server as they've already been processed. We should NOT assume that the sequence of - // instructions in the database makes any sense whatsoever, because it's all async. - var localIdentity = LocalIdentity; - - var lastId = 0; - - // tracks which ones have already been processed to avoid duplicates - var processed = new HashSet(); - - // It would have been nice to do this in a Query instead of Fetch using a data reader to save - // some memory however we cannot do that because inside of this loop the cache refreshers are also - // performing some lookups which cannot be done with an active reader open - foreach (var dto in database.Fetch(topSql)) - { - // If this flag gets set it means we're shutting down! In this case, we need to exit asap and cannot - // continue processing anything otherwise we'll hold up the app domain shutdown - if (_released) - { - break; - } - - if (dto.OriginIdentity == localIdentity) - { - // just skip that local one but update lastId nevertheless - lastId = dto.Id; - continue; - } - - // deserialize remote instructions & skip if it fails - JArray jsonA; - try - { - jsonA = JsonConvert.DeserializeObject(dto.Instructions); - } - catch (JsonException ex) - { - Logger.LogError(ex, "Failed to deserialize instructions ({DtoId}: '{DtoInstructions}').", - dto.Id, - dto.Instructions); - - lastId = dto.Id; // skip - continue; - } - - var instructionBatch = GetAllInstructions(jsonA); - - // process as per-normal - var success = ProcessDatabaseInstructions(instructionBatch, dto, processed, ref lastId); - - // if they couldn't be all processed (i.e. we're shutting down) then exit - if (success == false) - { - Logger.LogInformation("The current batch of instructions was not processed, app is shutting down"); - break; - } - - } - - if (lastId > 0) - SaveLastSynced(lastId); - } - - /// - /// Processes the instruction batch and checks for errors - /// - /// - /// - /// - /// Tracks which instructions have already been processed to avoid duplicates - /// - /// - /// - /// returns true if all instructions in the batch were processed, otherwise false if they could not be due to the app being shut down - /// - private bool ProcessDatabaseInstructions(IReadOnlyCollection instructionBatch, CacheInstructionDto dto, HashSet processed, ref int lastId) - { - // execute remote instructions & update lastId - try - { - var result = NotifyRefreshers(instructionBatch, processed); - if (result) - { - //if all instructions we're processed, set the last id - lastId = dto.Id; - } - return result; - } - //catch (ThreadAbortException ex) - //{ - // //This will occur if the instructions processing is taking too long since this is occurring on a request thread. - // // Or possibly if IIS terminates the appdomain. In any case, we should deal with this differently perhaps... - //} - catch (Exception ex) - { - Logger.LogError( - ex, - "DISTRIBUTED CACHE IS NOT UPDATED. Failed to execute instructions ({DtoId}: '{DtoInstructions}'). Instruction is being skipped/ignored", - dto.Id, - dto.Instructions); - - //we cannot throw here because this invalid instruction will just keep getting processed over and over and errors - // will be thrown over and over. The only thing we can do is ignore and move on. - lastId = dto.Id; - return false; - } - - ////if this is returned it will not be saved - //return -1; - } - - /// - /// Remove old instructions from the database - /// - /// - /// Always leave the last (most recent) record in the db table, this is so that not all instructions are removed which would cause - /// the site to cold boot if there's been no instruction activity for more than DaysToRetainInstructions. - /// See: http://issues.umbraco.org/issue/U4-7643#comment=67-25085 - /// - private void PruneOldInstructions(IUmbracoDatabase database) - { - var pruneDate = DateTime.UtcNow - GlobalSettings.DatabaseServerMessenger.TimeToRetainInstructions; - - // using 2 queries is faster than convoluted joins - - var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction;"); - - var delete = new Sql().Append(@"DELETE FROM umbracoCacheInstruction WHERE utcStamp < @pruneDate AND id < @maxId", - new { pruneDate, maxId }); - - database.Execute(delete); - } - - /// - /// Ensure that the last instruction that was processed is still in the database. - /// - /// - /// If the last instruction is not in the database anymore, then the messenger - /// should not try to process any instructions, because some instructions might be lost, - /// and it should instead cold-boot. - /// However, if the last synced instruction id is '0' and there are '0' records, then this indicates - /// that it's a fresh site and no user actions have taken place, in this circumstance we do not want to cold - /// boot. See: http://issues.umbraco.org/issue/U4-8627 - /// - private void EnsureInstructions(IUmbracoDatabase database) - { - if (_lastId == 0) - { - var sql = Sql().Select("COUNT(*)") - .From(); - - var count = database.ExecuteScalar(sql); - - //if there are instructions but we haven't synced, then a cold boot is necessary - if (count > 0) - _lastId = -1; - } - else - { - var sql = Sql().SelectAll() - .From() - .Where(dto => dto.Id == _lastId); - - var dtos = database.Fetch(sql); - - //if the last synced instruction is not found in the db, then a cold boot is necessary - if (dtos.Count == 0) - _lastId = -1; - } - } - + } + /// /// Reads the last-synced id from file into memory. /// @@ -543,11 +259,15 @@ namespace Umbraco.Cms.Infrastructure.Sync private void ReadLastSynced() { if (File.Exists(DistCacheFilePath) == false) + { return; + } var content = File.ReadAllText(DistCacheFilePath); if (int.TryParse(content, out var last)) + { _lastId = last; + } } /// @@ -563,18 +283,6 @@ namespace Umbraco.Cms.Infrastructure.Sync _lastId = id; } - /// - /// Gets the unique local identity of the executing AppDomain. - /// - /// - /// It is not only about the "server" (machine name and appDomainappId), but also about - /// an AppDomain, within a Process, on that server - because two AppDomains running at the same - /// time on the same server (eg during a restart) are, practically, a LB setup. - /// Practically, all we really need is the guid, the other infos are here for information - /// and debugging purposes. - /// - protected string LocalIdentity { get; } - private string GetDistCacheFilePath(IHostingEnvironment hostingEnvironment) { var fileName = _hostingEnvironment.ApplicationId.ReplaceNonAlphanumericChars(string.Empty) + "-lastsynced.txt"; @@ -584,151 +292,18 @@ namespace Umbraco.Cms.Infrastructure.Sync //ensure the folder exists var folder = Path.GetDirectoryName(distCacheFilePath); if (folder == null) + { throw new InvalidOperationException("The folder could not be determined for the file " + distCacheFilePath); + } + if (Directory.Exists(folder) == false) + { Directory.CreateDirectory(folder); + } return distCacheFilePath; } #endregion - - #region Notify refreshers - - private ICacheRefresher GetRefresher(Guid id) - { - var refresher = _cacheRefreshers[id]; - if (refresher == null) - throw new InvalidOperationException("Cache refresher with ID \"" + id + "\" does not exist."); - return refresher; - } - - private IJsonCacheRefresher GetJsonRefresher(Guid id) - { - return GetJsonRefresher(GetRefresher(id)); - } - - private static IJsonCacheRefresher GetJsonRefresher(ICacheRefresher refresher) - { - var jsonRefresher = refresher as IJsonCacheRefresher; - if (jsonRefresher == null) - throw new InvalidOperationException("Cache refresher with ID \"" + refresher.RefresherUniqueId + "\" does not implement " + typeof(IJsonCacheRefresher) + "."); - return jsonRefresher; - } - - /// - /// Parses out the individual instructions to be processed - /// - /// - /// - private static List GetAllInstructions(IEnumerable jsonArray) - { - var result = new List(); - foreach (var jsonItem in jsonArray) - { - // could be a JObject in which case we can convert to a RefreshInstruction, - // otherwise it could be another JArray - in which case we'll iterate that. - var jsonObj = jsonItem as JObject; - if (jsonObj != null) - { - var instruction = jsonObj.ToObject(); - result.Add(instruction); - } - else - { - var jsonInnerArray = (JArray)jsonItem; - result.AddRange(GetAllInstructions(jsonInnerArray)); // recurse - } - } - return result; - } - - /// - /// executes the instructions against the cache refresher instances - /// - /// - /// - /// - /// Returns true if all instructions were processed, otherwise false if the processing was interrupted (i.e. app shutdown) - /// - private bool NotifyRefreshers(IEnumerable instructions, HashSet processed) - { - foreach (var instruction in instructions) - { - //Check if the app is shutting down, we need to exit if this happens. - if (_released) - { - return false; - } - - //this has already been processed - if (processed.Contains(instruction)) - continue; - - switch (instruction.RefreshType) - { - case RefreshMethodType.RefreshAll: - RefreshAll(instruction.RefresherId); - break; - case RefreshMethodType.RefreshByGuid: - RefreshByGuid(instruction.RefresherId, instruction.GuidId); - break; - case RefreshMethodType.RefreshById: - RefreshById(instruction.RefresherId, instruction.IntId); - break; - case RefreshMethodType.RefreshByIds: - RefreshByIds(instruction.RefresherId, instruction.JsonIds); - break; - case RefreshMethodType.RefreshByJson: - RefreshByJson(instruction.RefresherId, instruction.JsonPayload); - break; - case RefreshMethodType.RemoveById: - RemoveById(instruction.RefresherId, instruction.IntId); - break; - } - - processed.Add(instruction); - } - return true; - } - - private void RefreshAll(Guid uniqueIdentifier) - { - var refresher = GetRefresher(uniqueIdentifier); - refresher.RefreshAll(); - } - - private void RefreshByGuid(Guid uniqueIdentifier, Guid id) - { - var refresher = GetRefresher(uniqueIdentifier); - refresher.Refresh(id); - } - - private void RefreshById(Guid uniqueIdentifier, int id) - { - var refresher = GetRefresher(uniqueIdentifier); - refresher.Refresh(id); - } - - private void RefreshByIds(Guid uniqueIdentifier, string jsonIds) - { - var refresher = GetRefresher(uniqueIdentifier); - foreach (var id in JsonConvert.DeserializeObject(jsonIds)) - refresher.Refresh(id); - } - - private void RefreshByJson(Guid uniqueIdentifier, string jsonPayload) - { - var refresher = GetJsonRefresher(uniqueIdentifier); - refresher.Refresh(jsonPayload); - } - - private void RemoveById(Guid uniqueIdentifier, int id) - { - var refresher = GetRefresher(uniqueIdentifier); - refresher.Remove(id); - } - - #endregion } }