diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs index 49b0d23862..ebc77dbdca 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs @@ -28,7 +28,7 @@ namespace Umbraco.Core.Sync // but only processes instructions coming from remote servers, // thus ensuring that instructions run only once // - public class DatabaseServerMessenger : ServerMessengerBase + public class DatabaseServerMessenger : ServerMessengerBase, ISyncBootStateAccessor { private readonly IRuntimeState _runtime; private readonly ManualResetEvent _syncIdle; @@ -172,35 +172,7 @@ namespace Umbraco.Core.Sync lock (_locko) { if (_released) return; - - var coldboot = false; - if (_lastId < 0) // never synced before - { - // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new - // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. - Logger.Warn("No last synced Id found, this generally means this is a new server/install." - + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" - + " the database and maintain cache updates based on that Id."); - - coldboot = true; - } - else - { - //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each - //row so we will sum these numbers to get the actual count. - var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); - if (count > Options.MaxProcessingInstructionCount) - { - //too many instructions, proceed to cold boot - Logger.Warn( - "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." - + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" - + " to the latest found in the database and maintain cache updates based on that Id.", - count, Options.MaxProcessingInstructionCount); - - coldboot = true; - } - } + var coldboot = IsColdBoot(database); if (coldboot) { @@ -223,6 +195,40 @@ namespace Umbraco.Core.Sync } } + private bool IsColdBoot(IUmbracoDatabase database) + { + var coldboot = false; + if (_lastId < 0) // never synced before + { + // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new + // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. + Logger.Warn("No last synced Id found, this generally means this is a new server/install." + + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" + + " the database and maintain cache updates based on that Id."); + + coldboot = true; + } + else + { + //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each + //row so we will sum these numbers to get the actual count. + var count = database.ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); + if (count > Options.MaxProcessingInstructionCount) + { + //too many instructions, proceed to cold boot + Logger.Warn( + "The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})." + + " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id" + + " to the latest found in the database and maintain cache updates based on that Id.", + count, Options.MaxProcessingInstructionCount); + + coldboot = true; + } + } + + return coldboot; + } + /// /// Synchronize the server (throttled). /// @@ -548,6 +554,30 @@ namespace Umbraco.Core.Sync #endregion + public SyncBootState GetSyncBootState() + { + try + { + ReadLastSynced(); // get _lastId + using (var scope = ScopeProvider.CreateScope()) + { + EnsureInstructions(scope.Database); + bool isColdBoot = IsColdBoot(scope.Database); + + if (isColdBoot) + { + return SyncBootState.ColdBoot; + } + return SyncBootState.HasSyncState; + } + } + catch(Exception ex) + { + Logger.Warn("Error determining Sync Boot State", ex); + return SyncBootState.Unknown; + } + } + #region Notify refreshers private static ICacheRefresher GetRefresher(Guid id) diff --git a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs new file mode 100644 index 0000000000..4b8500f2d9 --- /dev/null +++ b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + /// + /// Retrieve the state of the sync service + /// + public interface ISyncBootStateAccessor + { + /// + /// Get the boot state + /// + /// + SyncBootState GetSyncBootState(); + } +} diff --git a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs new file mode 100644 index 0000000000..70cec6cc96 --- /dev/null +++ b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + /// + /// Boot state implementation for when umbraco is not in the run state + /// + public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor + { + public SyncBootState GetSyncBootState() + { + return SyncBootState.Unknown; + } + } +} diff --git a/src/Umbraco.Core/Sync/SyncBootState.cs b/src/Umbraco.Core/Sync/SyncBootState.cs new file mode 100644 index 0000000000..4abc53abba --- /dev/null +++ b/src/Umbraco.Core/Sync/SyncBootState.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + public enum SyncBootState + { + /// + /// Unknown state. Treat as HasSyncState + /// + Unknown = 0, + /// + /// Cold boot. No Sync state + /// + ColdBoot = 1, + /// + /// Warm boot. Sync state present + /// + HasSyncState = 2 + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 1160881304..6b4725c48c 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -190,6 +190,9 @@ + + + diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index afba2dcc4f..75a20ade6f 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -158,6 +159,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index eee3500495..9feb0d703b 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Scoping; using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -204,6 +205,7 @@ namespace Umbraco.Tests.PublishedContent Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index be10db3a9d..ad372c00b9 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -101,6 +101,7 @@ namespace Umbraco.Tests.Scoping Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index aaad60f7e9..b252738fee 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.PropertyEditors; using Umbraco.Core.Services; using Umbraco.Core.Strings; using Umbraco.Core.Sync; +using Umbraco.Tests.TestHelpers; using Umbraco.Tests.TestHelpers.Entities; using Umbraco.Tests.Testing; using Umbraco.Web.PublishedCache; @@ -73,6 +74,7 @@ namespace Umbraco.Tests.Services Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs new file mode 100644 index 0000000000..e5f6989381 --- /dev/null +++ b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Sync; + +namespace Umbraco.Tests.TestHelpers +{ + class TestSyncBootStateAccessor : ISyncBootStateAccessor + { + private readonly SyncBootState _syncBootState; + + public TestSyncBootStateAccessor(SyncBootState syncBootState) + { + _syncBootState = syncBootState; + } + public SyncBootState GetSyncBootState() + { + return _syncBootState; + } + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 46d2216e82..4920bcda2a 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -184,6 +184,7 @@ + diff --git a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs index 2fa9d80779..26ba0db324 100644 --- a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs @@ -72,6 +72,7 @@ namespace Umbraco.Web.Compose composition.SetDatabaseServerMessengerOptions(GetDefaultOptions); composition.SetServerMessenger(); + composition.Register(factory=> factory.GetInstance() as BatchedDatabaseServerMessenger, Lifetime.Singleton); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index 0b15c0ba4b..1b96538dd0 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -13,11 +13,11 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource { public IDictionary ReadFrom(Stream stream) { - var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); // read properties count var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + var dict = new Dictionary(pcount,StringComparer.InvariantCultureIgnoreCase); // read each property for (var i = 0; i < pcount; i++) { @@ -28,13 +28,13 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); // create pdata and add to the dictionary - var pdatas = new List(); + var pdatas = new PropertyData[vcount]; // for each value, read and add to pdata for (var j = 0; j < vcount; j++) { var pdata = new PropertyData(); - pdatas.Add(pdata); + pdatas[j] =pdata; // everything that can be null is read/written as object // even though - culture and segment should never be null here, as 'null' represents @@ -46,7 +46,7 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource pdata.Value = ReadObject(stream); } - dict[key] = pdatas.ToArray(); + dict[key] = pdatas; } return dict; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index c4d40f721f..21cd0bf763 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,6 +1,8 @@ using Newtonsoft.Json; using System; +using System.Buffers; using System.Collections.Generic; +using System.IO; using Umbraco.Core.Models; using Umbraco.Core.Serialization; @@ -21,13 +23,20 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource DateTimeZoneHandling = DateTimeZoneHandling.Utc, DateFormatString = "o" }; - + private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); - return JsonConvert.DeserializeObject(stringData, _jsonSerializerSettings); + JsonSerializer serializer = JsonSerializer.Create(_jsonSerializerSettings); + using (JsonTextReader reader = new JsonTextReader(new StringReader(stringData))) + { + // reader will get buffer from array pool + reader.ArrayPool = JsonArrayPool.Instance; + reader.PropertyNameTable = _propertyNameTable; + return serializer.Deserialize(reader); + } } public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) @@ -39,4 +48,44 @@ namespace Umbraco.Web.PublishedCache.NuCache.DataSource return new ContentCacheDataSerializationResult(json, null); } } + public class JsonArrayPool : IArrayPool + { + public static readonly JsonArrayPool Instance = new JsonArrayPool(); + + public char[] Rent(int minimumLength) + { + // get char array from System.Buffers shared pool + return ArrayPool.Shared.Rent(minimumLength); + } + + public void Return(char[] array) + { + // return char array to System.Buffers shared pool + ArrayPool.Shared.Return(array); + } + } + public class AutomaticJsonNameTable : DefaultJsonNameTable + { + int nAutoAdded = 0; + int maxToAutoAdd; + + public AutomaticJsonNameTable(int maxToAdd) + { + this.maxToAutoAdd = maxToAdd; + } + + public override string Get(char[] key, int start, int length) + { + var s = base.Get(key, start, length); + + if (s == null && nAutoAdded < maxToAutoAdd) + { + s = new string(key, start, length); + Add(s); + nAutoAdded++; + } + + return s; + } + } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 98d8b91386..6dac3b9afb 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,6 +1,7 @@ using System.Configuration; using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Core.Sync; using Umbraco.Core.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -27,6 +28,9 @@ namespace Umbraco.Web.PublishedCache.NuCache composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + //Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer + composition.Register(Lifetime.Singleton); + // register the NuCache database data source composition.RegisterUnique(); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs old mode 100755 new mode 100644 index 3a055223a5..5b3980ad06 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -21,6 +21,7 @@ using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Services.Implement; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Web.Install; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -62,6 +63,8 @@ namespace Umbraco.Web.PublishedCache.NuCache private bool _localContentDbExists; private bool _localMediaDbExists; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + // define constant - determines whether to use cache when previewing // to store eg routes, property converted values, anything - caching // means faster execution, but uses memory - not sure if we want it @@ -80,7 +83,10 @@ namespace Umbraco.Web.PublishedCache.NuCache IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) + UrlSegmentProviderCollection urlSegmentProviders, + ISyncBootStateAccessor syncBootStateAccessor, + IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, + ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -100,6 +106,8 @@ namespace Umbraco.Web.PublishedCache.NuCache _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; _contentDataSerializer = contentDataSerializer; + _syncBootStateAccessor = syncBootStateAccessor; + // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member _entitySerializer = entitySerializer; @@ -218,7 +226,12 @@ namespace Umbraco.Web.PublishedCache.NuCache { var okContent = false; var okMedia = false; - + if (_syncBootStateAccessor.GetSyncBootState() == SyncBootState.ColdBoot) + { + _logger.Warn("Sync Service is in a Cold Boot state. Skip LoadCachesOnStartup as the Sync Service will trigger a full reload"); + _isReady = true; + return; + } try { if (_localContentDbExists) @@ -234,7 +247,7 @@ namespace Umbraco.Web.PublishedCache.NuCache if (!okMedia) _logger.Warn("Loading media from local db raised warnings, will reload from database."); } - + if (!okContent) LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));