From 6462329494b1fb75ed6a690cb3cb8393e13e661c Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 31 Oct 2017 11:54:08 +1100 Subject: [PATCH 1/2] U4-9120 Add instruction count column on the db instructions table --- .../Models/Rdbms/CacheInstructionDto.cs | 5 +++ .../AddInstructionCountColumn.cs | 36 +++++++++++++++++++ .../Sync/DatabaseServerMessenger.cs | 8 ++--- src/Umbraco.Core/Umbraco.Core.csproj | 1 + .../BatchedDatabaseServerMessenger.cs | 9 ++--- 5 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 src/Umbraco.Core/Persistence/Migrations/Upgrades/TargetVersionSevenEightZero/AddInstructionCountColumn.cs diff --git a/src/Umbraco.Core/Models/Rdbms/CacheInstructionDto.cs b/src/Umbraco.Core/Models/Rdbms/CacheInstructionDto.cs index c24004b7dc..8646190423 100644 --- a/src/Umbraco.Core/Models/Rdbms/CacheInstructionDto.cs +++ b/src/Umbraco.Core/Models/Rdbms/CacheInstructionDto.cs @@ -27,5 +27,10 @@ namespace Umbraco.Core.Models.Rdbms [NullSetting(NullSetting = NullSettings.NotNull)] [Length(500)] public string OriginIdentity { get; set; } + + [Column("instructionCount")] + [NullSetting(NullSetting = NullSettings.NotNull)] + [Constraint(Default = 1)] + public int InstructionCount { get; set; } } } \ No newline at end of file diff --git a/src/Umbraco.Core/Persistence/Migrations/Upgrades/TargetVersionSevenEightZero/AddInstructionCountColumn.cs b/src/Umbraco.Core/Persistence/Migrations/Upgrades/TargetVersionSevenEightZero/AddInstructionCountColumn.cs new file mode 100644 index 0000000000..920f319fea --- /dev/null +++ b/src/Umbraco.Core/Persistence/Migrations/Upgrades/TargetVersionSevenEightZero/AddInstructionCountColumn.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Logging; +using Umbraco.Core.Persistence.SqlSyntax; + +namespace Umbraco.Core.Persistence.Migrations.Upgrades.TargetVersionSevenEightZero +{ + [Migration("7.8.0", 2, Constants.System.UmbracoMigrationName)] + public class AddInstructionCountColumn : MigrationBase + { + public AddInstructionCountColumn(ISqlSyntaxProvider sqlSyntax, ILogger logger) + : base(sqlSyntax, logger) + { + } + + public override void Up() + { + //Don't exeucte if the column is already there + var columns = SqlSyntax.GetColumnsInSchema(Context.Database).ToArray(); + + if (columns.Any(x => x.TableName.InvariantEquals("umbracoCacheInstruction") && x.ColumnName.InvariantEquals("instructionCount")) == false) + Create.Column("instructionCount") + .OnTable("umbracoCacheInstruction") + .AsInt32() + .WithDefaultValue(1) + .NotNullable(); + } + + public override void Down() + { + } + } +} diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs index 6f1fc03281..1e639502e6 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs @@ -165,10 +165,10 @@ namespace Umbraco.Core.Sync } else { - //check for how many instructions there are to process - //TODO: In 7.6 we need to store the count of instructions per row since this is not affective because there can be far more than one (if not thousands) - // of instructions in a single row. - var count = _appContext.DatabaseContext.Database.ExecuteScalar("SELECT COUNT(*) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); + //check for how many instructions there are to process, each row contains a count of the number of instructions contained in each + //row so we will sum these numbers to get the actual count. + var count = _appContext.DatabaseContext.Database + .ExecuteScalar("SELECT SUM(instructionCount) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId}); if (count > Options.MaxProcessingInstructionCount) { //too many instructions, proceed to cold boot diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index fb9f8b06a1..4d55ec1590 100644 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -546,6 +546,7 @@ + diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 23edb2dde2..5cb3e93b22 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -88,18 +88,19 @@ namespace Umbraco.Web //Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) { - WriteInstructions(instructionsBatch); + WriteInstructions(instructionsBatch.ToArray()); } } - private void WriteInstructions(IEnumerable instructions) + private void WriteInstructions(RefreshInstruction[] instructions) { var dto = new CacheInstructionDto { UtcStamp = DateTime.UtcNow, Instructions = JsonConvert.SerializeObject(instructions, Formatting.None), - OriginIdentity = LocalIdentity + OriginIdentity = LocalIdentity, + InstructionCount = instructions.Length }; ApplicationContext.DatabaseContext.Database.Insert(dto); @@ -145,7 +146,7 @@ namespace Umbraco.Web //only write the json blob with a maximum count of the MaxProcessingInstructionCount foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) { - WriteInstructions(maxBatch); + WriteInstructions(maxBatch.ToArray()); } } else From 68e6ad76fb03731b3292fc1e9e308e9403d92354 Mon Sep 17 00:00:00 2001 From: Shannon Date: Tue, 31 Oct 2017 12:39:08 +1100 Subject: [PATCH 2/2] Make sure the count of instructions takes into account individual ids when using RefreshByIds --- .../Sync/DatabaseServerMessenger.cs | 3 +- src/Umbraco.Core/Sync/RefreshInstruction.cs | 39 +++++++++++++++++-- .../BatchedDatabaseServerMessenger.cs | 8 ++-- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs index 1e639502e6..420bf1322b 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs @@ -87,7 +87,8 @@ namespace Umbraco.Core.Sync { UtcStamp = DateTime.UtcNow, Instructions = JsonConvert.SerializeObject(instructions, Formatting.None), - OriginIdentity = LocalIdentity + OriginIdentity = LocalIdentity, + InstructionCount = instructions.Sum(x => x.JsonIdCount) }; ApplicationContext.DatabaseContext.Database.Insert(dto); diff --git a/src/Umbraco.Core/Sync/RefreshInstruction.cs b/src/Umbraco.Core/Sync/RefreshInstruction.cs index cd10019f00..921f280d6e 100644 --- a/src/Umbraco.Core/Sync/RefreshInstruction.cs +++ b/src/Umbraco.Core/Sync/RefreshInstruction.cs @@ -18,7 +18,10 @@ namespace Umbraco.Core.Sync // need this public, parameter-less constructor so the web service messenger // can de-serialize the instructions it receives public RefreshInstruction() - { } + { + //set default - this value is not used for reading after it's been deserialized, it's only used for persisting the instruction to the db + JsonIdCount = 1; + } // need this public one so it can be de-serialized - used by the Json thing // otherwise, should use GetInstructions(...) @@ -30,12 +33,16 @@ namespace Umbraco.Core.Sync IntId = intId; JsonIds = jsonIds; JsonPayload = jsonPayload; + //set default - this value is not used for reading after it's been deserialized, it's only used for persisting the instruction to the db + JsonIdCount = 1; } private RefreshInstruction(ICacheRefresher refresher, RefreshMethodType refreshType) { RefresherId = refresher.UniqueIdentifier; RefreshType = refreshType; + //set default - this value is not used for reading after it's been deserialized, it's only used for persisting the instruction to the db + JsonIdCount = 1; } private RefreshInstruction(ICacheRefresher refresher, RefreshMethodType refreshType, Guid guidId) @@ -50,9 +57,21 @@ namespace Umbraco.Core.Sync IntId = intId; } - private RefreshInstruction(ICacheRefresher refresher, RefreshMethodType refreshType, string json) + /// + /// A private constructor to create a new instance + /// + /// + /// + /// + /// + /// When the refresh method is we know how many Ids are being refreshed so we know the instruction + /// count which will be taken into account when we store this count in the database. + /// + private RefreshInstruction(ICacheRefresher refresher, RefreshMethodType refreshType, string json, int idCount = 1) : this(refresher, refreshType) { + JsonIdCount = idCount; + if (refreshType == RefreshMethodType.RefreshByJson) JsonPayload = json; else @@ -77,8 +96,12 @@ namespace Umbraco.Core.Sync case MessageType.RefreshById: if (idType == null) throw new InvalidOperationException("Cannot refresh by id if idType is null."); - if (idType == typeof (int)) // bulk of ints is supported - return new[] { new RefreshInstruction(refresher, RefreshMethodType.RefreshByIds, JsonConvert.SerializeObject(ids.Cast().ToArray())) }; + if (idType == typeof(int)) + { + // bulk of ints is supported + var intIds = ids.Cast().ToArray(); + return new[] { new RefreshInstruction(refresher, RefreshMethodType.RefreshByIds, JsonConvert.SerializeObject(intIds), intIds.Length) }; + } // else must be guids, bulk of guids is not supported, iterate return ids.Select(x => new RefreshInstruction(refresher, RefreshMethodType.RefreshByGuid, (Guid) x)); @@ -121,6 +144,14 @@ namespace Umbraco.Core.Sync /// public string JsonIds { get; set; } + /// + /// Gets or sets the number of Ids contained in the JsonIds json value + /// + /// + /// This is used to determine the instruction count per row + /// + public int JsonIdCount { get; set; } + /// /// Gets or sets the payload data value. /// diff --git a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs index 5cb3e93b22..e0d0eb0c46 100644 --- a/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs +++ b/src/Umbraco.Web/BatchedDatabaseServerMessenger.cs @@ -88,19 +88,19 @@ namespace Umbraco.Web //Write the instructions but only create JSON blobs with a max instruction count equal to MaxProcessingInstructionCount foreach (var instructionsBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) { - WriteInstructions(instructionsBatch.ToArray()); + WriteInstructions(instructionsBatch); } } - private void WriteInstructions(RefreshInstruction[] instructions) + private void WriteInstructions(IEnumerable instructions) { var dto = new CacheInstructionDto { UtcStamp = DateTime.UtcNow, Instructions = JsonConvert.SerializeObject(instructions, Formatting.None), OriginIdentity = LocalIdentity, - InstructionCount = instructions.Length + InstructionCount = instructions.Sum(x => x.JsonIdCount) }; ApplicationContext.DatabaseContext.Database.Insert(dto); @@ -146,7 +146,7 @@ namespace Umbraco.Web //only write the json blob with a maximum count of the MaxProcessingInstructionCount foreach (var maxBatch in instructions.InGroupsOf(Options.MaxProcessingInstructionCount)) { - WriteInstructions(maxBatch.ToArray()); + WriteInstructions(maxBatch); } } else