yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Examine 2.0 integration (#10241)

Browse Source 
* Init commit for examine 2.0 work, most old umb examine tests working, probably a lot that doesn't

* Gets Umbraco Examine tests passing and makes some sense out of them, fixes some underlying issues.

* Large refactor, remove TaskHelper, rename Notifications to be consistent, Gets all examine/lucene indexes building and startup ordered in the correct way, removes old files, creates new IUmbracoIndexingHandler for abstracting out all index operations for umbraco data, abstracts out IIndexRebuilder, Fixes Stack overflow with LiveModelsProvider and loading assemblies, ports some changes from v8 for startup handling with cold boots, refactors out LastSyncedFileManager

* fix up issues with rebuilding and management dashboard.

* removes old files, removes NetworkHelper, fixes LastSyncedFileManager implementation to ensure the machine name is used, fix up logging with cold boot state.

* Makes MainDom safer to use and makes PublishedSnapshotService lazily register with MainDom

* lazily acquire application id (fix unit tests)

* Fixes resource casing and missing test file

* Ensures caches when requiring internal services for PublishedSnapshotService, UseNuCache is a separate call, shouldn't be buried in AddWebComponents, was also causing issues in integration tests since nucache was being used for the Id2Key service.

* For UmbracoTestServerTestBase enable nucache services

* Fixing tests

* Fix another test

* Fixes tests, use TestHostingEnvironment, make Tests.Common use net5, remove old Lucene.Net.Contrib ref.

* Fixes up some review notes

* Fixes issue with doubly registering PublishedSnapshotService meanig there could be 2x instances of it

* Checks for parseexception when executing the query

* Use application root instead of duplicating functionality.

* Added Examine project to netcore only solution file

* Fixed casing issue with LazyLoad, that is not lowercase.

* uses cancellationToken instead of bool flag, fixes always reading lastId from the LastSyncedFileManager, fixes RecurringHostedServiceBase so that there isn't an overlapping thread for the same task type

* Fix tests

* remove legacy test project from solution file

* Fix test

Co-authored-by: Bjarke Berg <mail@bergmania.dk>
This commit is contained in:
Shannon Deminick
2021-05-18 18:31:38 +10:00
committed by GitHub
parent 9b7b1a7c8e
commit eba6373a12
146 changed files with 2899 additions and 2904 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/Umbraco.Infrastructure/Sync/BatchedDatabaseServerMessenger.cs
View File

@@ -27,15 +27,18 @@ namespace Umbraco.Cms.Infrastructure.Sync
/// </summary>
public BatchedDatabaseServerMessenger(
IMainDom mainDom,
CacheRefresherCollection cacheRefreshers,
IServerRoleAccessor serverRoleAccessor,
ILogger<BatchedDatabaseServerMessenger> logger,
DatabaseServerMessengerCallbacks callbacks,
ISyncBootStateAccessor syncBootStateAccessor,
IHostingEnvironment hostingEnvironment,
ICacheInstructionService cacheInstructionService,
IJsonSerializer jsonSerializer,
IRequestCache requestCache,
IRequestAccessor requestAccessor,
LastSyncedFileManager lastSyncedFileManager,
IOptions<GlobalSettings> globalSettings)
: base(mainDom, logger, true, callbacks, hostingEnvironment, cacheInstructionService, jsonSerializer, globalSettings)
: base(mainDom, cacheRefreshers, serverRoleAccessor, logger, true, syncBootStateAccessor, hostingEnvironment, cacheInstructionService, jsonSerializer, lastSyncedFileManager, globalSettings)
{
_requestCache = requestCache;
_requestAccessor = requestAccessor;

187
src/Umbraco.Infrastructure/Sync/DatabaseServerMessenger.cs
View File

@@ -1,8 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using Microsoft.Extensions.Logging;
@@ -15,7 +13,6 @@ using Umbraco.Cms.Core.Runtime;
using Umbraco.Cms.Core.Serialization;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Sync;
using Umbraco.Extensions;
namespace Umbraco.Cms.Infrastructure.Sync
{
@@ -31,57 +28,61 @@ namespace Umbraco.Cms.Infrastructure.Sync
*/
private readonly IMainDom _mainDom;
private readonly CacheRefresherCollection _cacheRefreshers;
private readonly IServerRoleAccessor _serverRoleAccessor;
private readonly ISyncBootStateAccessor _syncBootStateAccessor;
private readonly ManualResetEvent _syncIdle;
private readonly object _locko = new object();
private readonly IHostingEnvironment _hostingEnvironment;
private readonly Lazy<string> _distCacheFilePath;
private int _lastId = -1;
private readonly LastSyncedFileManager _lastSyncedFileManager;
private DateTime _lastSync;
private DateTime _lastPruned;
private readonly Lazy<bool> _initialized;
private readonly Lazy<SyncBootState?> _initialized;
private bool _syncing;
private bool _released;
private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
private readonly CancellationToken _cancellationToken;
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseServerMessenger"/> class.
/// </summary>
protected DatabaseServerMessenger(
IMainDom mainDom,
CacheRefresherCollection cacheRefreshers,
IServerRoleAccessor serverRoleAccessor,
ILogger<DatabaseServerMessenger> logger,
bool distributedEnabled,
DatabaseServerMessengerCallbacks callbacks,
ISyncBootStateAccessor syncBootStateAccessor,
IHostingEnvironment hostingEnvironment,
ICacheInstructionService cacheInstructionService,
IJsonSerializer jsonSerializer,
LastSyncedFileManager lastSyncedFileManager,
IOptions<GlobalSettings> globalSettings)
: base(distributedEnabled)
{
_cancellationToken = _cancellationTokenSource.Token;
_mainDom = mainDom;
_cacheRefreshers = cacheRefreshers;
_serverRoleAccessor = serverRoleAccessor;
_hostingEnvironment = hostingEnvironment;
Logger = logger;
Callbacks = callbacks ?? throw new ArgumentNullException(nameof(callbacks));
_syncBootStateAccessor = syncBootStateAccessor;
CacheInstructionService = cacheInstructionService;
JsonSerializer = jsonSerializer;
_lastSyncedFileManager = lastSyncedFileManager;
GlobalSettings = globalSettings.Value;
_lastPruned = _lastSync = DateTime.UtcNow;
_syncIdle = new ManualResetEvent(true);
_distCacheFilePath = new Lazy<string>(() => GetDistCacheFilePath(hostingEnvironment));
// See notes on _localIdentity
LocalIdentity = NetworkHelper.MachineName // eg DOMAIN\SERVER
LocalIdentity = Environment.MachineName // eg DOMAIN\SERVER
+ "/" + hostingEnvironment.ApplicationId // eg /LM/S3SVC/11/ROOT
+ " [P" + Process.GetCurrentProcess().Id // eg 1234
+ "/D" + AppDomain.CurrentDomain.Id // eg 22
+ "] " + Guid.NewGuid().ToString("N").ToUpper(); // make it truly unique
_initialized = new Lazy<bool>(EnsureInitialized);
_initialized = new Lazy<SyncBootState?>(InitializeWithMainDom);
}
private string DistCacheFilePath => _distCacheFilePath.Value;
public DatabaseServerMessengerCallbacks Callbacks { get; }
public GlobalSettings GlobalSettings { get; }
protected ILogger<DatabaseServerMessenger> Logger { get; }
@@ -102,12 +103,17 @@ namespace Umbraco.Cms.Infrastructure.Sync
/// </remarks>
protected string LocalIdentity { get; }
/// <summary>
/// Returns true if initialization was successfull (i.e. Is MainDom)
/// </summary>
protected bool EnsureInitialized() => _initialized.Value.HasValue;
#region Messenger
// we don't care if there are servers listed or not,
// if distributed call is enabled we will make the call
protected override bool RequiresDistributed(ICacheRefresher refresher, MessageType dispatchType)
=> _initialized.Value && DistributedEnabled;
=> EnsureInitialized() && DistributedEnabled;
protected override void DeliverRemote(
ICacheRefresher refresher,
@@ -134,7 +140,7 @@ namespace Umbraco.Cms.Infrastructure.Sync
/// <summary>
/// Boots the messenger.
/// </summary>
private bool EnsureInitialized()
private SyncBootState? InitializeWithMainDom()
{
// weight:10, must release *before* the published snapshot service, because once released
// the service will *not* be able to properly handle our notifications anymore.
@@ -145,15 +151,15 @@ namespace Umbraco.Cms.Infrastructure.Sync
{
lock (_locko)
{
_released = true; // no more syncs
_cancellationTokenSource.Cancel(); // no more syncs
}
// Wait a max of 5 seconds and then return, so that we don't block
// Wait a max of 3 seconds and then return, so that we don't block
// the entire MainDom callbacks chain and prevent the AppDomain from
// properly releasing MainDom - a timeout here means that one refresher
// is taking too much time processing, however when it's done we will
// not update lastId and stop everything.
var idle = _syncIdle.WaitOne(5000);
var idle = _syncIdle.WaitOne(3000);
if (idle == false)
{
Logger.LogWarning("The wait lock timed out, application is shutting down. The current instruction batch will be re-processed.");
@@ -163,17 +169,11 @@ namespace Umbraco.Cms.Infrastructure.Sync
if (registered == false)
{
return false;
// return null if we cannot initialize
return null;
}
ReadLastSynced(); // get _lastId
if (CacheInstructionService.IsColdBootRequired(_lastId))
{
_lastId = -1; // reset _lastId if instructions are missing
}
return Initialize(); // boot
return InitializeColdBootState();
}
// <summary>
@@ -183,70 +183,32 @@ namespace Umbraco.Cms.Infrastructure.Sync
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// Callers MUST ensure thread-safety.
/// </remarks>
private bool Initialize()
private SyncBootState InitializeColdBootState()
{
lock (_locko)
{
if (_released)
if (_cancellationToken.IsCancellationRequested)
{
return false;
return SyncBootState.Unknown;
}
var coldboot = false;
SyncBootState syncState = _syncBootStateAccessor.GetSyncBootState();
// Never synced before.
if (_lastId < 0)
{
// We haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
// server and it will need to rebuild it's own caches, e.g. Lucene or the XML cache file.
Logger.LogWarning("No last synced Id found, this generally means this is a new server/install."
+ " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
+ " the database and maintain cache updates based on that Id.");
coldboot = true;
}
else
{
// Check for how many instructions there are to process, each row contains a count of the number of instructions contained in each
// row so we will sum these numbers to get the actual count.
var limit = GlobalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount;
if (CacheInstructionService.IsInstructionCountOverLimit(_lastId, limit, out int count))
{
// Too many instructions, proceed to cold boot.
Logger.LogWarning(
"The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount})."
+ " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id"
+ " to the latest found in the database and maintain cache updates based on that Id.",
count, limit);
coldboot = true;
}
}
if (coldboot)
if (syncState == SyncBootState.ColdBoot)
{
// Get the last id in the db and store it.
// Note: Do it BEFORE initializing otherwise some instructions might get lost
// when doing it before. Some instructions might run twice but this is not an issue.
var maxId = CacheInstructionService.GetMaxInstructionId();
// If there is a max currently, or if we've never synced.
if (maxId > 0 || _lastId < 0)
// if there is a max currently, or if we've never synced
if (maxId > 0 || _lastSyncedFileManager.LastSyncedId < 0)
{
SaveLastSynced(maxId);
}
// Execute initializing callbacks.
if (Callbacks.InitializingCallbacks != null)
{
foreach (Action callback in Callbacks.InitializingCallbacks)
{
callback();
}
_lastSyncedFileManager.SaveLastSyncedId(maxId);
}
}
return true;
return syncState;
}
}
@@ -255,7 +217,7 @@ namespace Umbraco.Cms.Infrastructure.Sync
/// </summary>
public override void Sync()
{
if (!_initialized.Value)
if (!EnsureInitialized())
{
return;
}
@@ -268,7 +230,7 @@ namespace Umbraco.Cms.Infrastructure.Sync
}
// Don't continue if we are released
if (_released)
if (_cancellationToken.IsCancellationRequested)
{
return;
}
@@ -286,7 +248,14 @@ namespace Umbraco.Cms.Infrastructure.Sync
try
{
CacheInstructionServiceProcessInstructionsResult result = CacheInstructionService.ProcessInstructions(_released, LocalIdentity, _lastPruned, _lastId);
ProcessInstructionsResult result = CacheInstructionService.ProcessInstructions(
_cacheRefreshers,
_serverRoleAccessor.CurrentServerRole,
_cancellationToken,
LocalIdentity,
_lastPruned,
_lastSyncedFileManager.LastSyncedId);
if (result.InstructionsWerePruned)
{
_lastPruned = _lastSync;
@@ -294,7 +263,7 @@ namespace Umbraco.Cms.Infrastructure.Sync
if (result.LastId > 0)
{
SaveLastSynced(result.LastId);
_lastSyncedFileManager.SaveLastSyncedId(result.LastId);
}
}
finally
@@ -309,60 +278,6 @@ namespace Umbraco.Cms.Infrastructure.Sync
}
}
/// <summary>
/// Reads the last-synced id from file into memory.
/// </summary>
/// <remarks>
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// </remarks>
private void ReadLastSynced()
{
if (File.Exists(DistCacheFilePath) == false)
{
return;
}
var content = File.ReadAllText(DistCacheFilePath);
if (int.TryParse(content, out var last))
{
_lastId = last;
}
}
/// <summary>
/// Updates the in-memory last-synced id and persists it to file.
/// </summary>
/// <param name="id">The id.</param>
/// <remarks>
/// Thread safety: this is NOT thread safe. Because it is NOT meant to run multi-threaded.
/// </remarks>
private void SaveLastSynced(int id)
{
File.WriteAllText(DistCacheFilePath, id.ToString(CultureInfo.InvariantCulture));
_lastId = id;
}
private string GetDistCacheFilePath(IHostingEnvironment hostingEnvironment)
{
var fileName = _hostingEnvironment.ApplicationId.ReplaceNonAlphanumericChars(string.Empty) + "-lastsynced.txt";
var distCacheFilePath = Path.Combine(hostingEnvironment.LocalTempPath, "DistCache", fileName);
//ensure the folder exists
var folder = Path.GetDirectoryName(distCacheFilePath);
if (folder == null)
{
throw new InvalidOperationException("The folder could not be determined for the file " + distCacheFilePath);
}
if (Directory.Exists(folder) == false)
{
Directory.CreateDirectory(folder);
}
return distCacheFilePath;
}
#endregion
}
}

89
src/Umbraco.Infrastructure/Sync/LastSyncedFileManager.cs Normal file
View File

@@ -0,0 +1,89 @@
using System;
using System.Globalization;
using System.IO;
using System.Threading;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Extensions;
namespace Umbraco.Cms.Infrastructure.Sync
{
public sealed class LastSyncedFileManager
{
private string _distCacheFile;
private bool _lastIdReady;
private object _lastIdLock;
private int _lastId;
private readonly IHostingEnvironment _hostingEnvironment;
public LastSyncedFileManager(IHostingEnvironment hostingEnvironment)
=> _hostingEnvironment = hostingEnvironment;
/// <summary>
/// Persists the last-synced id to file.
/// </summary>
/// <param name="id">The id.</param>
public void SaveLastSyncedId(int id)
{
lock (_lastIdLock)
{
if (!_lastIdReady)
{
throw new InvalidOperationException("Cannot save the last synced id before it is read");
}
File.WriteAllText(DistCacheFilePath, id.ToString(CultureInfo.InvariantCulture));
_lastId = id;
}
}
/// <summary>
/// Returns the last-synced id.
/// </summary>
public int LastSyncedId => LazyInitializer.EnsureInitialized(
ref _lastId,
ref _lastIdReady,
ref _lastIdLock,
() =>
{
// On first load, read from file, else it will return the in-memory _lastId value
var distCacheFilePath = DistCacheFilePath;
if (File.Exists(distCacheFilePath))
{
var content = File.ReadAllText(distCacheFilePath);
if (int.TryParse(content, out var last))
{
return last;
}
}
return -1;
});
/// <summary>
/// Gets the dist cache file path (once).
/// </summary>
/// <returns></returns>
public string DistCacheFilePath => LazyInitializer.EnsureInitialized(ref _distCacheFile, () =>
{
var fileName = (Environment.MachineName + _hostingEnvironment.ApplicationId).GenerateHash() + "-lastsynced.txt";
var distCacheFilePath = Path.Combine(_hostingEnvironment.LocalTempPath, "DistCache", fileName);
//ensure the folder exists
var folder = Path.GetDirectoryName(distCacheFilePath);
if (folder == null)
{
throw new InvalidOperationException("The folder could not be determined for the file " + distCacheFilePath);
}
if (Directory.Exists(folder) == false)
{
Directory.CreateDirectory(folder);
}
return distCacheFilePath;
});
}
}

84
src/Umbraco.Infrastructure/Sync/SyncBootStateAccessor.cs Normal file
View File

@@ -0,0 +1,84 @@
using System.Threading;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Sync;
namespace Umbraco.Cms.Infrastructure.Sync
{
public class SyncBootStateAccessor : ISyncBootStateAccessor
{
private readonly ILogger<SyncBootStateAccessor> _logger;
private readonly LastSyncedFileManager _lastSyncedFileManager;
private readonly GlobalSettings _globalSettings;
private readonly ICacheInstructionService _cacheInstructionService;
private SyncBootState _syncBootState;
private bool _syncBootStateReady;
private object _syncBootStateLock;
public SyncBootStateAccessor(
ILogger<SyncBootStateAccessor> logger,
LastSyncedFileManager lastSyncedFileManager,
IOptions<GlobalSettings> globalSettings,
ICacheInstructionService cacheInstructionService)
{
_logger = logger;
_lastSyncedFileManager = lastSyncedFileManager;
_globalSettings = globalSettings.Value;
_cacheInstructionService = cacheInstructionService;
}
public SyncBootState GetSyncBootState()
=> LazyInitializer.EnsureInitialized(
ref _syncBootState,
ref _syncBootStateReady,
ref _syncBootStateLock,
() => InitializeColdBootState(_lastSyncedFileManager.LastSyncedId));
private SyncBootState InitializeColdBootState(int lastId)
{
var coldboot = false;
// Never synced before.
if (lastId < 0)
{
// We haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
// server and it will need to rebuild it's own caches, e.g. Lucene or the XML cache file.
_logger.LogWarning("No last synced Id found, this generally means this is a new server/install. "
+ "A cold boot will be triggered.");
coldboot = true;
}
else
{
if (_cacheInstructionService.IsColdBootRequired(lastId))
{
_logger.LogWarning("Last synced Id found {LastSyncedId} but was not found in the database. This generally means this server/install "
+ " has been idle for too long and the instructions in the database have been pruned. A cold boot will be triggered.", lastId);
coldboot = true;
}
else
{
// Check for how many instructions there are to process, each row contains a count of the number of instructions contained in each
// row so we will sum these numbers to get the actual count.
var limit = _globalSettings.DatabaseServerMessenger.MaxProcessingInstructionCount;
if (_cacheInstructionService.IsInstructionCountOverLimit(lastId, limit, out int count))
{
// Too many instructions, proceed to cold boot.
_logger.LogWarning(
"The instruction count ({InstructionCount}) exceeds the specified MaxProcessingInstructionCount ({MaxProcessingInstructionCount}). "
+ "A cold boot will be triggered.",
count, limit);
coldboot = true;
}
}
}
return coldboot ? SyncBootState.ColdBoot : SyncBootState.WarmBoot;
}
}
}