yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1071 from umbraco/temp-U4-7682

Browse Source 
U4-7682 Add option to DatabaseServerMessengerOptions to force a Cold …
This commit is contained in:
Stephan
2016-01-28 14:33:02 +01:00
parent 256849f8a6 c9c451be65
commit 107e66e4da
3 changed files with 118 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/Umbraco.Core/Sync/DatabaseServerMessenger.cs
View File

@@ -60,7 +60,7 @@ namespace Umbraco.Core.Sync
protected override bool RequiresDistributed(IEnumerable<IServerAddress> servers, ICacheRefresher refresher, MessageType dispatchType)
{
// we don't care if there's servers listed or not,
// we don't care if there's servers listed or not,
// if distributed call is enabled we will make the call
return _initialized && DistributedEnabled;
}
@@ -139,12 +139,35 @@ namespace Umbraco.Core.Sync
{
if (_released) return;
var coldboot = false;
if (_lastId < 0) // never synced before
{
// we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
// we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
// server and it will need to rebuild it's own caches, eg Lucene or the xml cache file.
_logger.Warn<DatabaseServerMessenger>("No last synced Id found, this generally means this is a new server/install. The server will rebuild its caches and indexes and then adjust it's last synced id to the latest found in the database and will start maintaining cache updates based on that id");
_logger.Warn<DatabaseServerMessenger>("No last synced Id found, this generally means this is a new server/install."
+ " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
+ " the database and maintain cache updates based on that Id.");
coldboot = true;
}
else
{
//check for how many instructions there are to process
var count = _appContext.DatabaseContext.Database.ExecuteScalar<int>("SELECT COUNT(*) FROM umbracoCacheInstruction WHERE id > @lastId", new {lastId = _lastId});
if (count > _options.MaxProcessingInstructionCount)
{
//too many instructions, proceed to cold boot
_logger.Warn<DatabaseServerMessenger>("The instruction count ({0}) exceeds the specified MaxProcessingInstructionCount ({1})."
+ " The server will skip existing instructions, rebuild its caches and indexes entirely, adjust its last synced Id"
+ " to the latest found in the database and maintain cache updates based on that Id.",
() => count, () => _options.MaxProcessingInstructionCount);
coldboot = true;
}
}
if (coldboot)
{
// go get the last id in the db and store it
// note: do it BEFORE initializing otherwise some instructions might get lost
// when doing it before, some instructions might run twice - not an issue
@@ -169,7 +192,7 @@ namespace Umbraco.Core.Sync
{
lock (_locko)
{
if (_syncing)
if (_syncing)
return;
if (_released)
@@ -213,9 +236,9 @@ namespace Umbraco.Core.Sync
private void ProcessDatabaseInstructions()
{
// NOTE
// we 'could' recurse to ensure that no remaining instructions are pending in the table before proceeding but I don't think that
// we 'could' recurse to ensure that no remaining instructions are pending in the table before proceeding but I don't think that
// would be a good idea since instructions could keep getting added and then all other threads will probably get stuck from serving requests
// (depending on what the cache refreshers are doing). I think it's best we do the one time check, process them and continue, if there are
// (depending on what the cache refreshers are doing). I think it's best we do the one time check, process them and continue, if there are
// pending requests after being processed, they'll just be processed on the next poll.
//
// FIXME not true if we're running on a background thread, assuming we can?
@@ -281,7 +304,7 @@ namespace Umbraco.Core.Sync
/// Remove old instructions from the database
/// </summary>
/// <remarks>
/// Always leave the last (most recent) record in the db table, this is so that not all instructions are removed which would cause
/// Always leave the last (most recent) record in the db table, this is so that not all instructions are removed which would cause
/// the site to cold boot if there's been no instruction activity for more than DaysToRetainInstructions.
/// See: http://issues.umbraco.org/issue/U4-7643#comment=67-25085
/// </remarks>
@@ -290,15 +313,15 @@ namespace Umbraco.Core.Sync
var pruneDate = DateTime.UtcNow.AddDays(-_options.DaysToRetainInstructions);
var sqlSyntax = _appContext.DatabaseContext.SqlSyntax;
//NOTE: this query could work on SQL server and MySQL:
//NOTE: this query could work on SQL server and MySQL:
/*
SELECT id
FROM umbracoCacheInstruction
WHERE utcStamp < getdate()
WHERE utcStamp < getdate()
AND id <> (SELECT MAX(id) FROM umbracoCacheInstruction)
*/
// However, this will not work on SQLCE and in fact it will be slower than the query we are
// using if the SQL server doesn't perform it's own query optimizations (i.e. since the above
// using if the SQL server doesn't perform it's own query optimizations (i.e. since the above
// query could actually execute a sub query for every row found). So we've had to go with an
// inner join which is faster and works on SQLCE but it's uglier to read.
@@ -331,9 +354,9 @@ namespace Umbraco.Core.Sync
var dtos = _appContext.DatabaseContext.Database.Fetch<CacheInstructionDto>(sql);
if (dtos.Count == 0)
_lastId = -1;
_lastId = -1;
}
/// <summary>
/// Reads the last-synced id from file into memory.
/// </summary>
@@ -502,4 +525,3 @@ namespace Umbraco.Core.Sync
#endregion
}
}

10
src/Umbraco.Core/Sync/DatabaseServerMessengerOptions.cs
View File

@@ -2,7 +2,7 @@ using System;
using System.Collections.Generic;
namespace Umbraco.Core.Sync
{
{
/// <summary>
/// Provides options to the <see cref="DatabaseServerMessenger"/>.
/// </summary>
@@ -14,9 +14,15 @@ namespace Umbraco.Core.Sync
public DatabaseServerMessengerOptions()
{
DaysToRetainInstructions = 2; // 2 days
ThrottleSeconds = 5; // 5 seconds
ThrottleSeconds = 5; // 5 second
MaxProcessingInstructionCount = 1000;
}
/// <summary>
/// The maximum number of instructions that can be processed at startup; otherwise the server cold-boots (rebuilds its caches).
/// </summary>
public int MaxProcessingInstructionCount { get; set; }
/// <summary>
/// A list of callbacks that will be invoked if the lastsynced.txt file does not exist.
/// </summary>

128
src/Umbraco.Web/WebBootManager.cs
View File

@@ -12,6 +12,7 @@ using System.Web.Routing;
using ClientDependency.Core.Config;
using Examine;
using Examine.Config;
using Examine.Providers;
using umbraco;
using Umbraco.Core;
using Umbraco.Core.Configuration;
@@ -51,13 +52,13 @@ using ProfilingViewEngine = Umbraco.Core.Profiling.ProfilingViewEngine;
namespace Umbraco.Web
{
/// <summary>
/// A bootstrapper for the Umbraco application which initializes all objects including the Web portion of the application
/// A bootstrapper for the Umbraco application which initializes all objects including the Web portion of the application
/// </summary>
public class WebBootManager : CoreBootManager
{
private readonly bool _isForTesting;
//NOTE: see the Initialize method for what this is used for
private readonly List<IIndexer> _indexesToRebuild = new List<IIndexer>();
private static readonly List<BaseIndexProvider> IndexesToRebuild = new List<BaseIndexProvider>();
public WebBootManager(UmbracoApplicationBase umbracoApplication)
: base(umbracoApplication)
@@ -104,7 +105,7 @@ namespace Umbraco.Web
public override IBootManager Initialize()
{
//This is basically a hack for this item: http://issues.umbraco.org/issue/U4-5976
// when Examine initializes it will try to rebuild if the indexes are empty, however in many cases not all of Examine's
// when Examine initializes it will try to rebuild if the indexes are empty, however in many cases not all of Examine's
// event handlers will be assigned during bootup when the rebuilding starts which is a problem. So with the examine 0.1.58.2941 build
// it has an event we can subscribe to in order to cancel this rebuilding process, but what we'll do is cancel it and postpone the rebuilding until the
// boot process has completed. It's a hack but it works.
@@ -145,7 +146,7 @@ namespace Umbraco.Web
{ "compositeFileHandlerPath", ClientDependencySettings.Instance.CompositeFileHandlerPath }
});
ClientDependencySettings.Instance.MvcRendererCollection.Add(renderer);
// Disable the X-AspNetMvc-Version HTTP Header
MvcHandler.DisableMvcResponseHeader = true;
@@ -154,9 +155,9 @@ namespace Umbraco.Web
return this;
}
/// <summary>
/// Override this method in order to ensure that the UmbracoContext is also created, this can only be
/// Override this method in order to ensure that the UmbracoContext is also created, this can only be
/// created after resolution is frozen!
/// </summary>
protected override void FreezeResolution()
@@ -170,8 +171,8 @@ namespace Umbraco.Web
httpContext,
ApplicationContext,
new WebSecurity(httpContext, ApplicationContext),
UmbracoConfig.For.UmbracoSettings(),
UrlProviderResolver.Current.Providers,
UmbracoConfig.For.UmbracoSettings(),
UrlProviderResolver.Current.Providers,
false);
}
@@ -187,7 +188,7 @@ namespace Umbraco.Web
ProfilerResolver.Current.SetProfiler(profiler);
profiler.Start();
}
/// <summary>
/// Ensure that the OnApplicationStarted methods of the IApplicationEvents are called
/// </summary>
@@ -205,14 +206,14 @@ namespace Umbraco.Web
//Now, startup all of our legacy startup handler
ApplicationEventsResolver.Current.InstantiateLegacyStartupHandlers();
//Ok, now that everything is complete we'll check if we've stored any references to index that need rebuilding and run them
//Ok, now that everything is complete we'll check if we've stored any references to index that need rebuilding and run them
// (see the initialize method for notes) - we'll ensure we remove the event handler too in case examine manager doesn't actually
// initialize during startup, in which case we want it to rebuild the indexes itself.
ExamineManager.Instance.BuildingEmptyIndexOnStartup -= OnInstanceOnBuildingEmptyIndexOnStartup;
if (_indexesToRebuild.Any())
if (IndexesToRebuild.Any())
{
foreach (var indexer in _indexesToRebuild)
foreach (var indexer in IndexesToRebuild)
{
indexer.RebuildIndex();
}
@@ -244,14 +245,14 @@ namespace Umbraco.Web
{
//create a web-based cache helper
var cacheHelper = new CacheHelper(
//we need to have the dep clone runtime cache provider to ensure
//we need to have the dep clone runtime cache provider to ensure
//all entities are cached properly (cloned in and cloned out)
new DeepCloneRuntimeCacheProvider(new HttpRuntimeCacheProvider(HttpRuntime.Cache)),
new StaticCacheProvider(),
//we have no request based cache when not running in web-based context
new NullCacheProvider(),
new IsolatedRuntimeCache(type =>
//we need to have the dep clone runtime cache provider to ensure
//we need to have the dep clone runtime cache provider to ensure
//all entities are cached properly (cloned in and cloned out)
new DeepCloneRuntimeCacheProvider(new ObjectCacheRuntimeCacheProvider())));
@@ -282,7 +283,7 @@ namespace Umbraco.Web
//plugin controllers must come first because the next route will catch many things
RoutePluginControllers();
}
private void RoutePluginControllers()
{
var umbracoPath = GlobalSettings.UmbracoMvcArea;
@@ -349,14 +350,14 @@ namespace Umbraco.Web
umbracoPath + "/Surface/" + meta.ControllerName + "/{action}/{id}",//url to match
new { controller = meta.ControllerName, action = "Index", id = UrlParameter.Optional },
new[] { meta.ControllerNamespace }); //look in this namespace to create the controller
route.DataTokens.Add("umbraco", "surface"); //ensure the umbraco token is set
route.DataTokens.Add("umbraco", "surface"); //ensure the umbraco token is set
route.DataTokens.Add("UseNamespaceFallback", false); //Don't look anywhere else except this namespace!
//make it use our custom/special SurfaceMvcHandler
route.RouteHandler = new SurfaceRouteHandler();
}
/// <summary>
/// Initializes all web based and core resolves
/// Initializes all web based and core resolves
/// </summary>
protected override void InitializeResolvers()
{
@@ -375,7 +376,7 @@ namespace Umbraco.Web
//set the legacy one by default - this maintains backwards compat
ServerMessengerResolver.Current.SetServerMessenger(new BatchedWebServiceServerMessenger(() =>
{
//we should not proceed to change this if the app/database is not configured since there will
//we should not proceed to change this if the app/database is not configured since there will
// be no user, plus we don't need to have server messages sent if this is the case.
if (ApplicationContext.IsConfigured && ApplicationContext.DatabaseContext.IsDatabaseConfigured)
{
@@ -403,49 +404,40 @@ namespace Umbraco.Web
else
{
// NOTE: This is IMPORTANT! ... we don't want to rebuild any index that is already flagged to be re-indexed
// on startup based on our _indexesToRebuild variable and how Examine auto-rebuilds when indexes are empty
// this callback is used below for the DatabaseServerMessenger startup options
//We are using a custom action here so we can check the examine settings value first, we don't want to
// put that check into the CreateIndexesOnColdBoot method because developers may choose to use this
// method directly and they will be in charge of this check if they need it
Action rebuildIndexes = () =>
{
//If the developer has explicitly opted out of rebuilding indexes on startup then we
//If the developer has explicitly opted out of rebuilding indexes on startup then we
// should adhere to that and not do it, this means that if they are load balancing things will be
// out of sync if they are auto-scaling but there's not much we can do about that.
if (ExamineSettings.Instance.RebuildOnAppStart == false) return;
if (_indexesToRebuild.Any())
foreach (var indexer in GetIndexesForColdBoot())
{
var otherIndexes = ExamineManager.Instance.IndexProviderCollection.Except(_indexesToRebuild);
foreach (var otherIndex in otherIndexes)
{
otherIndex.RebuildIndex();
}
}
else
{
//rebuild them all
ExamineManager.Instance.RebuildIndex();
indexer.RebuildIndex();
}
};
ServerMessengerResolver.Current.SetServerMessenger(new BatchedDatabaseServerMessenger(
ApplicationContext,
true,
//Default options for web including the required callbacks to build caches
new DatabaseServerMessengerOptions
{
//These callbacks will be executed if the server has not been synced
// (i.e. it is a new server or the lastsynced.txt file has been removed)
InitializingCallbacks = new Action[]
ApplicationContext,
true,
//Default options for web including the required callbacks to build caches
new DatabaseServerMessengerOptions
{
//rebuild the xml cache file if the server is not synced
() => global::umbraco.content.Instance.RefreshContentFromDatabase(),
//rebuild indexes if the server is not synced
// NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
// indexes then they can adjust this logic themselves.
rebuildIndexes
}
}));
//These callbacks will be executed if the server has not been synced
// (i.e. it is a new server or the lastsynced.txt file has been removed)
InitializingCallbacks = new Action[]
{
//rebuild the xml cache file if the server is not synced
() => global::umbraco.content.Instance.RefreshContentFromDatabase(),
//rebuild indexes if the server is not synced
// NOTE: This will rebuild ALL indexes including the members, if developers want to target specific
// indexes then they can adjust this logic themselves.
rebuildIndexes
}
}));
}
SurfaceControllerResolver.Current = new SurfaceControllerResolver(
@@ -470,7 +462,7 @@ namespace Umbraco.Web
new PublishedCache.XmlPublishedCache.PublishedContentCache(),
new PublishedCache.XmlPublishedCache.PublishedMediaCache(ApplicationContext)));
GlobalConfiguration.Configuration.Services.Replace(typeof(IHttpControllerSelector),
GlobalConfiguration.Configuration.Services.Replace(typeof(IHttpControllerSelector),
new NamespaceHttpControllerSelector(GlobalConfiguration.Configuration));
FilteredControllerFactoriesResolver.Current = new FilteredControllerFactoriesResolver(
@@ -533,12 +525,42 @@ namespace Umbraco.Web
new DefaultCultureDictionaryFactory());
}
/// <summary>
/// The method used to create indexes on a cold boot
/// </summary>
/// <remarks>
/// A cold boot is when the server determines it will not (or cannot) process instructions in the cache table and
/// will rebuild it's own caches itself.
/// </remarks>
public static IEnumerable<BaseIndexProvider> GetIndexesForColdBoot()
{
// NOTE: This is IMPORTANT! ... we don't want to rebuild any index that is already flagged to be re-indexed
// on startup based on our _indexesToRebuild variable and how Examine auto-rebuilds when indexes are empty.
// This callback is used above for the DatabaseServerMessenger startup options.
// all indexes
IEnumerable<BaseIndexProvider> indexes = ExamineManager.Instance.IndexProviderCollection;
// except those that are already flagged
// and are processed in Complete()
if (IndexesToRebuild.Any())
indexes = indexes.Except(IndexesToRebuild);
// return
foreach (var index in indexes)
yield return index;
// and clear
IndexesToRebuild.Clear();
}
private void OnInstanceOnBuildingEmptyIndexOnStartup(object sender, BuildingEmptyIndexOnStartupEventArgs args)
{
//store the indexer that needs rebuilding because it's empty for when the boot process
//store the indexer that needs rebuilding because it's empty for when the boot process
// is complete and cancel this current event so the rebuild process doesn't start right now.
args.Cancel = true;
_indexesToRebuild.Add(args.Indexer);
IndexesToRebuild.Add((BaseIndexProvider)args.Indexer);
}
}
}