yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'v8/bugfix/background-pure-live-rebuild-AB1833' into v8/bugfix/AB2684-purelive-model-regen

Browse Source 
# Conflicts:
#	src/Umbraco.Web/Cache/ContentTypeCacheRefresher.cs
#	src/Umbraco.Web/Cache/DataTypeCacheRefresher.cs
This commit is contained in:
Shannon
2019-09-16 17:38:10 +10:00
parent 00c720413a aa9ece94b7
commit 2b9ab0a38a
22 changed files with 497 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/Umbraco.Core/Configuration/GlobalSettings.cs
View File

@@ -33,7 +33,6 @@ namespace Umbraco.Core.Configuration
/// </summary>
private static void ResetInternal()
{
GlobalSettingsExtensions.Reset();
_reservedPaths = null;
_reservedUrls = null;
HasSmtpServer = null;

113
src/Umbraco.Core/Configuration/GlobalSettingsExtensions.cs
View File

@@ -1,6 +1,8 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Web;
using System.Web.Routing;
using Umbraco.Core.IO;
@@ -9,22 +11,9 @@ namespace Umbraco.Core.Configuration
{
public static class GlobalSettingsExtensions
{
/// <summary>
/// Used in unit testing to reset all config items, this is automatically called by GlobalSettings.Reset()
/// </summary>
internal static void Reset()
{
_reservedUrlsCache = null;
_mvcArea = null;
}
private static readonly object Locker = new object();
//make this volatile so that we can ensure thread safety with a double check lock
private static volatile string _reservedUrlsCache;
private static string _reservedPathsCache;
private static HashSet<string> _reservedList = new HashSet<string>();
private static string _mvcArea;
/// <summary>
/// This returns the string of the MVC Area route.
/// </summary>
@@ -40,6 +29,13 @@ namespace Umbraco.Core.Configuration
{
if (_mvcArea != null) return _mvcArea;
_mvcArea = GetUmbracoMvcAreaNoCache(globalSettings);
return _mvcArea;
}
internal static string GetUmbracoMvcAreaNoCache(this IGlobalSettings globalSettings)
{
if (globalSettings.Path.IsNullOrWhiteSpace())
{
throw new InvalidOperationException("Cannot create an MVC Area path without the umbracoPath specified");
@@ -48,95 +44,8 @@ namespace Umbraco.Core.Configuration
var path = globalSettings.Path;
if (path.StartsWith(SystemDirectories.Root)) // beware of TrimStart, see U4-2518
path = path.Substring(SystemDirectories.Root.Length);
_mvcArea = path.TrimStart('~').TrimStart('/').Replace('/', '-').Trim().ToLower();
return _mvcArea;
return path.TrimStart('~').TrimStart('/').Replace('/', '-').Trim().ToLower();
}
/// <summary>
/// Determines whether the specified URL is reserved or is inside a reserved path.
/// </summary>
/// <param name="globalSettings"></param>
/// <param name="url">The URL to check.</param>
/// <returns>
/// <c>true</c> if the specified URL is reserved; otherwise, <c>false</c>.
/// </returns>
internal static bool IsReservedPathOrUrl(this IGlobalSettings globalSettings, string url)
{
if (_reservedUrlsCache == null)
{
lock (Locker)
{
if (_reservedUrlsCache == null)
{
// store references to strings to determine changes
_reservedPathsCache = globalSettings.ReservedPaths;
_reservedUrlsCache = globalSettings.ReservedUrls;
// add URLs and paths to a new list
var newReservedList = new HashSet<string>();
foreach (var reservedUrlTrimmed in _reservedUrlsCache
.Split(new[] {","}, StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Trim().ToLowerInvariant())
.Where(x => x.IsNullOrWhiteSpace() == false)
.Select(reservedUrl => IOHelper.ResolveUrl(reservedUrl).Trim().EnsureStartsWith("/"))
.Where(reservedUrlTrimmed => reservedUrlTrimmed.IsNullOrWhiteSpace() == false))
{
newReservedList.Add(reservedUrlTrimmed);
}
foreach (var reservedPathTrimmed in _reservedPathsCache
.Split(new[] {","}, StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Trim().ToLowerInvariant())
.Where(x => x.IsNullOrWhiteSpace() == false)
.Select(reservedPath => IOHelper.ResolveUrl(reservedPath).Trim().EnsureStartsWith("/").EnsureEndsWith("/"))
.Where(reservedPathTrimmed => reservedPathTrimmed.IsNullOrWhiteSpace() == false))
{
newReservedList.Add(reservedPathTrimmed);
}
// use the new list from now on
_reservedList = newReservedList;
}
}
}
//The url should be cleaned up before checking:
// * If it doesn't contain an '.' in the path then we assume it is a path based URL, if that is the case we should add an trailing '/' because all of our reservedPaths use a trailing '/'
// * We shouldn't be comparing the query at all
var pathPart = url.Split(new[] {'?'}, StringSplitOptions.RemoveEmptyEntries)[0].ToLowerInvariant();
if (pathPart.Contains(".") == false)
{
pathPart = pathPart.EnsureEndsWith('/');
}
// return true if url starts with an element of the reserved list
return _reservedList.Any(x => pathPart.InvariantStartsWith(x));
}
/// <summary>
/// Determines whether the current request is reserved based on the route table and
/// whether the specified URL is reserved or is inside a reserved path.
/// </summary>
/// <param name="globalSettings"></param>
/// <param name="url"></param>
/// <param name="httpContext"></param>
/// <param name="routes">The route collection to lookup the request in</param>
/// <returns></returns>
internal static bool IsReservedPathOrUrl(this IGlobalSettings globalSettings, string url, HttpContextBase httpContext, RouteCollection routes)
{
if (httpContext == null) throw new ArgumentNullException(nameof(httpContext));
if (routes == null) throw new ArgumentNullException(nameof(routes));
//check if the current request matches a route, if so then it is reserved.
//TODO: This value should be cached! Else this is doing double routing in MVC every request!
var route = routes.GetRouteData(httpContext);
if (route != null)
return true;
//continue with the standard ignore routine
return globalSettings.IsReservedPathOrUrl(url);
}
}
}

74
src/Umbraco.Tests/Configurations/GlobalSettingsTests.cs
View File

@@ -1,6 +1,4 @@
using System.Web.Mvc;
using System.Web.Routing;
using Moq;
using Moq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Composing;
@@ -10,6 +8,7 @@ using Umbraco.Tests.TestHelpers;
namespace Umbraco.Tests.Configurations
{
[TestFixture]
public class GlobalSettingsTests : BaseWebTest
{
@@ -47,73 +46,18 @@ namespace Umbraco.Tests.Configurations
[TestCase("~/some-wacky/nestedPath", "/MyVirtualDir/NestedVDir/", "some-wacky-nestedpath")]
public void Umbraco_Mvc_Area(string path, string rootPath, string outcome)
{
var globalSettingsMock = Mock.Get(Factory.GetInstance<IGlobalSettings>()); //this will modify the IGlobalSettings instance stored in the container
globalSettingsMock.Setup(x => x.Path).Returns(IOHelper.ResolveUrl(path));
var globalSettings = SettingsForTests.GenerateMockGlobalSettings();
var globalSettingsMock = Mock.Get(globalSettings);
globalSettingsMock.Setup(x => x.Path).Returns(() => IOHelper.ResolveUrl(path));
SystemDirectories.Root = rootPath;
Assert.AreEqual(outcome, Current.Configs.Global().GetUmbracoMvcArea());
Assert.AreEqual(outcome, globalSettings.GetUmbracoMvcAreaNoCache());
}
[TestCase("/umbraco/editContent.aspx")]
[TestCase("/install/default.aspx")]
[TestCase("/install/")]
[TestCase("/install")]
[TestCase("/install/?installStep=asdf")]
[TestCase("/install/test.aspx")]
public void Is_Reserved_Path_Or_Url(string url)
{
var globalSettings = TestObjects.GetGlobalSettings();
Assert.IsTrue(globalSettings.IsReservedPathOrUrl(url));
}
[TestCase("/base/somebasehandler")]
[TestCase("/")]
[TestCase("/home.aspx")]
[TestCase("/umbraco-test")]
[TestCase("/install-test")]
[TestCase("/install.aspx")]
public void Is_Not_Reserved_Path_Or_Url(string url)
{
var globalSettings = TestObjects.GetGlobalSettings();
Assert.IsFalse(globalSettings.IsReservedPathOrUrl(url));
}
[TestCase("/Do/Not/match", false)]
[TestCase("/Umbraco/RenderMvcs", false)]
[TestCase("/Umbraco/RenderMvc", true)]
[TestCase("/Umbraco/RenderMvc/Index", true)]
[TestCase("/Umbraco/RenderMvc/Index/1234", true)]
[TestCase("/Umbraco/RenderMvc/Index/1234/9876", false)]
[TestCase("/api", true)]
[TestCase("/api/WebApiTest", true)]
[TestCase("/api/WebApiTest/1234", true)]
[TestCase("/api/WebApiTest/Index/1234", false)]
public void Is_Reserved_By_Route(string url, bool shouldMatch)
{
//reset the app config, we only want to test routes not the hard coded paths
var globalSettingsMock = Mock.Get(Factory.GetInstance<IGlobalSettings>()); //this will modify the IGlobalSettings instance stored in the container
globalSettingsMock.Setup(x => x.ReservedPaths).Returns("");
globalSettingsMock.Setup(x => x.ReservedUrls).Returns("");
var routes = new RouteCollection();
routes.MapRoute(
"Umbraco_default",
"Umbraco/RenderMvc/{action}/{id}",
new { controller = "RenderMvc", action = "Index", id = UrlParameter.Optional });
routes.MapRoute(
"WebAPI",
"api/{controller}/{id}",
new { controller = "WebApiTestController", action = "Index", id = UrlParameter.Optional });
var context = new FakeHttpContextFactory(url);
Assert.AreEqual(
shouldMatch,
globalSettingsMock.Object.IsReservedPathOrUrl(url, context.HttpContext, routes));
}
}
}

4
src/Umbraco.Tests/Packaging/PackageInstallationTest.cs
View File

@@ -11,6 +11,7 @@ using Umbraco.Core.Models;
using Umbraco.Core.Models.Packaging;
using Umbraco.Core.Packaging;
using Umbraco.Core.PropertyEditors;
using Umbraco.Core.Scoping;
using Umbraco.Tests.TestHelpers;
using Umbraco.Tests.Testing;
using File = System.IO.File;
@@ -45,7 +46,8 @@ namespace Umbraco.Tests.Packaging
Logger, ServiceContext.FileService, ServiceContext.MacroService, ServiceContext.LocalizationService,
ServiceContext.DataTypeService, ServiceContext.EntityService,
ServiceContext.ContentTypeService, ServiceContext.ContentService,
Factory.GetInstance<PropertyEditorCollection>());
Factory.GetInstance<PropertyEditorCollection>(),
Factory.GetInstance<IScopeProvider>());
private IPackageInstallation PackageInstallation => new PackageInstallation(
PackageDataInstallation,

80
src/Umbraco.Tests/Routing/RoutableDocumentFilterTests.cs Normal file
View File

@@ -0,0 +1,80 @@
using System.Web.Mvc;
using System.Web.Routing;
using Moq;
using NUnit.Framework;
using Umbraco.Core;
using Umbraco.Core.Configuration;
using Umbraco.Tests.TestHelpers;
using Umbraco.Web;
namespace Umbraco.Tests.Routing
{
[TestFixture]
public class RoutableDocumentFilterTests : BaseWebTest
{
[TestCase("/umbraco/editContent.aspx")]
[TestCase("/install/default.aspx")]
[TestCase("/install/")]
[TestCase("/install")]
[TestCase("/install/?installStep=asdf")]
[TestCase("/install/test.aspx")]
public void Is_Reserved_Path_Or_Url(string url)
{
var globalSettings = TestObjects.GetGlobalSettings();
var routableDocFilter = new RoutableDocumentFilter(globalSettings);
Assert.IsTrue(routableDocFilter.IsReservedPathOrUrl(url));
}
[TestCase("/base/somebasehandler")]
[TestCase("/")]
[TestCase("/home.aspx")]
[TestCase("/umbraco-test")]
[TestCase("/install-test")]
[TestCase("/install.aspx")]
public void Is_Not_Reserved_Path_Or_Url(string url)
{
var globalSettings = TestObjects.GetGlobalSettings();
var routableDocFilter = new RoutableDocumentFilter(globalSettings);
Assert.IsFalse(routableDocFilter.IsReservedPathOrUrl(url));
}
[TestCase("/Do/Not/match", false)]
[TestCase("/Umbraco/RenderMvcs", false)]
[TestCase("/Umbraco/RenderMvc", true)]
[TestCase("/Umbraco/RenderMvc/Index", true)]
[TestCase("/Umbraco/RenderMvc/Index/1234", true)]
[TestCase("/Umbraco/RenderMvc/Index/1234/9876", false)]
[TestCase("/api", true)]
[TestCase("/api/WebApiTest", true)]
[TestCase("/api/WebApiTest/1234", true)]
[TestCase("/api/WebApiTest/Index/1234", false)]
public void Is_Reserved_By_Route(string url, bool shouldMatch)
{
//reset the app config, we only want to test routes not the hard coded paths
var globalSettingsMock = Mock.Get(Factory.GetInstance<IGlobalSettings>()); //this will modify the IGlobalSettings instance stored in the container
globalSettingsMock.Setup(x => x.ReservedPaths).Returns("");
globalSettingsMock.Setup(x => x.ReservedUrls).Returns("");
var routableDocFilter = new RoutableDocumentFilter(globalSettingsMock.Object);
var routes = new RouteCollection();
routes.MapRoute(
"Umbraco_default",
"Umbraco/RenderMvc/{action}/{id}",
new { controller = "RenderMvc", action = "Index", id = UrlParameter.Optional });
routes.MapRoute(
"WebAPI",
"api/{controller}/{id}",
new { controller = "WebApiTestController", action = "Index", id = UrlParameter.Optional });
var context = new FakeHttpContextFactory(url);
Assert.AreEqual(
shouldMatch,
routableDocFilter.IsReservedPathOrUrl(url, context.HttpContext, routes));
}
}
}

13
src/Umbraco.Tests/Routing/UmbracoModuleTests.cs
View File

@@ -38,15 +38,16 @@ namespace Umbraco.Tests.Routing
_module = new UmbracoInjectedModule
(
globalSettings,
Mock.Of<IUmbracoContextAccessor>(),
Factory.GetInstance<IPublishedSnapshotService>(),
Factory.GetInstance<IUserService>(),
new UrlProviderCollection(new IUrlProvider[0]),
runtime,
logger,
null, // FIXME: PublishedRouter complexities...
Mock.Of<IVariationContextAccessor>(),
Mock.Of<IUmbracoContextFactory>()
Mock.Of<IUmbracoContextFactory>(),
Mock.Of<IPublishedModelFactory>(),
new Umbraco.Web.Cache.BackgroundPublishedSnapshotNotifier(
Factory.GetInstance<IPublishedModelFactory>(),
Factory.GetInstance<IPublishedSnapshotService>(),
Logger),
new RoutableDocumentFilter(globalSettings)
);
runtime.Level = RuntimeLevel.Run;

2
src/Umbraco.Tests/TestHelpers/TestObjects.cs
View File

@@ -178,7 +178,7 @@ namespace Umbraco.Tests.TestHelpers
new PackagesRepository(contentService.Value, contentTypeService.Value, dataTypeService.Value, fileService.Value, macroService.Value, localizationService.Value,
new EntityXmlSerializer(contentService.Value, mediaService.Value, dataTypeService.Value, userService.Value, localizationService.Value, contentTypeService.Value, urlSegmentProviders), logger, "installedPackages.config"),
new PackageInstallation(
new PackageDataInstallation(logger, fileService.Value, macroService.Value, localizationService.Value, dataTypeService.Value, entityService.Value, contentTypeService.Value, contentService.Value, propertyEditorCollection),
new PackageDataInstallation(logger, fileService.Value, macroService.Value, localizationService.Value, dataTypeService.Value, entityService.Value, contentTypeService.Value, contentService.Value, propertyEditorCollection, scopeProvider),
new PackageFileInstallation(compiledPackageXmlParser, new ProfilingLogger(logger, new TestProfiler())),
compiledPackageXmlParser, Mock.Of<IPackageActionRunner>(),
new DirectoryInfo(IOHelper.GetRootDirectorySafe())));

3
src/Umbraco.Tests/TestHelpers/TestWithDatabaseBase.cs
View File

@@ -31,6 +31,7 @@ using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.Persistence.Repositories;
using Umbraco.Tests.LegacyXmlPublishedCache;
using Umbraco.Tests.Testing.Objects.Accessors;
using Umbraco.Web.Cache;
namespace Umbraco.Tests.TestHelpers
{
@@ -90,6 +91,8 @@ namespace Umbraco.Tests.TestHelpers
factory.ResetForTests();
return factory;
});
Composition.RegisterUnique<BackgroundPublishedSnapshotNotifier>();
}
[OneTimeTearDown]

3
src/Umbraco.Tests/Umbraco.Tests.csproj
View File

@@ -120,6 +120,7 @@
<Compile Include="Composing\CompositionTests.cs" />
<Compile Include="Composing\LightInjectValidation.cs" />
<Compile Include="Composing\ContainerConformingTests.cs" />
<Compile Include="Configurations\GlobalSettingsTests.cs" />
<Compile Include="CoreThings\CallContextTests.cs" />
<Compile Include="Components\ComponentTests.cs" />
<Compile Include="CoreThings\EnumExtensionsTests.cs" />
@@ -145,6 +146,7 @@
<Compile Include="PublishedContent\SolidPublishedSnapshot.cs" />
<Compile Include="PublishedContent\NuCacheTests.cs" />
<Compile Include="Routing\MediaUrlProviderTests.cs" />
<Compile Include="Routing\RoutableDocumentFilterTests.cs" />
<Compile Include="Runtimes\StandaloneTests.cs" />
<Compile Include="Routing\GetContentUrlsTests.cs" />
<Compile Include="Services\AmbiguousEventTests.cs" />
@@ -455,7 +457,6 @@
<Compile Include="Cache\DistributedCache\DistributedCacheTests.cs" />
<Compile Include="TestHelpers\TestWithDatabaseBase.cs" />
<Compile Include="TestHelpers\SettingsForTests.cs" />
<Compile Include="Configurations\GlobalSettingsTests.cs" />
<Compile Include="Routing\ContentFinderByAliasTests.cs" />
<Compile Include="Routing\ContentFinderByIdTests.cs" />
<Compile Include="Routing\ContentFinderByPageIdQueryTests.cs" />

120
src/Umbraco.Web/Cache/BackgroundPublishedSnapshotNotifier.cs Normal file
View File

@@ -0,0 +1,120 @@
using System.Threading;
using System.Threading.Tasks;
using Umbraco.Core;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.Logging;
using Umbraco.Web.PublishedCache;
using Umbraco.Web.Scheduling;
namespace Umbraco.Web.Cache
{
/// <summary>
/// Used to notify the <see cref="IPublishedSnapshotService"/> of changes using a background thread
/// </summary>
/// <remarks>
/// When in Pure Live mode, the models need to be rebuilt before the IPublishedSnapshotService is notified which can result in performance penalties so
/// this performs these actions on a background thread so the user isn't waiting for the rebuilding to occur on a UI thread.
/// When using this, even when not in Pure Live mode, it still means that the cache is notified on a background thread.
/// In order to wait for the processing to complete, this class has a Wait() method which will block until the processing is finished.
/// </remarks>
public sealed class BackgroundPublishedSnapshotNotifier
{
private readonly IPublishedModelFactory _publishedModelFactory;
private readonly IPublishedSnapshotService _publishedSnapshotService;
private readonly BackgroundTaskRunner<IBackgroundTask> _runner;
/// <summary>
/// Constructor
/// </summary>
/// <param name="publishedModelFactory"></param>
/// <param name="publishedSnapshotService"></param>
/// <param name="logger"></param>
public BackgroundPublishedSnapshotNotifier(IPublishedModelFactory publishedModelFactory, IPublishedSnapshotService publishedSnapshotService, ILogger logger)
{
_publishedModelFactory = publishedModelFactory;
_publishedSnapshotService = publishedSnapshotService;
_runner = new BackgroundTaskRunner<IBackgroundTask>("RebuildModelsAndCache", logger);
}
/// <summary>
/// Blocks until the background operation is completed
/// </summary>
/// <returns>Returns true if waiting was necessary</returns>
public bool Wait()
{
var running = _runner.IsRunning;
_runner.StoppedAwaitable.GetAwaiter().GetResult(); //TODO: do we need a try/catch?
return running;
}
/// <summary>
/// Notify the <see cref="IPublishedSnapshotService"/> of content type changes
/// </summary>
/// <param name="payloads"></param>
public void NotifyWithSafeLiveFactory(ContentTypeCacheRefresher.JsonPayload[] payloads)
{
_runner.TryAdd(new RebuildModelsAndCacheTask(payloads, _publishedModelFactory, _publishedSnapshotService));
}
/// <summary>
/// Notify the <see cref="IPublishedSnapshotService"/> of data type changes
/// </summary>
/// <param name="payloads"></param>
public void NotifyWithSafeLiveFactory(DataTypeCacheRefresher.JsonPayload[] payloads)
{
_runner.TryAdd(new RebuildModelsAndCacheTask(payloads, _publishedModelFactory, _publishedSnapshotService));
}
/// <summary>
/// A simple background task that notifies the <see cref="IPublishedSnapshotService"/> of changes
/// </summary>
private class RebuildModelsAndCacheTask : IBackgroundTask
{
private readonly DataTypeCacheRefresher.JsonPayload[] _dataTypePayloads;
private readonly ContentTypeCacheRefresher.JsonPayload[] _contentTypePayloads;
private readonly IPublishedModelFactory _publishedModelFactory;
private readonly IPublishedSnapshotService _publishedSnapshotService;
private RebuildModelsAndCacheTask(IPublishedModelFactory publishedModelFactory, IPublishedSnapshotService publishedSnapshotService)
{
_publishedModelFactory = publishedModelFactory;
_publishedSnapshotService = publishedSnapshotService;
}
public RebuildModelsAndCacheTask(DataTypeCacheRefresher.JsonPayload[] payloads, IPublishedModelFactory publishedModelFactory, IPublishedSnapshotService publishedSnapshotService)
: this(publishedModelFactory, publishedSnapshotService)
{
_dataTypePayloads = payloads;
}
public RebuildModelsAndCacheTask(ContentTypeCacheRefresher.JsonPayload[] payloads, IPublishedModelFactory publishedModelFactory, IPublishedSnapshotService publishedSnapshotService)
: this(publishedModelFactory, publishedSnapshotService)
{
_contentTypePayloads = payloads;
}
public void Run()
{
// we have to refresh models before we notify the published snapshot
// service of changes, else factories may try to rebuild models while
// we are using the database to load content into caches
_publishedModelFactory.WithSafeLiveFactory(() =>
{
if (_dataTypePayloads != null)
_publishedSnapshotService.Notify(_dataTypePayloads);
if (_contentTypePayloads != null)
_publishedSnapshotService.Notify(_contentTypePayloads);
});
}
public Task RunAsync(CancellationToken token) => throw new System.NotImplementedException();
public bool IsAsync => false;
public void Dispose()
{
}
}
}
}

7
src/Umbraco.Web/Cache/ContentTypeCacheRefresher.cs
View File

@@ -1,12 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Umbraco.Core;
using Umbraco.Core.Cache;
using Umbraco.Core.Models;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.Persistence.Repositories;
using Umbraco.Core.Persistence.Repositories.Implement;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Web.PublishedCache;
@@ -86,10 +84,7 @@ namespace Umbraco.Web.Cache
// don't try to be clever - refresh all
MemberCacheRefresher.RefreshMemberTypes(AppCaches);
// we have to refresh models before we notify the published snapshot
// service of changes, else factories may try to rebuild models while
// we are using the database to load content into caches
// refresh the models and cache
_publishedModelFactory.WithSafeLiveFactoryReset(() =>
_publishedSnapshotService.Notify(payloads));

4
src/Umbraco.Web/Cache/DataTypeCacheRefresher.cs
View File

@@ -62,9 +62,7 @@ namespace Umbraco.Web.Cache
TagsValueConverter.ClearCaches();
SliderValueConverter.ClearCaches();
// we have to refresh models before we notify the published snapshot
// service of changes, else factories may try to rebuild models while
// we are using the database to load content into caches
// refresh the models and cache
_publishedModelFactory.WithSafeLiveFactoryReset(() =>
_publishedSnapshotService.Notify(payloads));

4
src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs
View File

@@ -1,12 +1,8 @@
using System;
using System.Threading;
using Umbraco.Core;
using Umbraco.Core.Compose;
using Umbraco.Core.Composing;
using Umbraco.Core.Configuration;
using Umbraco.Core.Logging;
using Umbraco.Core.Persistence;
using Umbraco.Core.Scoping;
using Umbraco.Core.Services;
using Umbraco.Core.Services.Changes;
using Umbraco.Core.Sync;

2
src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs
View File

@@ -160,7 +160,7 @@ namespace Umbraco.Web.PublishedCache.NuCache
_domainStore = new SnapDictionary<int, Domain>();
publishedModelFactory.WithSafeLiveFactory(LoadCachesOnStartup);
LoadCachesOnStartup();
Guid GetUid(ContentStore store, int id) => store.LiveSnapshot.Get(id)?.Uid ?? default;
int GetId(ContentStore store, Guid uid) => store.LiveSnapshot.Get(uid)?.Id ?? default;

207
src/Umbraco.Web/RoutableDocumentFilter.cs Normal file
View File

@@ -0,0 +1,207 @@
using System;
using System.IO;
using System.Web;
using System.Web.Routing;
using Umbraco.Core;
using Umbraco.Core.Logging;
using Umbraco.Core.Configuration;
using System.Threading;
using System.Collections.Generic;
using System.Linq;
using Umbraco.Core.IO;
using System.Collections.Concurrent;
using Umbraco.Core.Collections;
namespace Umbraco.Web
{
/// <summary>
/// Utility class used to check if the current request is for a front-end request
/// </summary>
/// <remarks>
/// There are various checks to determine if this is a front-end request such as checking if the request is part of any reserved paths or existing MVC routes.
/// </remarks>
public sealed class RoutableDocumentFilter
{
public RoutableDocumentFilter(IGlobalSettings globalSettings)
{
_globalSettings = globalSettings;
}
private static readonly ConcurrentDictionary<string, bool> RouteChecks = new ConcurrentDictionary<string, bool>();
private readonly IGlobalSettings _globalSettings;
private object _locker = new object();
private bool _isInit = false;
private int? _routeCount;
private HashSet<string> _reservedList;
/// <summary>
/// Checks if the request is a document request (i.e. one that the module should handle)
/// </summary>
/// <param name="httpContext"></param>
/// <param name="uri"></param>
/// <returns></returns>
public bool IsDocumentRequest(HttpContextBase httpContext, Uri uri)
{
var maybeDoc = true;
var lpath = uri.AbsolutePath.ToLowerInvariant();
// handle directory-urls used for asmx
// TODO: legacy - what's the point really?
var asmxPos = lpath.IndexOf(".asmx/", StringComparison.OrdinalIgnoreCase);
if (asmxPos >= 0)
{
// use uri.AbsolutePath, not path, 'cos path has been lowercased
httpContext.RewritePath(uri.AbsolutePath.Substring(0, asmxPos + 5), // filePath
uri.AbsolutePath.Substring(asmxPos + 5), // pathInfo
uri.Query.TrimStart('?'));
maybeDoc = false;
}
// a document request should be
// /foo/bar/nil
// /foo/bar/nil/
// /foo/bar/nil.aspx
// where /foo is not a reserved path
// if the path contains an extension that is not .aspx
// then it cannot be a document request
var extension = Path.GetExtension(lpath);
if (maybeDoc && extension.IsNullOrWhiteSpace() == false && extension != ".aspx")
maybeDoc = false;
// at that point, either we have no extension, or it is .aspx
// if the path is reserved then it cannot be a document request
if (maybeDoc && IsReservedPathOrUrl(lpath, httpContext, RouteTable.Routes))
maybeDoc = false;
//NOTE: No need to warn, plus if we do we should log the document, as this message doesn't really tell us anything :)
//if (!maybeDoc)
//{
// Logger.Warn<UmbracoModule>("Not a document");
//}
return maybeDoc;
}
/// <summary>
/// Determines whether the specified URL is reserved or is inside a reserved path.
/// </summary>
/// <param name="url">The URL to check.</param>
/// <returns>
/// <c>true</c> if the specified URL is reserved; otherwise, <c>false</c>.
/// </returns>
internal bool IsReservedPathOrUrl(string url)
{
LazyInitializer.EnsureInitialized(ref _reservedList, ref _isInit, ref _locker, () =>
{
// store references to strings to determine changes
var reservedPathsCache = _globalSettings.ReservedPaths;
var reservedUrlsCache = _globalSettings.ReservedUrls;
// add URLs and paths to a new list
var newReservedList = new HashSet<string>();
foreach (var reservedUrlTrimmed in reservedUrlsCache
.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Trim().ToLowerInvariant())
.Where(x => x.IsNullOrWhiteSpace() == false)
.Select(reservedUrl => IOHelper.ResolveUrl(reservedUrl).Trim().EnsureStartsWith("/"))
.Where(reservedUrlTrimmed => reservedUrlTrimmed.IsNullOrWhiteSpace() == false))
{
newReservedList.Add(reservedUrlTrimmed);
}
foreach (var reservedPathTrimmed in NormalizePaths(reservedPathsCache.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries)))
{
newReservedList.Add(reservedPathTrimmed);
}
foreach (var reservedPathTrimmed in NormalizePaths(ReservedPaths))
{
newReservedList.Add(reservedPathTrimmed);
}
// use the new list from now on
return newReservedList;
});
//The url should be cleaned up before checking:
// * If it doesn't contain an '.' in the path then we assume it is a path based URL, if that is the case we should add an trailing '/' because all of our reservedPaths use a trailing '/'
// * We shouldn't be comparing the query at all
var pathPart = url.Split(new[] { '?' }, StringSplitOptions.RemoveEmptyEntries)[0].ToLowerInvariant();
if (pathPart.Contains(".") == false)
{
pathPart = pathPart.EnsureEndsWith('/');
}
// return true if url starts with an element of the reserved list
return _reservedList.Any(x => pathPart.InvariantStartsWith(x));
}
private IEnumerable<string> NormalizePaths(IEnumerable<string> paths)
{
return paths
.Select(x => x.Trim().ToLowerInvariant())
.Where(x => x.IsNullOrWhiteSpace() == false)
.Select(reservedPath => IOHelper.ResolveUrl(reservedPath).Trim().EnsureStartsWith("/").EnsureEndsWith("/"))
.Where(reservedPathTrimmed => reservedPathTrimmed.IsNullOrWhiteSpace() == false);
}
/// <summary>
/// Determines whether the current request is reserved based on the route table and
/// whether the specified URL is reserved or is inside a reserved path.
/// </summary>
/// <param name="url"></param>
/// <param name="httpContext"></param>
/// <param name="routes">The route collection to lookup the request in</param>
/// <returns></returns>
internal bool IsReservedPathOrUrl(string url, HttpContextBase httpContext, RouteCollection routes)
{
if (httpContext == null) throw new ArgumentNullException(nameof(httpContext));
if (routes == null) throw new ArgumentNullException(nameof(routes));
//This is some rudimentary code to check if the route table has changed at runtime, we're basically just keeping a count
//of the routes. This isn't fail safe but there's no way to monitor changes to the route table. Else we need to create a hash
//of all routes and then recompare but that will be annoying to do on each request and then we might as well just do the whole MVC
//route on each request like we were doing before instead of caching the result of GetRouteData.
var changed = false;
using (routes.GetReadLock())
{
if (!_routeCount.HasValue || _routeCount.Value != routes.Count)
{
//the counts are not set or have changed, need to reset
changed = true;
}
}
if (changed)
{
using (routes.GetWriteLock())
{
_routeCount = routes.Count;
//try clearing each entry
foreach(var r in RouteChecks.Keys.ToList())
RouteChecks.TryRemove(r, out _);
}
}
var absPath = httpContext?.Request?.Url.AbsolutePath;
if (absPath.IsNullOrWhiteSpace())
return false;
//check if the current request matches a route, if so then it is reserved.
var hasRoute = RouteChecks.GetOrAdd(absPath, x => routes.GetRouteData(httpContext) != null);
if (hasRoute)
return true;
//continue with the standard ignore routine
return IsReservedPathOrUrl(url);
}
/// <summary>
/// This is used internally to track any registered callback paths for Identity providers. If the request path matches
/// any of the registered paths, then the module will let the request keep executing
/// </summary>
internal static readonly ConcurrentHashSet<string> ReservedPaths = new ConcurrentHashSet<string>();
}
}

5
src/Umbraco.Web/Runtime/WebInitialComposer.cs
View File

@@ -73,7 +73,7 @@ namespace Umbraco.Web.Runtime
// register accessors for cultures
composition.RegisterUnique<IDefaultCultureAccessor, DefaultCultureAccessor>();
composition.RegisterUnique<IVariationContextAccessor, HybridVariationContextAccessor>();
// register the http context and umbraco context accessors
// we *should* use the HttpContextUmbracoContextAccessor, however there are cases when
// we have no http context, eg when booting Umbraco or in background threads, so instead
@@ -125,6 +125,9 @@ namespace Umbraco.Web.Runtime
// register distributed cache
composition.RegisterUnique(f => new DistributedCache());
composition.RegisterUnique<BackgroundPublishedSnapshotNotifier>();
composition.RegisterUnique<RoutableDocumentFilter>();
// replace some services
composition.RegisterUnique<IEventMessagesFactory, DefaultEventMessagesFactory>();
composition.RegisterUnique<IEventMessagesAccessor, HybridEventMessagesAccessor>();

4
src/Umbraco.Web/Security/AuthenticationOptionsExtensions.cs
View File

@@ -101,7 +101,7 @@ namespace Umbraco.Web.Security
var path = (PathString) prop.GetValue(options);
if (path.HasValue)
{
UmbracoModule.ReservedPaths.TryAdd(path.ToString());
RoutableDocumentFilter.ReservedPaths.TryAdd(path.ToString());
}
}
}
@@ -112,7 +112,7 @@ namespace Umbraco.Web.Security
}
else
{
UmbracoModule.ReservedPaths.TryAdd(callbackPath);
RoutableDocumentFilter.ReservedPaths.TryAdd(callbackPath);
}
}
}

2
src/Umbraco.Web/Umbraco.Web.csproj
View File

@@ -115,6 +115,7 @@
<Compile Include="AppBuilderExtensions.cs" />
<Compile Include="AreaRegistrationContextExtensions.cs" />
<Compile Include="AspNetHttpContextAccessor.cs" />
<Compile Include="Cache\BackgroundPublishedSnapshotNotifier.cs" />
<Compile Include="Cache\DistributedCacheBinder.cs" />
<Compile Include="Cache\DistributedCacheBinderComposer.cs" />
<Compile Include="Cache\DistributedCacheBinder_Handlers.cs" />
@@ -230,6 +231,7 @@
<Compile Include="PublishedCache\NuCache\Snap\GenObj.cs" />
<Compile Include="PublishedCache\NuCache\Snap\GenRef.cs" />
<Compile Include="PublishedCache\NuCache\Snap\LinkedNode.cs" />
<Compile Include="RoutableDocumentFilter.cs" />
<Compile Include="Routing\DefaultMediaUrlProvider.cs" />
<Compile Include="Routing\IMediaUrlProvider.cs" />
<Compile Include="Routing\IPublishedRouter.cs" />

22
src/Umbraco.Web/UmbracoContext.cs
View File

@@ -1,16 +1,21 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Web;
using System.Web.Routing;
using Umbraco.Core;
using Umbraco.Core.Configuration;
using Umbraco.Core.Configuration.UmbracoSettings;
using Umbraco.Core.Events;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Web;
using Umbraco.Web.PublishedCache;
using Umbraco.Web.Routing;
using Umbraco.Web.Security;
namespace Umbraco.Web
{
/// <summary>
/// Class that encapsulates Umbraco information of a specific HTTP request
/// </summary>
@@ -286,6 +291,23 @@ namespace Umbraco.Web
_previewing = _previewToken.IsNullOrWhiteSpace() == false;
}
private bool? _isDocumentRequest;
/// <summary>
/// Checks if the request is a document request (i.e. one that the module should handle)
/// </summary>
/// <param name="httpContext"></param>
/// <param name="uri"></param>
/// <returns></returns>
internal bool IsDocumentRequest(RoutableDocumentFilter docLookup)
{
if (_isDocumentRequest.HasValue)
return _isDocumentRequest.Value;
_isDocumentRequest = docLookup.IsDocumentRequest(HttpContext, OriginalRequestUrl);
return _isDocumentRequest.Value;
}
// say we render a macro or RTE in a give 'preview' mode that might not be the 'current' one,
// then due to the way it all works at the moment, the 'current' published snapshot need to be in the proper
// default 'preview' mode - somehow we have to force it. and that could be recursive.

124
src/Umbraco.Web/UmbracoInjectedModule.cs
View File

@@ -2,7 +2,6 @@
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Web;
using System.Web.Routing;
using Umbraco.Core;
@@ -10,14 +9,11 @@ using Umbraco.Core.Configuration;
using Umbraco.Core.IO;
using Umbraco.Core.Logging;
using Umbraco.Web.Routing;
using Umbraco.Web.Security;
using Umbraco.Core.Exceptions;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.Core.Persistence.FaultHandling;
using Umbraco.Core.Security;
using Umbraco.Core.Services;
using Umbraco.Web.Composing;
using Umbraco.Web.PublishedCache;
using Umbraco.Web.Cache;
using Umbraco.Core.Models.PublishedContent;
namespace Umbraco.Web
{
@@ -39,40 +35,32 @@ namespace Umbraco.Web
public class UmbracoInjectedModule : IHttpModule
{
private readonly IGlobalSettings _globalSettings;
private readonly IUmbracoContextAccessor _umbracoContextAccessor;
private readonly IPublishedSnapshotService _publishedSnapshotService;
private readonly IUserService _userService;
private readonly UrlProviderCollection _urlProviders;
private readonly IRuntimeState _runtime;
private readonly ILogger _logger;
private readonly IPublishedRouter _publishedRouter;
private readonly IVariationContextAccessor _variationContextAccessor;
private readonly IUmbracoContextFactory _umbracoContextFactory;
private readonly IPublishedModelFactory _publishedModelFactory;
private readonly BackgroundPublishedSnapshotNotifier _backgroundNotifier;
private readonly RoutableDocumentFilter _routableDocumentLookup;
public UmbracoInjectedModule(
IGlobalSettings globalSettings,
IUmbracoContextAccessor umbracoContextAccessor,
IPublishedSnapshotService publishedSnapshotService,
IUserService userService,
UrlProviderCollection urlProviders,
IRuntimeState runtime,
ILogger logger,
IPublishedRouter publishedRouter,
IVariationContextAccessor variationContextAccessor,
IUmbracoContextFactory umbracoContextFactory)
IUmbracoContextFactory umbracoContextFactory,
IPublishedModelFactory publishedModelFactory,
BackgroundPublishedSnapshotNotifier backgroundNotifier,
RoutableDocumentFilter routableDocumentLookup)
{
_combinedRouteCollection = new Lazy<RouteCollection>(CreateRouteCollection);
_globalSettings = globalSettings;
_umbracoContextAccessor = umbracoContextAccessor;
_publishedSnapshotService = publishedSnapshotService;
_userService = userService;
_urlProviders = urlProviders;
_runtime = runtime;
_logger = logger;
_publishedRouter = publishedRouter;
_variationContextAccessor = variationContextAccessor;
_umbracoContextFactory = umbracoContextFactory;
_publishedModelFactory = publishedModelFactory;
_backgroundNotifier = backgroundNotifier;
_routableDocumentLookup = routableDocumentLookup;
}
#region HttpModule event handlers
@@ -182,18 +170,18 @@ namespace Umbraco.Web
var reason = EnsureRoutableOutcome.IsRoutable;
// ensure this is a document request
if (EnsureDocumentRequest(httpContext, uri) == false)
if (!context.IsDocumentRequest(_routableDocumentLookup))
{
reason = EnsureRoutableOutcome.NotDocumentRequest;
}
// ensure the runtime is in the proper state
// and deal with needed redirects, etc
else if (EnsureRuntime(httpContext, uri) == false)
else if (!EnsureRuntime(httpContext, uri))
{
reason = EnsureRoutableOutcome.NotReady;
}
// ensure Umbraco has documents to serve
else if (EnsureHasContent(context, httpContext) == false)
else if (!EnsureHasContent(context, httpContext))
{
reason = EnsureRoutableOutcome.NoContent;
}
@@ -201,55 +189,7 @@ namespace Umbraco.Web
return Attempt.If(reason == EnsureRoutableOutcome.IsRoutable, reason);
}
/// <summary>
/// Ensures that the request is a document request (i.e. one that the module should handle)
/// </summary>
/// <param name="httpContext"></param>
/// <param name="uri"></param>
/// <returns></returns>
private bool EnsureDocumentRequest(HttpContextBase httpContext, Uri uri)
{
var maybeDoc = true;
var lpath = uri.AbsolutePath.ToLowerInvariant();
// handle directory-urls used for asmx
// TODO: legacy - what's the point really?
var asmxPos = lpath.IndexOf(".asmx/", StringComparison.OrdinalIgnoreCase);
if (asmxPos >= 0)
{
// use uri.AbsolutePath, not path, 'cos path has been lowercased
httpContext.RewritePath(uri.AbsolutePath.Substring(0, asmxPos + 5), // filePath
uri.AbsolutePath.Substring(asmxPos + 5), // pathInfo
uri.Query.TrimStart('?'));
maybeDoc = false;
}
// a document request should be
// /foo/bar/nil
// /foo/bar/nil/
// /foo/bar/nil.aspx
// where /foo is not a reserved path
// if the path contains an extension that is not .aspx
// then it cannot be a document request
var extension = Path.GetExtension(lpath);
if (maybeDoc && extension.IsNullOrWhiteSpace() == false && extension != ".aspx")
maybeDoc = false;
// at that point, either we have no extension, or it is .aspx
// if the path is reserved then it cannot be a document request
if (maybeDoc && _globalSettings.IsReservedPathOrUrl(lpath, httpContext, _combinedRouteCollection.Value))
maybeDoc = false;
//NOTE: No need to warn, plus if we do we should log the document, as this message doesn't really tell us anything :)
//if (!maybeDoc)
//{
// Logger.Warn<UmbracoModule>("Not a document");
//}
return maybeDoc;
}
private bool EnsureRuntime(HttpContextBase httpContext, Uri uri)
{
@@ -505,36 +445,6 @@ namespace Umbraco.Web
#endregion
/// <summary>
/// This is used to be passed into the GlobalSettings.IsReservedPathOrUrl and will include some 'fake' routes
/// used to determine if a path is reserved.
/// </summary>
/// <remarks>
/// This is basically used to reserve paths dynamically
/// </remarks>
private readonly Lazy<RouteCollection> _combinedRouteCollection;
private RouteCollection CreateRouteCollection()
{
var routes = new RouteCollection();
foreach (var route in RouteTable.Routes)
routes.Add(route);
foreach (var reservedPath in UmbracoModule.ReservedPaths)
{
try
{
routes.Add("_umbreserved_" + reservedPath.ReplaceNonAlphanumericChars(""),
new Route(reservedPath.TrimStart('/'), new StopRoutingHandler()));
}
catch (Exception ex)
{
_logger.Error<UmbracoModule>("Could not add reserved path route", ex);
}
}
return routes;
}
}
}

5
src/Umbraco.Web/UmbracoModule.cs
View File

@@ -103,10 +103,5 @@ namespace Umbraco.Web
return end;
}
/// <summary>
/// This is used internally to track any registered callback paths for Identity providers. If the request path matches
/// any of the registered paths, then the module will let the request keep executing
/// </summary>
internal static readonly ConcurrentHashSet<string> ReservedPaths = new ConcurrentHashSet<string>();
}
}

6
src/umbraco.sln
View File

@@ -1,6 +1,6 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.27004.2005
# Visual Studio Version 16
VisualStudioVersion = 16.0.29209.152
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Umbraco.Web.UI", "Umbraco.Web.UI\Umbraco.Web.UI.csproj", "{4C4C194C-B5E4-4991-8F87-4373E24CC19F}"
EndProject
@@ -123,6 +123,7 @@ Global
{31785BC3-256C-4613-B2F5-A1B0BDDED8C1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{31785BC3-256C-4613-B2F5-A1B0BDDED8C1}.Release|Any CPU.Build.0 = Release|Any CPU
{5D3B8245-ADA6-453F-A008-50ED04BFE770}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5D3B8245-ADA6-453F-A008-50ED04BFE770}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5D3B8245-ADA6-453F-A008-50ED04BFE770}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5D3B8245-ADA6-453F-A008-50ED04BFE770}.Release|Any CPU.Build.0 = Release|Any CPU
{07FBC26B-2927-4A22-8D96-D644C667FECC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
@@ -130,6 +131,7 @@ Global
{07FBC26B-2927-4A22-8D96-D644C667FECC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{07FBC26B-2927-4A22-8D96-D644C667FECC}.Release|Any CPU.Build.0 = Release|Any CPU
{3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3A33ADC9-C6C0-4DB1-A613-A9AF0210DF3D}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection