From 94774113f6427cc40dbfa0cabd92a9bff6fb3d20 Mon Sep 17 00:00:00 2001 From: Mole Date: Fri, 7 Oct 2022 10:42:32 +0200 Subject: [PATCH] V11: Fix InMemoryAuto modelsbuilder mode (#13107) * POC of a solution that works * Add razor reference manager * Ensure the compilation options are correct * Move InMemory classes to its own namespace These are all internal, so it should be fine. * Throw proper exceptions when compilation fails * Add CheckSumValidator * Clear the ViewCompiler cache when models changed This means we no longer need the RefreshingRazorViewEngine \o/ * Remove unused constructor injection * Make UmbracoAssemblyLoadContext non internal * Add WIP * Clear the RazorViewEngine cache when generating new models This uses reflection, which isn't super nice, however, the alternative is to clone'n'own the entire RazorViewEngine, which is arguably worse * Fix circular dependency * Remove ModelsChanged event This is no longer necessary * Fix precompiled views path We need to normalize these paths to ensure they matches with the keys in _precompiledViews * Clean * Fix content tests * Add logging * Update the comment in UmbracoBuilderDependencyInjectionExtensions to reflect changes * Remove RefreshingRazorViewEngine as its no longer needed * Remove unused ViewEngine hack from DI * Fix langversion This is required since dotnet 7 is still in preview * Add modelsbuilder tests * Add more tests * fixed comment Co-authored-by: Bjarke Berg --- .../Logging/RegisteredReloadableLogger.cs | 3 +- ...acoBuilderDependencyInjectionExtensions.cs | 53 +- .../InMemoryAuto/ChecksumValidator.cs | 125 +++++ .../CollectibleRuntimeViewCompiler.cs | 483 ++++++++++++++++++ .../CompilationExceptionFactory.cs | 137 +++++ .../CompilationOptionsProvider.cs | 212 ++++++++ .../InMemoryAssemblyLoadContextManager.cs | 66 +++ .../InMemoryModelFactory.cs | 78 +-- .../RuntimeCompilationCacheBuster.cs | 53 ++ .../UmbracoAssemblyLoadContext.cs | 2 +- .../UmbracoCompilationException.cs | 8 + .../UmbracoRazorReferenceManager.cs | 89 ++++ .../UmbracoViewCompilerProvider.cs | 88 ++++ .../RefreshingRazorViewEngine.cs | 194 ------- .../ModelsBuilder/modelsbuilder.spec.ts | 280 ++++++++++ 15 files changed, 1599 insertions(+), 272 deletions(-) create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/ChecksumValidator.cs create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CollectibleRuntimeViewCompiler.cs create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationExceptionFactory.cs create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationOptionsProvider.cs create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/InMemoryAssemblyLoadContextManager.cs rename src/Umbraco.Web.Common/ModelsBuilder/{ => InMemoryAuto}/InMemoryModelFactory.cs (90%) create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/RuntimeCompilationCacheBuster.cs rename src/Umbraco.Web.Common/ModelsBuilder/{ => InMemoryAuto}/UmbracoAssemblyLoadContext.cs (91%) create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoCompilationException.cs create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoRazorReferenceManager.cs create mode 100644 src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoViewCompilerProvider.cs delete mode 100644 src/Umbraco.Web.Common/ModelsBuilder/RefreshingRazorViewEngine.cs create mode 100644 tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/ModelsBuilder/modelsbuilder.spec.ts diff --git a/src/Umbraco.Web.Common/Logging/RegisteredReloadableLogger.cs b/src/Umbraco.Web.Common/Logging/RegisteredReloadableLogger.cs index c146198097..63a34ca73f 100644 --- a/src/Umbraco.Web.Common/Logging/RegisteredReloadableLogger.cs +++ b/src/Umbraco.Web.Common/Logging/RegisteredReloadableLogger.cs @@ -1,12 +1,13 @@ using Serilog; using Serilog.Extensions.Hosting; +using Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; namespace Umbraco.Cms.Web.Common.Logging; /// /// HACK: /// Ensures freeze is only called a single time even when resolving a logger from the snapshot container -/// built for . +/// built for . /// internal class RegisteredReloadableLogger { diff --git a/src/Umbraco.Web.Common/ModelsBuilder/DependencyInjection/UmbracoBuilderDependencyInjectionExtensions.cs b/src/Umbraco.Web.Common/ModelsBuilder/DependencyInjection/UmbracoBuilderDependencyInjectionExtensions.cs index 95ae91d7b7..7e79d8787d 100644 --- a/src/Umbraco.Web.Common/ModelsBuilder/DependencyInjection/UmbracoBuilderDependencyInjectionExtensions.cs +++ b/src/Umbraco.Web.Common/ModelsBuilder/DependencyInjection/UmbracoBuilderDependencyInjectionExtensions.cs @@ -1,7 +1,12 @@ +using Microsoft.AspNetCore.Mvc.ApplicationParts; using Microsoft.AspNetCore.Mvc.Razor; +using Microsoft.AspNetCore.Mvc.Razor.Compilation; +using Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation; +using Microsoft.AspNetCore.Razor.Language; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Umbraco.Cms.Core.Configuration; using Umbraco.Cms.Core.Configuration.Models; @@ -11,6 +16,7 @@ using Umbraco.Cms.Core.Notifications; using Umbraco.Cms.Infrastructure.ModelsBuilder; using Umbraco.Cms.Infrastructure.ModelsBuilder.Building; using Umbraco.Cms.Web.Common.ModelsBuilder; +using Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; /* * OVERVIEW: @@ -58,16 +64,27 @@ using Umbraco.Cms.Web.Common.ModelsBuilder; * requires re-resolving the original services from a pre-built DI container. In effect this re-creates these * services from scratch which means there is no caches. * - * ... Option C works, we will use that but need to verify how this affects memory since ideally the old services will be GC'd. + * ... Option C worked, however after a breaking change from dotnet, we cannot go with this options any longer. + * The reason for this is that when the default RuntimeViewCompiler loads in the assembly using Assembly.Load, + * This will not work for us since this loads the compiled views into the default AssemblyLoadContext, + * and our compiled models are loaded in the collectible UmbracoAssemblyLoadContext, and as per the breaking change + * you're no longer allowed reference a collectible load context from a non-collectible one + * That is the non-collectible compiled views are not allowed to reference the collectible InMemoryAuto models. + * https://learn.microsoft.com/en-us/dotnet/core/compatibility/core-libraries/7.0/collectible-assemblies * - * Option C, how its done: - * - Before we add our custom razor services to the container, we make a copy of the services collection which is the snapshot of registered services - * with razor defaults before ours are added. - * - We replace the default implementation of IRazorViewEngine with our own. This is a wrapping service that wraps the default RazorViewEngine instance. - * The ctor for this service takes in a Factory method to re-construct the default RazorViewEngine and all of it's dependency graph. - * - When the models change, the Factory is invoked and the default razor services are all re-created, thus clearing their caches and the newly - * created instance is wrapped. The RazorViewEngine is the only service that needs to be replaced and wrapped for this to work because it's dependency - * graph includes all of the above mentioned services, all the way up to the RazorProjectEngine and it's LazyMetadataReferenceFeature. + * So what do we do then? + * We've had to go with option a unfortunately, and we've cloned the above classes + * There has had to be some modifications to the ViewCompiler (CollectibleRuntimeViewCompiler) + * First off we've added a new class InMemoryAssemblyLoadContextManager, the role of this class is to ensure that + * no one will take a reference to the assembly load context (you cannot unload an assembly load context if there's any references to it). + * This means that both the InMemoryAutoFactory and the ViewCompiler uses the LoadContextManager to load their assemblies. + * This serves another purpose being that it keeps track of the location of the models assembly. + * This means that we no longer use the RazorReferencesManager to resolve that specific dependency, but instead add and explicit dependency to the models assembly. + * + * With this our assembly load context issue is solved, however the caching issue still persists now that we no longer use the RefreshingRazorViewEngine + * To clear these caches another class the RuntimeCompilationCacheBuster has been introduced, + * this keeps a reference to the CollectibleRuntimeViewCompiler and the RazorViewEngine and is injected into the InMemoryModelsFactory to clear the caches when rebuilding modes. + * In order to avoid having to copy all the RazorViewEngine code the cache buster uses reflection to call the internal ClearCache method of the RazorViewEngine. */ namespace Umbraco.Extensions; @@ -129,19 +146,11 @@ public static class UmbracoBuilderDependencyInjectionExtensions // copy the current collection, we need to use this later to rebuild a container // to re-create the razor compiler provider - var initialCollection = new ServiceCollection { builder.Services }; - - // Replace the default with our custom engine - builder.Services.AddSingleton( - s => new RefreshingRazorViewEngine( - () => - { - // re-create the original container so that a brand new IRazorPageActivator - // is produced, if we don't re-create the container then it will just return the same instance. - ServiceProvider recreatedServices = initialCollection.BuildServiceProvider(); - return recreatedServices.GetRequiredService(); - }, - s.GetRequiredService())); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/ChecksumValidator.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/ChecksumValidator.cs new file mode 100644 index 0000000000..654766d9b1 --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/ChecksumValidator.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using Microsoft.AspNetCore.Razor.Hosting; +using Microsoft.AspNetCore.Razor.Language; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +/* + * This is a clone of Microsofts implementation. + * https://github.com/dotnet/aspnetcore/blob/main/src/Mvc/Mvc.Razor.RuntimeCompilation/src/ChecksumValidator.cs + * The purpose of this class is to check if compiled views has changed and needs to be recompiled. + */ +internal static class ChecksumValidator +{ + public static bool IsRecompilationSupported(RazorCompiledItem item) + { + if (item == null) + { + throw new ArgumentNullException(nameof(item)); + } + + // A Razor item only supports recompilation if its primary source file has a checksum. + // + // Other files (view imports) may or may not have existed at the time of compilation, + // so we may not have checksums for them. + var checksums = item.GetChecksumMetadata(); + return checksums.Any(c => string.Equals(item.Identifier, c.Identifier, StringComparison.OrdinalIgnoreCase)); + } + + // Validates that we can use an existing precompiled view by comparing checksums with files on + // disk. + public static bool IsItemValid(RazorProjectFileSystem fileSystem, RazorCompiledItem item) + { + if (fileSystem == null) + { + throw new ArgumentNullException(nameof(fileSystem)); + } + + if (item == null) + { + throw new ArgumentNullException(nameof(item)); + } + + var checksums = item.GetChecksumMetadata(); + + // The checksum that matches 'Item.Identity' in this list is significant. That represents the main file. + // + // We don't really care about the validation unless the main file exists. This is because we expect + // most sites to have some _ViewImports in common location. That means that in the case you're + // using views from a 3rd party library, you'll always have **some** conflicts. + // + // The presence of the main file with the same content is a very strong signal that you're in a + // development scenario. + var primaryChecksum = checksums + .FirstOrDefault(c => string.Equals(item.Identifier, c.Identifier, StringComparison.OrdinalIgnoreCase)); + if (primaryChecksum == null) + { + // No primary checksum, assume valid. + return true; + } + + var projectItem = fileSystem.GetItem(primaryChecksum.Identifier, fileKind: null); + if (!projectItem.Exists) + { + // Main file doesn't exist - assume valid. + return true; + } + + var sourceDocument = RazorSourceDocument.ReadFrom(projectItem); + if (!string.Equals(sourceDocument.GetChecksumAlgorithm(), primaryChecksum.ChecksumAlgorithm) || + !ChecksumsEqual(primaryChecksum.Checksum, sourceDocument.GetChecksum())) + { + // Main file exists, but checksums not equal. + return false; + } + + for (var i = 0; i < checksums.Count; i++) + { + var checksum = checksums[i]; + if (string.Equals(item.Identifier, checksum.Identifier, StringComparison.OrdinalIgnoreCase)) + { + // Ignore primary checksum on this pass. + continue; + } + + var importItem = fileSystem.GetItem(checksum.Identifier, fileKind: null); + if (!importItem.Exists) + { + // Import file doesn't exist - assume invalid. + return false; + } + + sourceDocument = RazorSourceDocument.ReadFrom(importItem); + if (!string.Equals(sourceDocument.GetChecksumAlgorithm(), checksum.ChecksumAlgorithm) || + !ChecksumsEqual(checksum.Checksum, sourceDocument.GetChecksum())) + { + // Import file exists, but checksums not equal. + return false; + } + } + + return true; + } + + private static bool ChecksumsEqual(string checksum, byte[] bytes) + { + if (bytes.Length * 2 != checksum.Length) + { + return false; + } + + for (var i = 0; i < bytes.Length; i++) + { + var text = bytes[i].ToString("x2", CultureInfo.InvariantCulture); + if (checksum[i * 2] != text[0] || checksum[i * 2 + 1] != text[1]) + { + return false; + } + } + + return true; + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CollectibleRuntimeViewCompiler.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CollectibleRuntimeViewCompiler.cs new file mode 100644 index 0000000000..742e7ffc29 --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CollectibleRuntimeViewCompiler.cs @@ -0,0 +1,483 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Reflection; +using System.Text; +using Microsoft.AspNetCore.Mvc.Razor.Compilation; +using Microsoft.AspNetCore.Razor.Hosting; +using Microsoft.AspNetCore.Razor.Language; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Emit; +using Microsoft.CodeAnalysis.Text; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +internal class CollectibleRuntimeViewCompiler : IViewCompiler +{ + private readonly object _cacheLock = new object(); + private readonly Dictionary _precompiledViews; + private readonly ConcurrentDictionary _normalizedPathCache; + private readonly IFileProvider _fileProvider; + private readonly RazorProjectEngine _projectEngine; + private IMemoryCache _cache; + private readonly ILogger _logger; + private readonly UmbracoRazorReferenceManager _referenceManager; + private readonly CompilationOptionsProvider _compilationOptionsProvider; + private readonly InMemoryAssemblyLoadContextManager _loadContextManager; + + public CollectibleRuntimeViewCompiler( + IFileProvider fileProvider, + RazorProjectEngine projectEngine, + IList precompiledViews, + ILogger logger, + UmbracoRazorReferenceManager referenceManager, + CompilationOptionsProvider compilationOptionsProvider, + InMemoryAssemblyLoadContextManager loadContextManager) + { + if (fileProvider == null) + { + throw new ArgumentNullException(nameof(fileProvider)); + } + + if (projectEngine == null) + { + throw new ArgumentNullException(nameof(projectEngine)); + } + + if (precompiledViews == null) + { + throw new ArgumentNullException(nameof(precompiledViews)); + } + + if (logger == null) + { + throw new ArgumentNullException(nameof(logger)); + } + + _fileProvider = fileProvider; + _projectEngine = projectEngine; + _logger = logger; + _referenceManager = referenceManager; + _compilationOptionsProvider = compilationOptionsProvider; + _loadContextManager = loadContextManager; + + _normalizedPathCache = new ConcurrentDictionary(StringComparer.Ordinal); + + // This is our L0 cache, and is a durable store. Views migrate into the cache as they are requested + // from either the set of known precompiled views, or by being compiled. + _cache = new MemoryCache(new MemoryCacheOptions()); + + // We need to validate that the all of the precompiled views are unique by path (case-insensitive). + // We do this because there's no good way to canonicalize paths on windows, and it will create + // problems when deploying to linux. Rather than deal with these issues, we just don't support + // views that differ only by case. + _precompiledViews = new Dictionary( + precompiledViews.Count, + StringComparer.OrdinalIgnoreCase); + + foreach (var precompiledView in precompiledViews) + { + _logger.LogDebug("Initializing Razor view compiler with compiled view: '{ViewName}'", precompiledView.RelativePath); + if (!_precompiledViews.ContainsKey(precompiledView.RelativePath)) + { + // View ordering has precedence semantics, a view with a higher precedence was + // already added to the list. + _precompiledViews.Add(precompiledView.RelativePath, precompiledView); + } + } + + if (_precompiledViews.Count == 0) + { + _logger.LogDebug("Initializing Razor view compiler with no compiled views"); + } + } + + internal void ClearCache() + { + // I'm pretty sure this is not necessary, since it should be an atomic operation, + // but let's make sure that we don't end up resolving any views while clearing the cache. + lock (_cacheLock) + { + _cache = new MemoryCache(new MemoryCacheOptions()); + } + } + + public Task CompileAsync(string relativePath) + { + if (relativePath == null) + { + throw new ArgumentNullException(nameof(relativePath)); + } + + // Attempt to lookup the cache entry using the passed in path. This will succeed if the path is already + // normalized and a cache entry exists. + if (_cache.TryGetValue>(relativePath, out var cachedResult) && cachedResult is not null) + { + return cachedResult; + } + + var normalizedPath = GetNormalizedPath(relativePath); + if (_cache.TryGetValue(normalizedPath, out cachedResult) && cachedResult is not null) + { + return cachedResult; + } + + // Entry does not exist. Attempt to create one. + cachedResult = OnCacheMiss(normalizedPath); + return cachedResult; + } + + private Task OnCacheMiss(string normalizedPath) + { + ViewCompilerWorkItem item; + TaskCompletionSource taskSource; + MemoryCacheEntryOptions cacheEntryOptions; + + // Safe races cannot be allowed when compiling Razor pages. To ensure only one compilation request succeeds + // per file, we'll lock the creation of a cache entry. Creating the cache entry should be very quick. The + // actual work for compiling files happens outside the critical section. + lock (_cacheLock) + { + // Double-checked locking to handle a possible race. + if (_cache.TryGetValue>(normalizedPath, out var result) && result is not null) + { + return result; + } + + if (_precompiledViews.TryGetValue(normalizedPath, out var precompiledView)) + { + _logger.LogTrace("Located compiled view for view at path '{Path}'", normalizedPath); + item = CreatePrecompiledWorkItem(normalizedPath, precompiledView); + } + else + { + item = CreateRuntimeCompilationWorkItem(normalizedPath); + } + + // At this point, we've decided what to do - but we should create the cache entry and + // release the lock first. + cacheEntryOptions = new MemoryCacheEntryOptions(); + + Debug.Assert(item.ExpirationTokens != null); + for (var i = 0; i < item.ExpirationTokens.Count; i++) + { + cacheEntryOptions.ExpirationTokens.Add(item.ExpirationTokens[i]); + } + + taskSource = new TaskCompletionSource(creationOptions: TaskCreationOptions.RunContinuationsAsynchronously); + if (item.SupportsCompilation) + { + // We'll compile in just a sec, be patient. + } + else + { + // If we can't compile, we should have already created the descriptor + Debug.Assert(item.Descriptor != null); + taskSource.SetResult(item.Descriptor); + } + + _cache.Set(normalizedPath, taskSource.Task, cacheEntryOptions); + } + + // Now the lock has been released so we can do more expensive processing. + if (item.SupportsCompilation) + { + Debug.Assert(taskSource != null); + + if (item.Descriptor?.Item != null && + ChecksumValidator.IsItemValid(_projectEngine.FileSystem, item.Descriptor.Item) ) + { + // If the item has checksums to validate, we should also have a precompiled view. + Debug.Assert(item.Descriptor != null); + + taskSource.SetResult(item.Descriptor); + return taskSource.Task; + } + + _logger.LogTrace("Invalidating compiled view at path '{Path}' with a file since the checksum did not match", item.NormalizedPath); + try + { + var descriptor = CompileAndEmit(normalizedPath); + descriptor.ExpirationTokens = cacheEntryOptions.ExpirationTokens; + taskSource.SetResult(descriptor); + } + catch (Exception ex) + { + taskSource.SetException(ex); + } + } + + return taskSource.Task; + } + + private ViewCompilerWorkItem CreatePrecompiledWorkItem(string normalizedPath, CompiledViewDescriptor precompiledView) + { + // We have a precompiled view - but we're not sure that we can use it yet. + // + // We need to determine first if we have enough information to 'recompile' this view. If that's the case + // we'll create change tokens for all of the files. + // + // Then we'll attempt to validate if any of those files have different content than the original sources + // based on checksums. + if (precompiledView.Item == null || !ChecksumValidator.IsRecompilationSupported(precompiledView.Item)) + { + return new ViewCompilerWorkItem() + { + // If we don't have a checksum for the primary source file we can't recompile. + SupportsCompilation = false, + + ExpirationTokens = Array.Empty(), // Never expire because we can't recompile. + Descriptor = precompiledView, // This will be used as-is. + }; + } + + var item = new ViewCompilerWorkItem() + { + SupportsCompilation = true, + + Descriptor = precompiledView, // This might be used, if the checksums match. + + // Used to validate and recompile + NormalizedPath = normalizedPath, + + ExpirationTokens = GetExpirationTokens(precompiledView), + }; + + // We also need to create a new descriptor, because the original one doesn't have expiration tokens on + // it. These will be used by the view location cache, which is like an L1 cache for views (this class is + // the L2 cache). + item.Descriptor = new CompiledViewDescriptor() + { + ExpirationTokens = item.ExpirationTokens, + Item = precompiledView.Item, + RelativePath = precompiledView.RelativePath, + }; + + return item; + } + + private ViewCompilerWorkItem CreateRuntimeCompilationWorkItem(string normalizedPath) + { + IList expirationTokens = new List + { + _fileProvider.Watch(normalizedPath), + }; + + var projectItem = _projectEngine.FileSystem.GetItem(normalizedPath, fileKind: null); + if (!projectItem.Exists) + { + _logger.LogTrace("Could not find a file for view at path '{Path}'", normalizedPath); + // If the file doesn't exist, we can't do compilation right now - we still want to cache + // the fact that we tried. This will allow us to re-trigger compilation if the view file + // is added. + return new ViewCompilerWorkItem() + { + // We don't have enough information to compile + SupportsCompilation = false, + + Descriptor = new CompiledViewDescriptor() + { + RelativePath = normalizedPath, + ExpirationTokens = expirationTokens, + }, + + // We can try again if the file gets created. + ExpirationTokens = expirationTokens, + }; + } + + _logger.LogTrace("Found file at path '{Path}'", normalizedPath); + + GetChangeTokensFromImports(expirationTokens, projectItem); + + return new ViewCompilerWorkItem() + { + SupportsCompilation = true, + + NormalizedPath = normalizedPath, + ExpirationTokens = expirationTokens, + }; + } + + private IList GetExpirationTokens(CompiledViewDescriptor precompiledView) + { + var checksums = precompiledView.Item.GetChecksumMetadata(); + var expirationTokens = new List(checksums.Count); + + for (var i = 0; i < checksums.Count; i++) + { + // We rely on Razor to provide the right set of checksums. Trust the compiler, it has to do a good job, + // so it probably will. + expirationTokens.Add(_fileProvider.Watch(checksums[i].Identifier)); + } + + return expirationTokens; + } + + private void GetChangeTokensFromImports(IList expirationTokens, RazorProjectItem projectItem) + { + // OK this means we can do compilation. For now let's just identify the other files we need to watch + // so we can create the cache entry. Compilation will happen after we release the lock. + var importFeature = _projectEngine.ProjectFeatures.OfType().ToArray(); + foreach (var feature in importFeature) + { + foreach (var file in feature.GetImports(projectItem)) + { + if (file.FilePath != null) + { + expirationTokens.Add(_fileProvider.Watch(file.FilePath)); + } + } + } + } + + protected virtual CompiledViewDescriptor CompileAndEmit(string relativePath) + { + var projectItem = _projectEngine.FileSystem.GetItem(relativePath, fileKind: null); + var codeDocument = _projectEngine.Process(projectItem); + var cSharpDocument = codeDocument.GetCSharpDocument(); + + if (cSharpDocument.Diagnostics.Count > 0) + { + throw CompilationExceptionFactory.Create( + codeDocument, + cSharpDocument.Diagnostics); + } + + var assembly = CompileAndEmit(codeDocument, cSharpDocument.GeneratedCode); + + // Anything we compile from source will use Razor 2.1 and so should have the new metadata. + var loader = new RazorCompiledItemLoader(); + var item = loader.LoadItems(assembly).Single(); + return new CompiledViewDescriptor(item); + } + + internal Assembly CompileAndEmit(RazorCodeDocument codeDocument, string generatedCode) + { + var startTimestamp = _logger.IsEnabled(LogLevel.Debug) ? Stopwatch.GetTimestamp() : 0; + + var assemblyName = Path.GetRandomFileName(); + CSharpCompilation compilation = CreateCompilation(generatedCode, assemblyName); + + EmitOptions emitOptions = _compilationOptionsProvider.EmitOptions; + var emitPdbFile = _compilationOptionsProvider.EmitPdb && emitOptions.DebugInformationFormat != DebugInformationFormat.Embedded; + + + using (var assemblyStream = new MemoryStream()) + using (MemoryStream? pdbStream = emitPdbFile ? new MemoryStream() : null) + { + var result = compilation.Emit( + assemblyStream, + pdbStream, + options: _compilationOptionsProvider.EmitOptions); + + if (!result.Success) + { + throw CompilationExceptionFactory.Create( + codeDocument, + generatedCode, + assemblyName, + result.Diagnostics); + } + + assemblyStream.Seek(0, SeekOrigin.Begin); + pdbStream?.Seek(0, SeekOrigin.Begin); + + Assembly assembly = _loadContextManager.LoadCollectibleAssemblyFromStream(assemblyStream, pdbStream); + + return assembly; + } + } + + private CSharpCompilation CreateCompilation(string compilationContent, string assemblyName) + { + IReadOnlyList refs = _referenceManager.CompilationReferences; + // We'll add the reference to the InMemory assembly directly, this means we don't have to hack around with assembly parts. + if (_loadContextManager.ModelsAssemblyLocation is null) + { + throw new InvalidOperationException("No InMemory assembly available, cannot compile views"); + } + + PortableExecutableReference inMemoryAutoReference = MetadataReference.CreateFromFile(_loadContextManager.ModelsAssemblyLocation); + + + var sourceText = SourceText.From(compilationContent, Encoding.UTF8); + SyntaxTree syntaxTree = SyntaxFactory + .ParseSyntaxTree(sourceText, _compilationOptionsProvider.ParseOptions) + .WithFilePath(assemblyName); + + return CSharpCompilation + .Create(assemblyName) + .AddSyntaxTrees(syntaxTree) + .AddReferences(refs) + .AddReferences(inMemoryAutoReference) + .WithOptions(_compilationOptionsProvider.CSharpCompilationOptions); + } + + private string GetNormalizedPath(string relativePath) + { + Debug.Assert(relativePath != null); + if (relativePath.Length == 0) + { + return relativePath; + } + + if (!_normalizedPathCache.TryGetValue(relativePath, out var normalizedPath)) + { + normalizedPath = NormalizePath(relativePath); + _normalizedPathCache[relativePath] = normalizedPath; + } + + return normalizedPath; + } + + // Taken from: https://github.com/dotnet/aspnetcore/blob/a450cb69b5e4549f5515cdb057a68771f56cefd7/src/Mvc/Mvc.Razor/src/ViewPath.cs + // This normalizes the relative path to the view, ensuring that it matches with what we have as keys in _precompiledViews + private string NormalizePath(string path) + { + var addLeadingSlash = path[0] != '\\' && path[0] != '/'; + var transformSlashes = path.IndexOf('\\') != -1; + + if (!addLeadingSlash && !transformSlashes) + { + return path; + } + + var length = path.Length; + if (addLeadingSlash) + { + length++; + } + + return string.Create(length, (path, addLeadingSlash), (span, tuple) => + { + var (pathValue, addLeadingSlashValue) = tuple; + var spanIndex = 0; + + if (addLeadingSlashValue) + { + span[spanIndex++] = '/'; + } + + foreach (var ch in pathValue) + { + span[spanIndex++] = ch == '\\' ? '/' : ch; + } + }); + } + + private sealed class ViewCompilerWorkItem + { + public bool SupportsCompilation { get; set; } = default!; + + public string NormalizedPath { get; set; } = default!; + + public IList ExpirationTokens { get; set; } = default!; + + public CompiledViewDescriptor Descriptor { get; set; } = default!; + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationExceptionFactory.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationExceptionFactory.cs new file mode 100644 index 0000000000..ddb79c4c3c --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationExceptionFactory.cs @@ -0,0 +1,137 @@ +using System.Globalization; +using Microsoft.AspNetCore.Diagnostics; +using Microsoft.AspNetCore.Razor.Language; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +/* + * This is a partial clone of the frameworks CompilationFailedExceptionFactory, a few things has been simplified to fit our needs. + * https://github.com/dotnet/aspnetcore/blob/main/src/Mvc/Mvc.Razor.RuntimeCompilation/src/CompilationFailedExceptionFactory.cs + */ +internal static class CompilationExceptionFactory +{ + public static UmbracoCompilationException Create( + RazorCodeDocument codeDocument, + IEnumerable diagnostics) + { + // If a SourceLocation does not specify a file path, assume it is produced from parsing the current file. + var messageGroups = diagnostics.GroupBy( + razorError => razorError.Span.FilePath ?? codeDocument.Source.FilePath, + StringComparer.Ordinal); + + var failures = new List(); + foreach (var group in messageGroups) + { + var filePath = group.Key; + var fileContent = ReadContent(codeDocument, filePath); + var compilationFailure = new CompilationFailure( + filePath, + fileContent, + compiledContent: string.Empty, + messages: group.Select(parserError => CreateDiagnosticMessage(parserError, filePath))); + failures.Add(compilationFailure); + } + + return new UmbracoCompilationException{CompilationFailures = failures}; + } + + public static UmbracoCompilationException Create( + RazorCodeDocument codeDocument, + string compilationContent, + string assemblyName, + IEnumerable diagnostics) + { + var diagnosticGroups = diagnostics + .Where(diagnostic => diagnostic.IsWarningAsError || diagnostic.Severity == DiagnosticSeverity.Error) + .GroupBy(diagnostic => GetFilePath(codeDocument, diagnostic), StringComparer.Ordinal); + + var failures = new List(); + foreach (var group in diagnosticGroups) + { + var sourceFilePath = group.Key; + string sourceFileContent; + if (string.Equals(assemblyName, sourceFilePath, StringComparison.Ordinal)) + { + // The error is in the generated code and does not have a mapping line pragma + sourceFileContent = compilationContent; + } + else + { + sourceFileContent = ReadContent(codeDocument, sourceFilePath!); + } + + var compilationFailure = new CompilationFailure( + sourceFilePath, + sourceFileContent, + compilationContent, + group.Select(GetDiagnosticMessage)); + + failures.Add(compilationFailure); + } + + return new UmbracoCompilationException{ CompilationFailures = failures}; + } + + private static string ReadContent(RazorCodeDocument codeDocument, string filePath) + { + RazorSourceDocument? sourceDocument; + if (string.IsNullOrEmpty(filePath) || string.Equals(codeDocument.Source.FilePath, filePath, StringComparison.Ordinal)) + { + sourceDocument = codeDocument.Source; + } + else + { + sourceDocument = codeDocument.Imports.FirstOrDefault(f => string.Equals(f.FilePath, filePath, StringComparison.Ordinal)); + } + + if (sourceDocument != null) + { + var contentChars = new char[sourceDocument.Length]; + sourceDocument.CopyTo(0, contentChars, 0, sourceDocument.Length); + return new string(contentChars); + } + + return string.Empty; + } + + private static DiagnosticMessage GetDiagnosticMessage(Diagnostic diagnostic) + { + var mappedLineSpan = diagnostic.Location.GetMappedLineSpan(); + return new DiagnosticMessage( + diagnostic.GetMessage(CultureInfo.CurrentCulture), + CSharpDiagnosticFormatter.Instance.Format(diagnostic, CultureInfo.CurrentCulture), + mappedLineSpan.Path, + mappedLineSpan.StartLinePosition.Line + 1, + mappedLineSpan.StartLinePosition.Character + 1, + mappedLineSpan.EndLinePosition.Line + 1, + mappedLineSpan.EndLinePosition.Character + 1); + } + + private static string GetFilePath(RazorCodeDocument codeDocument, Diagnostic diagnostic) + { + if (diagnostic.Location == Location.None) + { + return codeDocument.Source.FilePath; + } + + return diagnostic.Location.GetMappedLineSpan().Path; + } + + private static DiagnosticMessage CreateDiagnosticMessage( + RazorDiagnostic razorDiagnostic, + string filePath) + { + var sourceSpan = razorDiagnostic.Span; + var message = razorDiagnostic.GetMessage(CultureInfo.CurrentCulture); + return new DiagnosticMessage( + message: message, + formattedMessage: razorDiagnostic.ToString(), + filePath: filePath, + startLine: sourceSpan.LineIndex + 1, + startColumn: sourceSpan.CharacterIndex, + endLine: sourceSpan.LineIndex + 1, + endColumn: sourceSpan.CharacterIndex + sourceSpan.Length); + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationOptionsProvider.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationOptionsProvider.cs new file mode 100644 index 0000000000..4f2a421981 --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/CompilationOptionsProvider.cs @@ -0,0 +1,212 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using Microsoft.AspNetCore.Hosting; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Emit; +using Microsoft.Extensions.DependencyModel; +using Microsoft.Extensions.Hosting; +using DependencyContextCompilationOptions = Microsoft.Extensions.DependencyModel.CompilationOptions; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +/* + * This is a partial Clone'n'Own of microsofts CSharpCompiler, this is just the parts relevant for getting the CompilationOptions + * Essentially, what this does is that it looks at the compilation options of the Dotnet project and "copies" that + * https://github.com/dotnet/aspnetcore/blob/main/src/Mvc/Mvc.Razor.RuntimeCompilation/src/CSharpCompiler.cs + */ +internal class CompilationOptionsProvider +{ + private readonly IWebHostEnvironment _hostingEnvironment; + private CSharpParseOptions? _parseOptions; + private CSharpCompilationOptions? _compilationOptions; + private bool _emitPdb; + private EmitOptions? _emitOptions; + private bool _optionsInitialized; + + public CompilationOptionsProvider(IWebHostEnvironment hostingEnvironment) => + _hostingEnvironment = hostingEnvironment; + + public virtual CSharpParseOptions ParseOptions + { + get + { + EnsureOptions(); + return _parseOptions; + } + } + + public virtual CSharpCompilationOptions CSharpCompilationOptions + { + get + { + EnsureOptions(); + return _compilationOptions; + } + } + + public virtual bool EmitPdb + { + get + { + EnsureOptions(); + return _emitPdb; + } + } + + public virtual EmitOptions EmitOptions + { + get + { + EnsureOptions(); + return _emitOptions; + } + } + + [MemberNotNull(nameof(_emitOptions), nameof(_parseOptions), nameof(_compilationOptions))] + private void EnsureOptions() + { + if (!_optionsInitialized) + { + var dependencyContextOptions = GetDependencyContextCompilationOptions(); + _parseOptions = GetParseOptions(_hostingEnvironment, dependencyContextOptions); + _compilationOptions = GetCompilationOptions(_hostingEnvironment, dependencyContextOptions); + _emitOptions = GetEmitOptions(dependencyContextOptions); + + _optionsInitialized = true; + } + + Debug.Assert(_parseOptions is not null); + Debug.Assert(_compilationOptions is not null); + Debug.Assert(_emitOptions is not null); + } + + private DependencyContextCompilationOptions GetDependencyContextCompilationOptions() + { + if (!string.IsNullOrEmpty(_hostingEnvironment.ApplicationName)) + { + var applicationAssembly = Assembly.Load(new AssemblyName(_hostingEnvironment.ApplicationName)); + var dependencyContext = DependencyContext.Load(applicationAssembly); + if (dependencyContext?.CompilationOptions != null) + { + return dependencyContext.CompilationOptions; + } + } + + return DependencyContextCompilationOptions.Default; + } + + private EmitOptions GetEmitOptions(DependencyContextCompilationOptions dependencyContextOptions) + { + // Assume we're always producing pdbs unless DebugType = none + _emitPdb = true; + DebugInformationFormat debugInformationFormat; + if (string.IsNullOrEmpty(dependencyContextOptions.DebugType)) + { + debugInformationFormat = DebugInformationFormat.PortablePdb; + } + else + { + // Based on https://github.com/dotnet/roslyn/blob/1d28ff9ba248b332de3c84d23194a1d7bde07e4d/src/Compilers/CSharp/Portable/CommandLine/CSharpCommandLineParser.cs#L624-L640 + switch (dependencyContextOptions.DebugType.ToLowerInvariant()) + { + case "none": + // There isn't a way to represent none in DebugInformationFormat. + // We'll set EmitPdb to false and let callers handle it by setting a null pdb-stream. + _emitPdb = false; + return new EmitOptions(); + case "portable": + debugInformationFormat = DebugInformationFormat.PortablePdb; + break; + case "embedded": + // Roslyn does not expose enough public APIs to produce a binary with embedded pdbs. + // We'll produce PortablePdb instead to continue providing a reasonable user experience. + debugInformationFormat = DebugInformationFormat.PortablePdb; + break; + case "full": + case "pdbonly": + debugInformationFormat = DebugInformationFormat.PortablePdb; + break; + default: + throw new InvalidOperationException(); + } + } + + var emitOptions = new EmitOptions(debugInformationFormat: debugInformationFormat); + return emitOptions; + } + + private static CSharpCompilationOptions GetCompilationOptions( + IWebHostEnvironment hostingEnvironment, + DependencyContextCompilationOptions dependencyContextOptions) + { + var csharpCompilationOptions = new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary); + + // Disable 1702 until roslyn turns this off by default + csharpCompilationOptions = csharpCompilationOptions.WithSpecificDiagnosticOptions( + new Dictionary + { + {"CS1701", ReportDiagnostic.Suppress}, // Binding redirects + {"CS1702", ReportDiagnostic.Suppress}, + {"CS1705", ReportDiagnostic.Suppress} + }); + + if (dependencyContextOptions.AllowUnsafe.HasValue) + { + csharpCompilationOptions = csharpCompilationOptions.WithAllowUnsafe( + dependencyContextOptions.AllowUnsafe.Value); + } + + OptimizationLevel optimizationLevel; + if (dependencyContextOptions.Optimize.HasValue) + { + optimizationLevel = dependencyContextOptions.Optimize.Value ? + OptimizationLevel.Release : + OptimizationLevel.Debug; + } + else + { + optimizationLevel = hostingEnvironment.IsDevelopment() ? + OptimizationLevel.Debug : + OptimizationLevel.Release; + } + csharpCompilationOptions = csharpCompilationOptions.WithOptimizationLevel(optimizationLevel); + + if (dependencyContextOptions.WarningsAsErrors.HasValue) + { + var reportDiagnostic = dependencyContextOptions.WarningsAsErrors.Value ? + ReportDiagnostic.Error : + ReportDiagnostic.Default; + csharpCompilationOptions = csharpCompilationOptions.WithGeneralDiagnosticOption(reportDiagnostic); + } + + return csharpCompilationOptions; + } + + private static CSharpParseOptions GetParseOptions( + IWebHostEnvironment hostingEnvironment, + DependencyContextCompilationOptions dependencyContextOptions) + { + var configurationSymbol = hostingEnvironment.IsDevelopment() ? "DEBUG" : "RELEASE"; + var defines = dependencyContextOptions.Defines.Concat(new[] { configurationSymbol }).Where(define => define != null); + + var parseOptions = new CSharpParseOptions(preprocessorSymbols: (IEnumerable)defines); + + if (string.IsNullOrEmpty(dependencyContextOptions.LanguageVersion)) + { + // If the user does not specify a LanguageVersion, assume CSharp 8.0. This matches the language version Razor 3.0 targets by default. + parseOptions = parseOptions.WithLanguageVersion(LanguageVersion.CSharp8); + } + else if (LanguageVersionFacts.TryParse(dependencyContextOptions.LanguageVersion, out var languageVersion)) + { + parseOptions = parseOptions.WithLanguageVersion(languageVersion); + } + else + { + Debug.Fail($"LanguageVersion {languageVersion} specified in the deps file could not be parsed."); + } + + return parseOptions; + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/InMemoryAssemblyLoadContextManager.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/InMemoryAssemblyLoadContextManager.cs new file mode 100644 index 0000000000..15a280574d --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/InMemoryAssemblyLoadContextManager.cs @@ -0,0 +1,66 @@ +using System.Reflection; +using System.Runtime.Loader; +using Umbraco.Cms.Infrastructure.ModelsBuilder; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +internal class InMemoryAssemblyLoadContextManager +{ + private UmbracoAssemblyLoadContext? _currentAssemblyLoadContext; + + public InMemoryAssemblyLoadContextManager() => + AssemblyLoadContext.Default.Resolving += OnResolvingDefaultAssemblyLoadContext; + + private string? _modelsAssemblyLocation; + + public string? ModelsAssemblyLocation => _modelsAssemblyLocation; + + /// + /// Handle the event when a reference cannot be resolved from the default context and return our custom MB assembly reference if we have one + /// + /// + /// This is required because the razor engine will only try to load things from the default context, it doesn't know anything + /// about our context so we need to proxy. + /// + private Assembly? OnResolvingDefaultAssemblyLoadContext(AssemblyLoadContext assemblyLoadContext, AssemblyName assemblyName) + => assemblyName.Name == RoslynCompiler.GeneratedAssemblyName + ? _currentAssemblyLoadContext?.LoadFromAssemblyName(assemblyName) + : null; + + internal void RenewAssemblyLoadContext() + { + // If there's a current AssemblyLoadContext, unload it before creating a new one. + _currentAssemblyLoadContext?.Unload(); + + // We must create a new assembly load context + // as long as theres a reference to the assembly load context we can't delete the assembly it loaded + _currentAssemblyLoadContext = new UmbracoAssemblyLoadContext(); + } + + /// + /// Loads an assembly into the collectible assembly used by the factory + /// + /// + /// This is essentially just a wrapper around the , + /// because we don't want to allow other clases to take a reference on the AssemblyLoadContext + /// + /// The loaded assembly + public Assembly LoadCollectibleAssemblyFromStream(Stream assembly, Stream? assemblySymbols) + { + _currentAssemblyLoadContext ??= new UmbracoAssemblyLoadContext(); + return _currentAssemblyLoadContext.LoadFromStream(assembly, assemblySymbols); + } + + public Assembly LoadCollectibleAssemblyFromPath(string path) + { + _currentAssemblyLoadContext ??= new UmbracoAssemblyLoadContext(); + return _currentAssemblyLoadContext.LoadFromAssemblyPath(path); + } + + public Assembly LoadModelsAssembly(string path) + { + Assembly assembly = LoadCollectibleAssemblyFromPath(path); + _modelsAssemblyLocation = assembly.Location; + return assembly; + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryModelFactory.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/InMemoryModelFactory.cs similarity index 90% rename from src/Umbraco.Web.Common/ModelsBuilder/InMemoryModelFactory.cs rename to src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/InMemoryModelFactory.cs index d3b47f6fb8..12846f6bdd 100644 --- a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryModelFactory.cs +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/InMemoryModelFactory.cs @@ -4,7 +4,7 @@ using System.Reflection.Emit; using System.Runtime.Loader; using System.Text; using System.Text.RegularExpressions; -using Microsoft.AspNetCore.Mvc.ApplicationParts; +using Microsoft.AspNetCore.Mvc.Razor; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Umbraco.Cms.Core; @@ -18,7 +18,7 @@ using Umbraco.Cms.Infrastructure.ModelsBuilder.Building; using Umbraco.Extensions; using File = System.IO.File; -namespace Umbraco.Cms.Web.Common.ModelsBuilder +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto { internal class InMemoryModelFactory : IAutoPublishedModelFactory, IRegisteredObject, IDisposable { @@ -35,7 +35,8 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder private readonly IApplicationShutdownRegistry _hostingLifetime; private readonly ModelsGenerationError _errors; private readonly IPublishedValueFallback _publishedValueFallback; - private readonly ApplicationPartManager _applicationPartManager; + private readonly InMemoryAssemblyLoadContextManager _loadContextManager; + private readonly RuntimeCompilationCacheBuster _runtimeCompilationCacheBuster; private readonly Lazy _pureLiveDirectory = null!; private readonly int _debugLevel; private Infos _infos = new Infos { ModelInfos = null, ModelTypeMap = new Dictionary() }; @@ -44,7 +45,6 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder private int _ver; private int? _skipver; private RoslynCompiler? _roslynCompiler; - private UmbracoAssemblyLoadContext? _currentAssemblyLoadContext; private ModelsBuilderSettings _config; private bool _disposedValue; @@ -56,7 +56,8 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder IHostingEnvironment hostingEnvironment, IApplicationShutdownRegistry hostingLifetime, IPublishedValueFallback publishedValueFallback, - ApplicationPartManager applicationPartManager) + InMemoryAssemblyLoadContextManager loadContextManager, + RuntimeCompilationCacheBuster runtimeCompilationCacheBuster) { _umbracoServices = umbracoServices; _profilingLogger = profilingLogger; @@ -65,7 +66,8 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder _hostingEnvironment = hostingEnvironment; _hostingLifetime = hostingLifetime; _publishedValueFallback = publishedValueFallback; - _applicationPartManager = applicationPartManager; + _loadContextManager = loadContextManager; + _runtimeCompilationCacheBuster = runtimeCompilationCacheBuster; _errors = new ModelsGenerationError(config, _hostingEnvironment); _ver = 1; // zero is for when we had no version _skipver = -1; // nothing to skip @@ -93,12 +95,8 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder // get it here, this need to be fast _debugLevel = _config.DebugLevel; - - AssemblyLoadContext.Default.Resolving += OnResolvingDefaultAssemblyLoadContext; } - public event EventHandler? ModelsChanged; - /// /// Gets the currently loaded Live models assembly /// @@ -132,18 +130,6 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder /// public bool Enabled => _config.ModelsMode == ModelsMode.InMemoryAuto; - /// - /// Handle the event when a reference cannot be resolved from the default context and return our custom MB assembly reference if we have one - /// - /// - /// This is required because the razor engine will only try to load things from the default context, it doesn't know anything - /// about our context so we need to proxy. - /// - private Assembly? OnResolvingDefaultAssemblyLoadContext(AssemblyLoadContext assemblyLoadContext, AssemblyName assemblyName) - => assemblyName.Name == RoslynCompiler.GeneratedAssemblyName - ? _currentAssemblyLoadContext?.LoadFromAssemblyName(assemblyName) - : null; - public IPublishedElement CreateModel(IPublishedElement element) { // get models, rebuilding them if needed @@ -313,12 +299,20 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder CurrentModelsAssembly = assembly; - // Raise the model changing event. - // NOTE: That on first load, if there is content, this will execute before the razor view engine - // has loaded which means it hasn't yet bound to this event so there's no need to worry about if - // it will be eagerly re-generated unecessarily on first render. BUT we should be aware that if we - // change this to use the event aggregator that will no longer be the case. - ModelsChanged?.Invoke(this, new EventArgs()); + /* + * We used to use an event here, and a RefreshingRazorViewEngine to bust the caches, + * this worked by essentially completely recreating the entire ViewEngine/ViewCompiler every time we generate models. + * There was this note about first load: + * NOTE: That on first load, if there is content, this will execute before the razor view engine + * has loaded which means it hasn't yet bound to this event so there's no need to worry about if + * it will be eagerly re-generated unnecessarily on first render. BUT we should be aware that if we + * change this to use the event aggregator that will no longer be the case. + * + * Now we have our own ViewCompiler, and clear the caches more directly, however what the comment mentioned + * is not really a big problem since this will execute before the razor view engine has loaded, + * which means the cache will be empty already. + */ + _runtimeCompilationCacheBuster.BustCache(); IEnumerable types = assembly.ExportedTypes.Where(x => x.Inherits() || x.Inherits()); _infos = RegisterModels(types); @@ -364,22 +358,7 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder // This is NOT thread safe but it is only called from within a lock private Assembly ReloadAssembly(string pathToAssembly) { - // If there's a current AssemblyLoadContext, unload it before creating a new one. - if (!(_currentAssemblyLoadContext is null)) - { - _currentAssemblyLoadContext.Unload(); - - // we need to remove the current part too - ApplicationPart? currentPart = _applicationPartManager.ApplicationParts.FirstOrDefault(x => x.Name == RoslynCompiler.GeneratedAssemblyName); - if (currentPart != null) - { - _applicationPartManager.ApplicationParts.Remove(currentPart); - } - } - - // We must create a new assembly load context - // as long as theres a reference to the assembly load context we can't delete the assembly it loaded - _currentAssemblyLoadContext = new UmbracoAssemblyLoadContext(); + _loadContextManager.RenewAssemblyLoadContext(); // NOTE: We cannot use in-memory assemblies due to the way the razor engine works which must use // application parts in order to add references to it's own CSharpCompiler. @@ -393,16 +372,7 @@ namespace Umbraco.Cms.Web.Common.ModelsBuilder File.Copy(pathToAssembly, tempFile, true); // Load it in - Assembly assembly = _currentAssemblyLoadContext.LoadFromAssemblyPath(tempFile); - - // Add the assembly to the application parts - this is required because this is how - // the razor ReferenceManager resolves what to load, see - // https://github.com/dotnet/aspnetcore/blob/master/src/Mvc/Mvc.Razor.RuntimeCompilation/src/RazorReferenceManager.cs#L53 - var partFactory = ApplicationPartFactory.GetApplicationPartFactory(assembly); - foreach (ApplicationPart applicationPart in partFactory.GetApplicationParts(assembly)) - { - _applicationPartManager.ApplicationParts.Add(applicationPart); - } + Assembly assembly = _loadContextManager.LoadModelsAssembly(tempFile); return assembly; } diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/RuntimeCompilationCacheBuster.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/RuntimeCompilationCacheBuster.cs new file mode 100644 index 0000000000..51774b92f2 --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/RuntimeCompilationCacheBuster.cs @@ -0,0 +1,53 @@ +using Microsoft.AspNetCore.Mvc.Razor; +using Microsoft.AspNetCore.Mvc.Razor.Compilation; +using Umbraco.Cms.Core; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +internal class RuntimeCompilationCacheBuster +{ + private readonly IViewCompilerProvider _viewCompilerProvider; + private readonly IRazorViewEngine _razorViewEngine; + + public RuntimeCompilationCacheBuster( + IViewCompilerProvider viewCompilerProvider, + IRazorViewEngine razorViewEngine) + { + _viewCompilerProvider = viewCompilerProvider; + _razorViewEngine = razorViewEngine; + } + + private RazorViewEngine ViewEngine + { + get + { + if (_razorViewEngine is RazorViewEngine typedViewEngine) + { + return typedViewEngine; + } + + throw new InvalidOperationException( + "Unable to resolve RazorViewEngine, this means we can't clear the cache and views won't render properly"); + } + } + + private CollectibleRuntimeViewCompiler ViewCompiler + { + get + { + if (_viewCompilerProvider.GetCompiler() is CollectibleRuntimeViewCompiler collectibleCompiler) + { + return collectibleCompiler; + } + + throw new InvalidOperationException("Unable to resolve CollectibleRuntimeViewCompiler, and is unable to clear the cache"); + } + } + + public void BustCache() + { + ViewCompiler.ClearCache(); + Action? clearCacheMethod = ReflectionUtilities.EmitMethod>("ClearCache"); + clearCacheMethod?.Invoke(ViewEngine); + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/UmbracoAssemblyLoadContext.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoAssemblyLoadContext.cs similarity index 91% rename from src/Umbraco.Web.Common/ModelsBuilder/UmbracoAssemblyLoadContext.cs rename to src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoAssemblyLoadContext.cs index 17f148002e..5ca90c2df9 100644 --- a/src/Umbraco.Web.Common/ModelsBuilder/UmbracoAssemblyLoadContext.cs +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoAssemblyLoadContext.cs @@ -1,7 +1,7 @@ using System.Reflection; using System.Runtime.Loader; -namespace Umbraco.Cms.Web.Common.ModelsBuilder; +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; internal class UmbracoAssemblyLoadContext : AssemblyLoadContext { diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoCompilationException.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoCompilationException.cs new file mode 100644 index 0000000000..cb180a6264 --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoCompilationException.cs @@ -0,0 +1,8 @@ +using Microsoft.AspNetCore.Diagnostics; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +internal class UmbracoCompilationException : Exception, ICompilationException +{ + public IEnumerable? CompilationFailures { get; init; } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoRazorReferenceManager.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoRazorReferenceManager.cs new file mode 100644 index 0000000000..3a90e43be5 --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoRazorReferenceManager.cs @@ -0,0 +1,89 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection.PortableExecutable; +using Microsoft.AspNetCore.Mvc.ApplicationParts; +using Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation; +using Microsoft.CodeAnalysis; +using Microsoft.Extensions.Options; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + + +/* + * This is a clone'n'own of Microsofts RazorReferenceManager, please check if there's been any updates to the file on their end + * before trying to mock about fixing issues: + * https://github.com/dotnet/aspnetcore/blob/main/src/Mvc/Mvc.Razor.RuntimeCompilation/src/RazorReferenceManager.cs + * + * This class is used by the the ViewCompiler (CollectibleRuntimeViewCompiler) to find the required references when compiling views. + */ +internal class UmbracoRazorReferenceManager +{ + private readonly ApplicationPartManager _partManager; + private readonly MvcRazorRuntimeCompilationOptions _options; + private object _compilationReferencesLock = new object(); + private bool _compilationReferencesInitialized; + private IReadOnlyList? _compilationReferences; + + public UmbracoRazorReferenceManager( + ApplicationPartManager partManager, + IOptions options) + { + _partManager = partManager; + _options = options.Value; + } + + public virtual IReadOnlyList CompilationReferences + { + get + { + return LazyInitializer.EnsureInitialized( + ref _compilationReferences, + ref _compilationReferencesInitialized, + ref _compilationReferencesLock, + GetCompilationReferences)!; + } + } + + private IReadOnlyList GetCompilationReferences() + { + var referencePaths = GetReferencePaths(); + + return referencePaths + .Select(CreateMetadataReference) + .ToList(); + } + + // For unit testing + internal IEnumerable GetReferencePaths() + { + var referencePaths = new List(_options.AdditionalReferencePaths.Count); + + foreach (var part in _partManager.ApplicationParts) + { + if (part is ICompilationReferencesProvider compilationReferenceProvider) + { + referencePaths.AddRange(compilationReferenceProvider.GetReferencePaths()); + } + else if (part is AssemblyPart assemblyPart) + { + referencePaths.AddRange(assemblyPart.GetReferencePaths()); + } + } + + referencePaths.AddRange(_options.AdditionalReferencePaths); + + return referencePaths; + } + + private static MetadataReference CreateMetadataReference(string path) + { + using (var stream = File.OpenRead(path)) + { + var moduleMetadata = ModuleMetadata.CreateFromStream(stream, PEStreamOptions.PrefetchMetadata); + var assemblyMetadata = AssemblyMetadata.Create(moduleMetadata); + + return assemblyMetadata.GetReference(filePath: path); + } + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoViewCompilerProvider.cs b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoViewCompilerProvider.cs new file mode 100644 index 0000000000..f981402456 --- /dev/null +++ b/src/Umbraco.Web.Common/ModelsBuilder/InMemoryAuto/UmbracoViewCompilerProvider.cs @@ -0,0 +1,88 @@ +using Microsoft.AspNetCore.Mvc.ApplicationParts; +using Microsoft.AspNetCore.Mvc.Razor.Compilation; +using Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation; +using Microsoft.AspNetCore.Razor.Language; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Umbraco.Cms.Core.Exceptions; + +namespace Umbraco.Cms.Web.Common.ModelsBuilder.InMemoryAuto; + +internal class UmbracoViewCompilerProvider : IViewCompilerProvider +{ + private readonly RazorProjectEngine _razorProjectEngine; + private readonly UmbracoRazorReferenceManager _umbracoRazorReferenceManager; + private readonly CompilationOptionsProvider _compilationOptionsProvider; + private readonly InMemoryAssemblyLoadContextManager _loadContextManager; + + private readonly ApplicationPartManager _applicationPartManager; + + private readonly ILogger _logger; + private readonly Func _createCompiler; + + private object _initializeLock = new object(); + private bool _initialized; + private IViewCompiler? _compiler; + private readonly MvcRazorRuntimeCompilationOptions _options; + + + public UmbracoViewCompilerProvider( + ApplicationPartManager applicationPartManager, + RazorProjectEngine razorProjectEngine, + ILoggerFactory loggerFactory, + IOptions options, + UmbracoRazorReferenceManager umbracoRazorReferenceManager, + CompilationOptionsProvider compilationOptionsProvider, + InMemoryAssemblyLoadContextManager loadContextManager) + { + _applicationPartManager = applicationPartManager; + _razorProjectEngine = razorProjectEngine; + _umbracoRazorReferenceManager = umbracoRazorReferenceManager; + _compilationOptionsProvider = compilationOptionsProvider; + _loadContextManager = loadContextManager; + _options = options.Value; + + _logger = loggerFactory.CreateLogger(); + _createCompiler = CreateCompiler; + } + + public IViewCompiler GetCompiler() + { + return LazyInitializer.EnsureInitialized( + ref _compiler, + ref _initialized, + ref _initializeLock, + _createCompiler)!; + } + + private IViewCompiler CreateCompiler() + { + var feature = new ViewsFeature(); + _applicationPartManager.PopulateFeature(feature); + + return new CollectibleRuntimeViewCompiler( + GetCompositeFileProvider(_options), + _razorProjectEngine, + feature.ViewDescriptors, + _logger, + _umbracoRazorReferenceManager, + _compilationOptionsProvider, + _loadContextManager); + } + + private static IFileProvider GetCompositeFileProvider(MvcRazorRuntimeCompilationOptions options) + { + var fileProviders = options.FileProviders; + if (fileProviders.Count == 0) + { + throw new PanicException(); + } + else if (fileProviders.Count == 1) + { + return fileProviders[0]; + } + + return new CompositeFileProvider(fileProviders); + } +} diff --git a/src/Umbraco.Web.Common/ModelsBuilder/RefreshingRazorViewEngine.cs b/src/Umbraco.Web.Common/ModelsBuilder/RefreshingRazorViewEngine.cs deleted file mode 100644 index 6235c86fe3..0000000000 --- a/src/Umbraco.Web.Common/ModelsBuilder/RefreshingRazorViewEngine.cs +++ /dev/null @@ -1,194 +0,0 @@ -using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Razor; -using Microsoft.AspNetCore.Mvc.ViewEngines; - -/* - * OVERVIEW: - * - * The CSharpCompiler is responsible for the actual compilation of razor at runtime. - * It creates a CSharpCompilation instance to do the compilation. This is where DLL references - * are applied. However, the way this works is not flexible for dynamic assemblies since the references - * are only discovered and loaded once before the first compilation occurs. This is done here: - * https://github.com/dotnet/aspnetcore/blob/114f0f6d1ef1d777fb93d90c87ac506027c55ea0/src/Mvc/Mvc.Razor.RuntimeCompilation/src/CSharpCompiler.cs#L79 - * The CSharpCompiler is internal and cannot be replaced or extended, however it's references come from: - * RazorReferenceManager. Unfortunately this is also internal and cannot be replaced, though it can be extended - * using MvcRazorRuntimeCompilationOptions, except this is the place where references are only loaded once which - * is done with a LazyInitializer. See https://github.com/dotnet/aspnetcore/blob/master/src/Mvc/Mvc.Razor.RuntimeCompilation/src/RazorReferenceManager.cs#L35. - * - * The way that RazorReferenceManager works is by resolving references from the ApplicationPartsManager - either by - * an application part that is specifically an ICompilationReferencesProvider or an AssemblyPart. So to fulfill this - * requirement, we add the MB assembly to the assembly parts manager within the InMemoryModelFactory when the assembly - * is (re)generated. But due to the above restrictions, when re-generating, this will have no effect since the references - * have already been resolved with the LazyInitializer in the RazorReferenceManager. - * - * The services that can be replaced are: IViewCompilerProvider (default is the internal RuntimeViewCompilerProvider) and - * IViewCompiler (default is the internal RuntimeViewCompiler). There is one specific public extension point that I was - * hoping would solve all of the problems which was IMetadataReferenceFeature (implemented by LazyMetadataReferenceFeature - * which uses RazorReferencesManager) which is a razor feature that you can add - * to the RazorProjectEngine. It is used to resolve roslyn references and by default is backed by RazorReferencesManager. - * Unfortunately, this service is not used by the CSharpCompiler, it seems to only be used by some tag helper compilations. - * - * There are caches at several levels, all of which are not publicly accessible APIs (apart from RazorViewEngine.ViewLookupCache - * which is possible to clear by casting and then calling cache.Compact(100); but that doesn't get us far enough). - * - * For this to work, several caches must be cleared: - * - RazorViewEngine.ViewLookupCache - * - RazorReferencesManager._compilationReferences - * - RazorPageActivator._activationInfo (though this one may be optional) - * - RuntimeViewCompiler._cache - * - * What are our options? - * - * a) We can copy a ton of code into our application: CSharpCompiler, RuntimeViewCompilerProvider, RuntimeViewCompiler and - * RazorReferenceManager (probably more depending on the extent of Internal references). - * b) We can use reflection to try to access all of the above resources and try to forcefully clear caches and reset initialization flags. - * c) We hack these replace-able services with our own implementations that wrap the default services. To do this - * requires re-resolving the original services from a pre-built DI container. In effect this re-creates these - * services from scratch which means there is no caches. - * - * ... Option C works, we will use that but need to verify how this affects memory since ideally the old services will be GC'd. - * - * Option C, how its done: - * - Before we add our custom razor services to the container, we make a copy of the services collection which is the snapshot of registered services - * with razor defaults before ours are added. - * - We replace the default implementation of IRazorViewEngine with our own. This is a wrapping service that wraps the default RazorViewEngine instance. - * The ctor for this service takes in a Factory method to re-construct the default RazorViewEngine and all of it's dependency graph. - * - When the models change, the Factory is invoked and the default razor services are all re-created, thus clearing their caches and the newly - * created instance is wrapped. The RazorViewEngine is the only service that needs to be replaced and wrapped for this to work because it's dependency - * graph includes all of the above mentioned services, all the way up to the RazorProjectEngine and it's LazyMetadataReferenceFeature. - */ - -namespace Umbraco.Cms.Web.Common.ModelsBuilder; - -/// -/// Custom that wraps aspnetcore's default implementation -/// -/// -/// This is used so that when new models are built, the entire razor stack is re-constructed so all razor -/// caches and assembly references, etc... are cleared. -/// -internal class RefreshingRazorViewEngine : IRazorViewEngine, IDisposable -{ - private readonly Func _defaultRazorViewEngineFactory; - private readonly InMemoryModelFactory _inMemoryModelFactory; - private readonly ReaderWriterLockSlim _locker = new(); - private IRazorViewEngine _current; - private bool _disposedValue; - - /// - /// Initializes a new instance of the class. - /// - /// - /// A factory method used to re-construct the default aspnetcore - /// - /// The - public RefreshingRazorViewEngine( - Func defaultRazorViewEngineFactory, - InMemoryModelFactory inMemoryModelFactory) - { - _inMemoryModelFactory = inMemoryModelFactory; - _defaultRazorViewEngineFactory = defaultRazorViewEngineFactory; - _current = _defaultRazorViewEngineFactory(); - _inMemoryModelFactory.ModelsChanged += InMemoryModelFactoryModelsChanged; - } - - public void Dispose() => - - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(true); - - public RazorPageResult FindPage(ActionContext context, string pageName) - { - _locker.EnterReadLock(); - try - { - return _current.FindPage(context, pageName); - } - finally - { - _locker.ExitReadLock(); - } - } - - public string? GetAbsolutePath(string? executingFilePath, string? pagePath) - { - _locker.EnterReadLock(); - try - { - return _current.GetAbsolutePath(executingFilePath, pagePath); - } - finally - { - _locker.ExitReadLock(); - } - } - - public RazorPageResult GetPage(string executingFilePath, string pagePath) - { - _locker.EnterReadLock(); - try - { - return _current.GetPage(executingFilePath, pagePath); - } - finally - { - _locker.ExitReadLock(); - } - } - - public ViewEngineResult FindView(ActionContext context, string viewName, bool isMainPage) - { - _locker.EnterReadLock(); - try - { - return _current.FindView(context, viewName, isMainPage); - } - finally - { - _locker.ExitReadLock(); - } - } - - public ViewEngineResult GetView(string? executingFilePath, string viewPath, bool isMainPage) - { - _locker.EnterReadLock(); - try - { - return _current.GetView(executingFilePath, viewPath, isMainPage); - } - finally - { - _locker.ExitReadLock(); - } - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - _inMemoryModelFactory.ModelsChanged -= InMemoryModelFactoryModelsChanged; - _locker.Dispose(); - } - - _disposedValue = true; - } - } - - /// - /// When the models change, re-construct the razor stack - /// - private void InMemoryModelFactoryModelsChanged(object? sender, EventArgs e) - { - _locker.EnterWriteLock(); - try - { - _current = _defaultRazorViewEngineFactory(); - } - finally - { - _locker.ExitWriteLock(); - } - } -} diff --git a/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/ModelsBuilder/modelsbuilder.spec.ts b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/ModelsBuilder/modelsbuilder.spec.ts new file mode 100644 index 0000000000..bb3403de82 --- /dev/null +++ b/tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/ModelsBuilder/modelsbuilder.spec.ts @@ -0,0 +1,280 @@ +import {AliasHelper, ApiHelpers, ConstantHelper, test, UiHelpers} from '@umbraco/playwright-testhelpers'; +import { + ContentBuilder, + DocumentTypeBuilder, +} from "@umbraco/json-models-builders"; + +test.describe('Modelsbuilder tests', () => { + + test.beforeEach(async ({page, umbracoApi}) => { + await umbracoApi.login(); + }); + + test('Can create and render content', async ({page, umbracoApi, umbracoUi}) => { + const docTypeName = "TestDocument"; + const docTypeAlias = AliasHelper.toAlias(docTypeName); + const contentName = "Home"; + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName); + + const docType = new DocumentTypeBuilder() + .withName(docTypeName) + .withAlias(docTypeAlias) + .withAllowAsRoot(true) + .withDefaultTemplate(docTypeAlias) + .addTab() + .withName("Content") + .addTextBoxProperty() + .withAlias("title") + .done() + .done() + .build(); + + await umbracoApi.documentTypes.save(docType); + await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels; +@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage +@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels; +@{ +\tLayout = null; +} + +

@Model.Title

`); + + // Time to manually create the content + await umbracoUi.createContentWithDocumentType(docTypeName); + await umbracoUi.setEditorHeaderName(contentName); + // Fortunately for us the input field of a text box has the alias of the property as an id :) + await page.locator("#title").type("Hello world!") + await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.saveAndPublish)); + await umbracoUi.isSuccessNotificationVisible(); + // Ensure that we can render it on the frontend = we can compile the models and views + await umbracoApi.content.verifyRenderedContent("/", "

Hello world!

", true); + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName); + }); + + test('Can update document type without updating view', async ({page, umbracoApi, umbracoUi}) => { + const docTypeName = "TestDocument"; + const docTypeAlias = AliasHelper.toAlias(docTypeName); + const propertyAlias = "title"; + const propertyValue = "Hello world!" + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName); + + const docType = new DocumentTypeBuilder() + .withName(docTypeName) + .withAlias(docTypeAlias) + .withAllowAsRoot(true) + .withDefaultTemplate(docTypeAlias) + .addTab() + .withName("Content") + .addTextBoxProperty() + .withAlias(propertyAlias) + .done() + .done() + .build(); + + const savedDocType = await umbracoApi.documentTypes.save(docType); + await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels; +@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage +@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels; +@{ +\tLayout = null; +} + +

@Model.Title

`); + + const content = new ContentBuilder() + .withContentTypeAlias(savedDocType["alias"]) + .withAction("publishNew") + .addVariant() + .withName("Home") + .withSave(true) + .withPublish(true) + .addProperty() + .withAlias(propertyAlias) + .withValue(propertyValue) + .done() + .done() + .build() + + await umbracoApi.content.save(content); + + // Navigate to the document type + await umbracoUi.goToSection(ConstantHelper.sections.settings); + await umbracoUi.clickElement(umbracoUi.getTreeItem("settings", ["Document Types", docTypeName])); + // Add a new property (this might cause a version error if the viewcache is not cleared, hence this test + await page.locator('.umb-box-content >> [data-element="property-add"]').click(); + await page.locator('[data-element="property-name"]').type("Second Title"); + await page.locator('[data-element="editor-add"]').click(); + await page.locator('[input-id="datatype-search"]').type("Textstring"); + await page.locator('.umb-card-grid >> [title="Textstring"]').click(); + await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.submit)); + await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save)); + await umbracoUi.isSuccessNotificationVisible(); + + // Now that the content is updated and the models are rebuilt, ensure that we can still render the frontend. + await umbracoApi.content.verifyRenderedContent("/", "

" + propertyValue + "

", true) + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName); + }); + + test('Can update view without updating document type', async ({page, umbracoApi, umbracoUi}) => { + const docTypeName = "TestDocument"; + const docTypeAlias = AliasHelper.toAlias(docTypeName); + const propertyAlias = "title"; + const propertyValue = "Hello world!" + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName); + + const docType = new DocumentTypeBuilder() + .withName(docTypeName) + .withAlias(docTypeAlias) + .withAllowAsRoot(true) + .withDefaultTemplate(docTypeAlias) + .addTab() + .withName("Content") + .addTextBoxProperty() + .withAlias(propertyAlias) + .done() + .done() + .build(); + + const savedDocType = await umbracoApi.documentTypes.save(docType); + await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels; +@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage +@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels; +@{ +\tLayout = null; +} + +

@Model.Title

`); + + const content = new ContentBuilder() + .withContentTypeAlias(savedDocType["alias"]) + .withAction("publishNew") + .addVariant() + .withName("Home") + .withSave(true) + .withPublish(true) + .addProperty() + .withAlias(propertyAlias) + .withValue(propertyValue) + .done() + .done() + .build() + + await umbracoApi.content.save(content); + + // Navigate to the document type + await umbracoUi.goToSection(ConstantHelper.sections.settings); + await umbracoUi.clickElement(umbracoUi.getTreeItem("settings", ["templates", docTypeName])); + const editor = await page.locator('.ace_content'); + await editor.click(); + // We only have to type out the opening tag, the editor adds the closing tag automatically. + await editor.type("

Edited") + await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save)) + + await umbracoUi.isSuccessNotificationVisible(); + await umbracoApi.content.verifyRenderedContent("/", "

" + propertyValue + "

Edited

", true) + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName); + }); + + test('Can update view and document type', async ({page, umbracoApi, umbracoUi}) => { + const docTypeName = "TestDocument"; + const docTypeAlias = AliasHelper.toAlias(docTypeName); + const propertyAlias = "title"; + const propertyValue = "Hello world!" + const contentName = "Home"; + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName); + + const docType = new DocumentTypeBuilder() + .withName(docTypeName) + .withAlias(docTypeAlias) + .withAllowAsRoot(true) + .withDefaultTemplate(docTypeAlias) + .addTab() + .withName("Content") + .addTextBoxProperty() + .withAlias(propertyAlias) + .done() + .done() + .build(); + + const savedDocType = await umbracoApi.documentTypes.save(docType); + await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels; +@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage +@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels; +@{ +\tLayout = null; +} + +

@Model.Title

`); + + const content = new ContentBuilder() + .withContentTypeAlias(savedDocType["alias"]) + .withAction("publishNew") + .addVariant() + .withName(contentName) + .withSave(true) + .withPublish(true) + .addProperty() + .withAlias(propertyAlias) + .withValue(propertyValue) + .done() + .done() + .build() + + await umbracoApi.content.save(content); + + // Navigate to the document type + await umbracoUi.goToSection(ConstantHelper.sections.settings); + await umbracoUi.clickElement(umbracoUi.getTreeItem("settings", ["Document Types", docTypeName])); + // Add a new property (this might cause a version error if the viewcache is not cleared, hence this test + await page.locator('.umb-box-content >> [data-element="property-add"]').click(); + await page.locator('[data-element="property-name"]').type("Bod"); + await page.locator('[data-element="editor-add"]').click(); + await page.locator('[input-id="datatype-search"]').type("Textstring"); + await page.locator('.umb-card-grid >> [title="Textstring"]').click(); + await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.submit)); + await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save)); + await umbracoUi.isSuccessNotificationVisible(); + + // Update the template + await umbracoUi.clickElement(umbracoUi.getTreeItem("settings", ["templates", docTypeName])); + const editor = await page.locator('.ace_content'); + await editor.click(); + // We only have to type out the opening tag, the editor adds the closing tag automatically. + await editor.type("

@Model.Bod") + await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save)) + await umbracoUi.isSuccessNotificationVisible(); + + // Navigate to the content section and update the content + await umbracoUi.goToSection(ConstantHelper.sections.content); + await umbracoUi.refreshContentTree(); + await umbracoUi.clickElement(umbracoUi.getTreeItem("content", [contentName])); + await page.locator("#bod").type("Fancy body text"); + + await umbracoApi.content.verifyRenderedContent("/", "

" + propertyValue + "

Fancy body text

", true); + + await umbracoApi.content.deleteAllContent(); + await umbracoApi.documentTypes.ensureNameNotExists(docTypeName); + await umbracoApi.templates.ensureNameNotExists(docTypeName) + }); +});