Files
Umbraco-CMS/src/Umbraco.ModelsBuilder.Embedded/PureLiveModelFactory.cs

701 lines
29 KiB
C#
Raw Normal View History

2019-06-24 11:58:36 +02:00
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Reflection.Emit;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Web;
using System.Web.Compilation;
using System.Web.WebPages.Razor;
using Umbraco.Core.Configuration;
2019-06-24 11:58:36 +02:00
using Umbraco.Core;
using Umbraco.Core.Hosting;
using Umbraco.Core.IO;
2019-06-24 11:58:36 +02:00
using Umbraco.Core.Logging;
using Umbraco.Core.Models.PublishedContent;
using Umbraco.ModelsBuilder.Embedded.Building;
2019-06-24 11:58:36 +02:00
using File = System.IO.File;
using Umbraco.Core.Configuration.Models;
using Microsoft.Extensions.Options;
2019-06-24 11:58:36 +02:00
namespace Umbraco.ModelsBuilder.Embedded
2019-06-24 11:58:36 +02:00
{
internal class PureLiveModelFactory : ILivePublishedModelFactory, IRegisteredObject
{
private Assembly _modelsAssembly;
private Infos _infos = new Infos { ModelInfos = null, ModelTypeMap = new Dictionary<string, Type>() };
private readonly ReaderWriterLockSlim _locker = new ReaderWriterLockSlim();
private volatile bool _hasModels; // volatile 'cos reading outside lock
private bool _pendingRebuild;
private readonly IProfilingLogger _logger;
private readonly FileSystemWatcher _watcher;
private int _ver, _skipver;
private readonly int _debugLevel;
private BuildManager _theBuildManager;
private readonly Lazy<UmbracoServices> _umbracoServices; // fixme: this is because of circular refs :(
2019-06-24 11:58:36 +02:00
private UmbracoServices UmbracoServices => _umbracoServices.Value;
private static readonly Regex AssemblyVersionRegex = new Regex("AssemblyVersion\\(\"[0-9]+.[0-9]+.[0-9]+.[0-9]+\"\\)", RegexOptions.Compiled);
private const string ProjVirt = "~/App_Data/Models/all.generated.cs";
private static readonly string[] OurFiles = { "models.hash", "models.generated.cs", "all.generated.cs", "all.dll.path", "models.err" };
private readonly ModelsBuilderConfig _config;
private readonly IApplicationShutdownRegistry _hostingLifetime;
private readonly IIOHelper _ioHelper;
private readonly ModelsGenerationError _errors;
2019-06-24 11:58:36 +02:00
public PureLiveModelFactory(
Lazy<UmbracoServices> umbracoServices,
IProfilingLogger logger,
IOptions<ModelsBuilderConfig> config,
IHostingEnvironment hostingEnvironment,
IApplicationShutdownRegistry hostingLifetime,
IIOHelper ioHelper)
2019-06-24 11:58:36 +02:00
{
_umbracoServices = umbracoServices;
_logger = logger;
_config = config.Value;
_hostingLifetime = hostingLifetime;
_ioHelper = ioHelper;
_errors = new ModelsGenerationError(config, ioHelper);
2019-06-24 11:58:36 +02:00
_ver = 1; // zero is for when we had no version
_skipver = -1; // nothing to skip
RazorBuildProvider.CodeGenerationStarted += RazorBuildProvider_CodeGenerationStarted;
if (!hostingEnvironment.IsHosted) return;
2019-06-24 11:58:36 +02:00
var modelsDirectory = _config.ModelsDirectoryAbsolute(_ioHelper);
2019-06-24 11:58:36 +02:00
if (!Directory.Exists(modelsDirectory))
Directory.CreateDirectory(modelsDirectory);
// BEWARE! if the watcher is not properly released then for some reason the
// BuildManager will start confusing types - using a 'registered object' here
// though we should probably plug into Umbraco's MainDom - which is internal
_hostingLifetime.RegisterObject(this);
2019-06-24 11:58:36 +02:00
_watcher = new FileSystemWatcher(modelsDirectory);
_watcher.Changed += WatcherOnChanged;
_watcher.EnableRaisingEvents = true;
// get it here, this need to be fast
_debugLevel = _config.DebugLevel;
}
#region ILivePublishedModelFactory
/// <inheritdoc />
public object SyncRoot { get; } = new object();
/// <inheritdoc />
public void Refresh()
{
ResetModels();
EnsureModels();
}
#endregion
#region IPublishedModelFactory
public IPublishedElement CreateModel(IPublishedElement element)
{
// get models, rebuilding them if needed
var infos = EnsureModels()?.ModelInfos;
if (infos == null)
return element;
// be case-insensitive
var contentTypeAlias = element.ContentType.Alias;
// lookup model constructor (else null)
infos.TryGetValue(contentTypeAlias, out var info);
2019-06-24 11:58:36 +02:00
// create model
return info == null ? element : info.Ctor(element);
}
// this runs only once the factory is ready
// NOT when building models
public Type MapModelType(Type type)
{
var infos = EnsureModels();
return ModelType.Map(type, infos.ModelTypeMap);
}
// this runs only once the factory is ready
// NOT when building models
public IList CreateModelList(string alias)
{
var infos = EnsureModels();
// fail fast
if (infos == null)
return new List<IPublishedElement>();
if (!infos.ModelInfos.TryGetValue(alias, out var modelInfo))
return new List<IPublishedElement>();
var ctor = modelInfo.ListCtor;
if (ctor != null) return ctor();
var listType = typeof(List<>).MakeGenericType(modelInfo.ModelType);
ctor = modelInfo.ListCtor = ReflectionUtilities.EmitConstructor<Func<IList>>(declaring: listType);
return ctor();
}
/// <inheritdoc />
public bool Enabled => _config.Enable;
/// <inheritdoc />
public void Reset()
{
if (_config.Enable)
ResetModels();
}
2019-06-24 11:58:36 +02:00
#endregion
#region Compilation
// deadlock note
//
// when RazorBuildProvider_CodeGenerationStarted runs, the thread has Monitor.Enter-ed the BuildManager
// singleton instance, through a call to CompilationLock.GetLock in BuildManager.GetVPathBuildResultInternal,
// and now wants to lock _locker.
// when EnsureModels runs, the thread locks _locker and then wants BuildManager to compile, which in turns
// requires that the BuildManager can Monitor.Enter-ed itself.
// so:
//
// T1 - needs to ensure models, locks _locker
// T2 - needs to compile a view, locks BuildManager
// hits RazorBuildProvider_CodeGenerationStarted
// wants to lock _locker, wait
// T1 - needs to compile models, using BuildManager
// wants to lock itself, wait
// <deadlock>
//
// until ASP.NET kills the long-running request (thread abort)
//
// problem is, we *want* to suspend views compilation while the models assembly is being changed else we
// end up with views compiled and cached with the old assembly, while models come from the new assembly,
// which gives more YSOD. so we *have* to lock _locker in RazorBuildProvider_CodeGenerationStarted.
//
// one "easy" solution consists in locking the BuildManager *before* _locker in EnsureModels, thus ensuring
// we always lock in the same order, and getting rid of deadlocks - but that requires having access to the
// current BuildManager instance, which is BuildManager.TheBuildManager, which is an internal property.
//
// well, that's what we are doing in this class' TheBuildManager property, using reflection.
private void RazorBuildProvider_CodeGenerationStarted(object sender, EventArgs e)
{
try
{
_locker.EnterReadLock();
// just be safe - can happen if the first view is not an Umbraco view,
// or if something went wrong and we don't have an assembly at all
if (_modelsAssembly == null) return;
if (_debugLevel > 0)
_logger.Debug<PureLiveModelFactory>("RazorBuildProvider.CodeGenerationStarted");
if (!(sender is RazorBuildProvider provider)) return;
// add the assembly, and add a dependency to a text file that will change on each
// compilation as in some environments (could not figure which/why) the BuildManager
// would not re-compile the views when the models assembly is rebuilt.
provider.AssemblyBuilder.AddAssemblyReference(_modelsAssembly);
provider.AddVirtualPathDependency(ProjVirt);
}
finally
{
if (_locker.IsReadLockHeld)
_locker.ExitReadLock();
}
}
// tells the factory that it should build a new generation of models
private void ResetModels()
{
_logger.Debug<PureLiveModelFactory>("Resetting models.");
try
{
_locker.EnterWriteLock();
_hasModels = false;
_pendingRebuild = true;
var modelsDirectory = _config.ModelsDirectoryAbsolute(_ioHelper);
2019-06-24 11:58:36 +02:00
if (!Directory.Exists(modelsDirectory))
Directory.CreateDirectory(modelsDirectory);
// clear stuff
var modelsHashFile = Path.Combine(modelsDirectory, "models.hash");
//var modelsSrcFile = Path.Combine(modelsDirectory, "models.generated.cs");
//var projFile = Path.Combine(modelsDirectory, "all.generated.cs");
var dllPathFile = Path.Combine(modelsDirectory, "all.dll.path");
if (File.Exists(dllPathFile)) File.Delete(dllPathFile);
if (File.Exists(modelsHashFile)) File.Delete(modelsHashFile);
}
finally
{
if (_locker.IsWriteLockHeld)
_locker.ExitWriteLock();
}
}
// gets "the" build manager
private BuildManager TheBuildManager
{
get
{
if (_theBuildManager != null) return _theBuildManager;
var prop = typeof(BuildManager).GetProperty("TheBuildManager", BindingFlags.NonPublic | BindingFlags.Static);
2019-06-24 11:58:36 +02:00
if (prop == null)
throw new InvalidOperationException("Could not get BuildManager.TheBuildManager property.");
_theBuildManager = (BuildManager)prop.GetValue(null);
2019-06-24 11:58:36 +02:00
return _theBuildManager;
}
}
// ensure that the factory is running with the lastest generation of models
internal Infos EnsureModels()
{
if (_debugLevel > 0)
_logger.Debug<PureLiveModelFactory>("Ensuring models.");
// don't use an upgradeable lock here because only 1 thread at a time could enter it
try
{
_locker.EnterReadLock();
if (_hasModels)
return _infos;
}
finally
{
if (_locker.IsReadLockHeld)
_locker.ExitReadLock();
}
var buildManagerLocked = false;
try
{
// always take the BuildManager lock *before* taking the _locker lock
// to avoid possible deadlock situations (see notes above)
Monitor.Enter(TheBuildManager, ref buildManagerLocked);
_locker.EnterUpgradeableReadLock();
if (_hasModels) return _infos;
_locker.EnterWriteLock();
// we don't have models,
// either they haven't been loaded from the cache yet
// or they have been reseted and are pending a rebuild
using (_logger.DebugDuration<PureLiveModelFactory>("Get models.", "Got models."))
{
try
{
var assembly = GetModelsAssembly(_pendingRebuild);
// the one below can be used to simulate an issue with BuildManager, ie it will register
// the models with the factory but NOT with the BuildManager, which will not recompile views.
// this is for U4-8043 which is an obvious issue but I cannot replicate
//_modelsAssembly = _modelsAssembly ?? assembly;
// the one below is the normal one
_modelsAssembly = assembly;
var types = assembly.ExportedTypes.Where(x => x.Inherits<PublishedContentModel>() || x.Inherits<PublishedElementModel>());
_infos = RegisterModels(types);
_errors.Clear();
2019-06-24 11:58:36 +02:00
}
catch (Exception e)
{
try
{
_logger.Error<PureLiveModelFactory>("Failed to build models.", e);
_logger.Warn<PureLiveModelFactory>("Running without models."); // be explicit
_errors.Report("Failed to build PureLive models.", e);
2019-06-24 11:58:36 +02:00
}
finally
{
_modelsAssembly = null;
_infos = new Infos { ModelInfos = null, ModelTypeMap = new Dictionary<string, Type>() };
}
}
// don't even try again
_hasModels = true;
}
return _infos;
}
finally
{
if (_locker.IsWriteLockHeld)
_locker.ExitWriteLock();
if (_locker.IsUpgradeableReadLockHeld)
_locker.ExitUpgradeableReadLock();
if (buildManagerLocked)
Monitor.Exit(TheBuildManager);
}
}
private Assembly GetModelsAssembly(bool forceRebuild)
{
var modelsDirectory = _config.ModelsDirectoryAbsolute(_ioHelper);
2019-06-24 11:58:36 +02:00
if (!Directory.Exists(modelsDirectory))
Directory.CreateDirectory(modelsDirectory);
var typeModels = UmbracoServices.GetAllTypes();
var currentHash = TypeModelHasher.Hash(typeModels);
2019-06-24 11:58:36 +02:00
var modelsHashFile = Path.Combine(modelsDirectory, "models.hash");
var modelsSrcFile = Path.Combine(modelsDirectory, "models.generated.cs");
var projFile = Path.Combine(modelsDirectory, "all.generated.cs");
var dllPathFile = Path.Combine(modelsDirectory, "all.dll.path");
// caching the generated models speeds up booting
// currentHash hashes both the types & the user's partials
if (!forceRebuild)
{
_logger.Debug<PureLiveModelFactory>("Looking for cached models.");
if (File.Exists(modelsHashFile) && File.Exists(projFile))
{
var cachedHash = File.ReadAllText(modelsHashFile);
if (currentHash != cachedHash)
{
_logger.Debug<PureLiveModelFactory>("Found obsolete cached models.");
forceRebuild = true;
}
// else cachedHash matches currentHash, we can try to load an existing dll
}
else
{
_logger.Debug<PureLiveModelFactory>("Could not find cached models.");
forceRebuild = true;
}
}
Assembly assembly;
if (!forceRebuild)
{
// try to load the dll directly (avoid rebuilding)
//
// ensure that the .dll file does not have a corresponding .dll.delete file
// as that would mean the the .dll file is going to be deleted and should not
// be re-used - that should not happen in theory, but better be safe
//
// ensure that the .dll file is in the current codegen directory - when IIS
// or Express does a full restart, it can switch to an entirely new codegen
// directory, and then we end up referencing a dll which is *not* in that
// directory, and BuildManager fails to instantiate views ("the view found
// at ... was not created").
//
if (File.Exists(dllPathFile))
{
var dllPath = File.ReadAllText(dllPathFile);
var codegen = HttpRuntime.CodegenDir;
_logger.Debug<PureLiveModelFactory>($"Cached models dll at {dllPath}.");
if (File.Exists(dllPath) && !File.Exists(dllPath + ".delete") && dllPath.StartsWith(codegen))
{
assembly = Assembly.LoadFile(dllPath);
var attr = assembly.GetCustomAttribute<ModelsBuilderAssemblyAttribute>();
if (attr != null && attr.PureLive && attr.SourceHash == currentHash)
{
// if we were to resume at that revision, then _ver would keep increasing
// and that is probably a bad idea - so, we'll always rebuild starting at
// ver 1, but we remember we want to skip that one - so we never end up
// with the "same but different" version of the assembly in memory
_skipver = assembly.GetName().Version.Revision;
_logger.Debug<PureLiveModelFactory>("Loading cached models (dll).");
return assembly;
}
_logger.Debug<PureLiveModelFactory>("Cached models dll cannot be loaded (invalid assembly).");
}
else if (!File.Exists(dllPath))
_logger.Debug<PureLiveModelFactory>("Cached models dll does not exist.");
else if (File.Exists(dllPath + ".delete"))
_logger.Debug<PureLiveModelFactory>("Cached models dll is marked for deletion.");
else if (!dllPath.StartsWith(codegen))
_logger.Debug<PureLiveModelFactory>("Cached models dll is in a different codegen directory.");
else
_logger.Debug<PureLiveModelFactory>("Cached models dll cannot be loaded (why?).");
}
// must reset the version in the file else it would keep growing
// loading cached modules only happens when the app restarts
var text = File.ReadAllText(projFile);
var match = AssemblyVersionRegex.Match(text);
if (match.Success)
{
text = text.Replace(match.Value, "AssemblyVersion(\"0.0.0." + _ver + "\")");
File.WriteAllText(projFile, text);
}
// generate a marker file that will be a dependency
// see note in RazorBuildProvider_CodeGenerationStarted
// NO: using all.generated.cs as a dependency
//File.WriteAllText(Path.Combine(modelsDirectory, "models.dep"), "VER:" + _ver);
_ver++;
try
{
assembly = BuildManager.GetCompiledAssembly(ProjVirt);
File.WriteAllText(dllPathFile, assembly.Location);
}
catch
{
ClearOnFailingToCompile(dllPathFile, modelsHashFile, projFile);
throw;
}
_logger.Debug<PureLiveModelFactory>("Loading cached models (source).");
return assembly;
}
// need to rebuild
_logger.Debug<PureLiveModelFactory>("Rebuilding models.");
// generate code, save
var code = GenerateModelsCode(typeModels);
// add extra attributes,
// PureLiveAssembly helps identifying Assemblies that contain PureLive models
// AssemblyVersion is so that we have a different version for each rebuild
var ver = _ver == _skipver ? ++_ver : _ver;
_ver++;
code = code.Replace("//ASSATTR", $@"[assembly:ModelsBuilderAssembly(PureLive = true, SourceHash = ""{currentHash}"")]
2019-06-24 11:58:36 +02:00
[assembly:System.Reflection.AssemblyVersion(""0.0.0.{ver}"")]");
File.WriteAllText(modelsSrcFile, code);
// generate proj, save
var projFiles = new Dictionary<string, string>
{
{ "models.generated.cs", code }
};
var proj = GenerateModelsProj(projFiles);
File.WriteAllText(projFile, proj);
// compile and register
try
{
assembly = BuildManager.GetCompiledAssembly(ProjVirt);
File.WriteAllText(dllPathFile, assembly.Location);
File.WriteAllText(modelsHashFile, currentHash);
}
catch
{
ClearOnFailingToCompile(dllPathFile, modelsHashFile, projFile);
throw;
}
_logger.Debug<PureLiveModelFactory>("Done rebuilding.");
return assembly;
}
private void ClearOnFailingToCompile(string dllPathFile, string modelsHashFile, string projFile)
{
_logger.Debug<PureLiveModelFactory>("Failed to compile.");
// the dll file reference still points to the previous dll, which is obsolete
// now and will be deleted by ASP.NET eventually, so better clear that reference.
// also touch the proj file to force views to recompile - don't delete as it's
// useful to have the source around for debugging.
try
{
if (File.Exists(dllPathFile)) File.Delete(dllPathFile);
if (File.Exists(modelsHashFile)) File.Delete(modelsHashFile);
if (File.Exists(projFile)) File.SetLastWriteTime(projFile, DateTime.Now);
}
catch { /* enough */ }
}
private static Infos RegisterModels(IEnumerable<Type> types)
{
var ctorArgTypes = new[] { typeof(IPublishedElement) };
2019-06-24 11:58:36 +02:00
var modelInfos = new Dictionary<string, ModelInfo>(StringComparer.InvariantCultureIgnoreCase);
var map = new Dictionary<string, Type>();
foreach (var type in types)
{
ConstructorInfo constructor = null;
Type parameterType = null;
foreach (var ctor in type.GetConstructors())
{
var parms = ctor.GetParameters();
if (parms.Length == 1 && typeof(IPublishedElement).IsAssignableFrom(parms[0].ParameterType))
2019-06-24 11:58:36 +02:00
{
if (constructor != null)
throw new InvalidOperationException($"Type {type.FullName} has more than one public constructor with one argument of type, or implementing, IPropertySet.");
constructor = ctor;
parameterType = parms[0].ParameterType;
}
}
if (constructor == null)
throw new InvalidOperationException($"Type {type.FullName} is missing a public constructor with one argument of type, or implementing, IPropertySet.");
var attribute = type.GetCustomAttribute<PublishedModelAttribute>(false);
var typeName = attribute == null ? type.Name : attribute.ContentTypeAlias;
if (modelInfos.TryGetValue(typeName, out var modelInfo))
2019-06-24 11:58:36 +02:00
throw new InvalidOperationException($"Both types {type.FullName} and {modelInfo.ModelType.FullName} want to be a model type for content type with alias \"{typeName}\".");
2020-07-14 10:08:59 +10:00
// TODO: use Core's ReflectionUtilities.EmitCtor !!
// Yes .. DynamicMethod is uber slow
2020-07-14 10:08:59 +10:00
// TODO: But perhaps https://docs.microsoft.com/en-us/dotnet/api/system.reflection.emit.constructorbuilder?view=netcore-3.1 is better still?
// See CtorInvokeBenchmarks
var meth = new DynamicMethod(string.Empty, typeof(IPublishedElement), ctorArgTypes, type.Module, true);
2019-06-24 11:58:36 +02:00
var gen = meth.GetILGenerator();
gen.Emit(OpCodes.Ldarg_0);
gen.Emit(OpCodes.Newobj, constructor);
gen.Emit(OpCodes.Ret);
var func = (Func<IPublishedElement, IPublishedElement>)meth.CreateDelegate(typeof(Func<IPublishedElement, IPublishedElement>));
2019-06-24 11:58:36 +02:00
modelInfos[typeName] = new ModelInfo { ParameterType = parameterType, Ctor = func, ModelType = type };
map[typeName] = type;
}
return new Infos { ModelInfos = modelInfos.Count > 0 ? modelInfos : null, ModelTypeMap = map };
}
private string GenerateModelsCode(IList<TypeModel> typeModels)
{
var modelsDirectory = _config.ModelsDirectoryAbsolute(_ioHelper);
2019-06-24 11:58:36 +02:00
if (!Directory.Exists(modelsDirectory))
Directory.CreateDirectory(modelsDirectory);
foreach (var file in Directory.GetFiles(modelsDirectory, "*.generated.cs"))
File.Delete(file);
var builder = new TextBuilder(_config, typeModels);
2019-06-24 11:58:36 +02:00
var codeBuilder = new StringBuilder();
builder.Generate(codeBuilder, builder.GetModelsToGenerate());
var code = codeBuilder.ToString();
return code;
}
private static readonly Regex UsingRegex = new Regex("^using(.*);", RegexOptions.Compiled | RegexOptions.Multiline);
private static readonly Regex AattrRegex = new Regex("^\\[assembly:(.*)\\]", RegexOptions.Compiled | RegexOptions.Multiline);
private static string GenerateModelsProj(IDictionary<string, string> files)
{
// ideally we would generate a CSPROJ file but then we'd need a BuildProvider for csproj
// trying to keep things simple for the time being, just write everything to one big file
// group all 'using' at the top of the file (else fails)
var usings = new List<string>();
foreach (var k in files.Keys.ToList())
files[k] = UsingRegex.Replace(files[k], m =>
{
usings.Add(m.Groups[1].Value);
return string.Empty;
});
// group all '[assembly:...]' at the top of the file (else fails)
var aattrs = new List<string>();
foreach (var k in files.Keys.ToList())
files[k] = AattrRegex.Replace(files[k], m =>
{
aattrs.Add(m.Groups[1].Value);
return string.Empty;
});
var text = new StringBuilder();
foreach (var u in usings.Distinct())
{
text.Append("using ");
text.Append(u);
text.Append(";\r\n");
}
foreach (var a in aattrs)
{
text.Append("[assembly:");
text.Append(a);
text.Append("]\r\n");
}
text.Append("\r\n\r\n");
foreach (var f in files)
{
text.Append("// FILE: ");
text.Append(f.Key);
text.Append("\r\n\r\n");
text.Append(f.Value);
text.Append("\r\n\r\n\r\n");
}
text.Append("// EOF\r\n");
return text.ToString();
}
internal class Infos
{
public Dictionary<string, Type> ModelTypeMap { get; set; }
public Dictionary<string, ModelInfo> ModelInfos { get; set; }
}
internal class ModelInfo
{
public Type ParameterType { get; set; }
public Func<IPublishedElement, IPublishedElement> Ctor { get; set; }
public Type ModelType { get; set; }
public Func<IList> ListCtor { get; set; }
}
#endregion
#region Watching
private void WatcherOnChanged(object sender, FileSystemEventArgs args)
{
var changed = args.Name;
// don't reset when our files change because we are building!
//
// comment it out, and always ignore our files, because it seems that some
// race conditions can occur on slow Cloud filesystems and then we keep
// rebuilding
//if (_building && OurFiles.Contains(changed))
//{
// //_logger.Info<PureLiveModelFactory>("Ignoring files self-changes.");
// return;
//}
// always ignore our own file changes
if (OurFiles.Contains(changed))
return;
_logger.Info<PureLiveModelFactory>("Detected files changes.");
lock (SyncRoot) // don't reset while being locked
ResetModels();
}
public void Stop(bool immediate)
{
_watcher.EnableRaisingEvents = false;
_watcher.Dispose();
_hostingLifetime.UnregisterObject(this);
2019-06-24 11:58:36 +02:00
}
#endregion
}
}