using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; using System.Reflection.Emit; using System.Text; using System.Text.RegularExpressions; using System.Threading; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Umbraco.Core; using Umbraco.Core.Configuration; using Umbraco.Core.Configuration.Models; using Umbraco.Core.Hosting; using Umbraco.Core.Logging; using Umbraco.Core.Models.PublishedContent; using Umbraco.ModelsBuilder.Embedded.Building; using File = System.IO.File; namespace Umbraco.ModelsBuilder.Embedded { internal class PureLiveModelFactory : ILivePublishedModelFactory, IRegisteredObject { private Infos _infos = new Infos { ModelInfos = null, ModelTypeMap = new Dictionary() }; private readonly ReaderWriterLockSlim _locker = new ReaderWriterLockSlim(); private volatile bool _hasModels; // volatile 'cos reading outside lock private bool _pendingRebuild; private readonly IProfilingLogger _profilingLogger; private readonly ILogger _logger; private readonly FileSystemWatcher _watcher; private int _ver; private int _skipver; private readonly int _debugLevel; private RoslynCompiler _roslynCompiler; private UmbracoAssemblyLoadContext _currentAssemblyLoadContext; private readonly Lazy _umbracoServices; // fixme: this is because of circular refs :( private static readonly Regex s_assemblyVersionRegex = new Regex("AssemblyVersion\\(\"[0-9]+.[0-9]+.[0-9]+.[0-9]+\"\\)", RegexOptions.Compiled); private static readonly string[] s_ourFiles = { "models.hash", "models.generated.cs", "all.generated.cs", "all.dll.path", "models.err", "Compiled" }; private readonly ModelsBuilderSettings _config; private readonly IHostingEnvironment _hostingEnvironment; private readonly IApplicationShutdownRegistry _hostingLifetime; private readonly ModelsGenerationError _errors; private readonly IPublishedValueFallback _publishedValueFallback; private static readonly Regex s_usingRegex = new Regex("^using(.*);", RegexOptions.Compiled | RegexOptions.Multiline); private static readonly Regex s_aattrRegex = new Regex("^\\[assembly:(.*)\\]", RegexOptions.Compiled | RegexOptions.Multiline); public PureLiveModelFactory( Lazy umbracoServices, IProfilingLogger profilingLogger, ILogger logger, IOptions config, IHostingEnvironment hostingEnvironment, IApplicationShutdownRegistry hostingLifetime, IPublishedValueFallback publishedValueFallback) { _umbracoServices = umbracoServices; _profilingLogger = profilingLogger; _logger = logger; _config = config.Value; _hostingEnvironment = hostingEnvironment; _hostingLifetime = hostingLifetime; _publishedValueFallback = publishedValueFallback; _errors = new ModelsGenerationError(config, _hostingEnvironment); _ver = 1; // zero is for when we had no version _skipver = -1; // nothing to skip if (!hostingEnvironment.IsHosted) { return; } var modelsDirectory = _config.ModelsDirectoryAbsolute(_hostingEnvironment); if (!Directory.Exists(modelsDirectory)) { Directory.CreateDirectory(modelsDirectory); } // BEWARE! if the watcher is not properly released then for some reason the // BuildManager will start confusing types - using a 'registered object' here // though we should probably plug into Umbraco's MainDom - which is internal _hostingLifetime.RegisterObject(this); _watcher = new FileSystemWatcher(modelsDirectory); _watcher.Changed += WatcherOnChanged; _watcher.EnableRaisingEvents = true; // get it here, this need to be fast _debugLevel = _config.DebugLevel; } private UmbracoServices UmbracoServices => _umbracoServices.Value; /// public object SyncRoot { get; } = new object(); // gets the RoslynCompiler private RoslynCompiler RoslynCompiler { get { if (_roslynCompiler != null) { return _roslynCompiler; } _roslynCompiler = new RoslynCompiler(System.Runtime.Loader.AssemblyLoadContext.All.SelectMany(x => x.Assemblies)); return _roslynCompiler; } } /// public bool Enabled => _config.Enable; /// public void Refresh() { ResetModels(); EnsureModels(); } public IPublishedElement CreateModel(IPublishedElement element) { // get models, rebuilding them if needed Dictionary infos = EnsureModels()?.ModelInfos; if (infos == null) { return element; } // be case-insensitive var contentTypeAlias = element.ContentType.Alias; // lookup model constructor (else null) infos.TryGetValue(contentTypeAlias, out var info); // create model return info == null ? element : info.Ctor(element, _publishedValueFallback); } // this runs only once the factory is ready // NOT when building models public Type MapModelType(Type type) { Infos infos = EnsureModels(); return ModelType.Map(type, infos.ModelTypeMap); } // this runs only once the factory is ready // NOT when building models public IList CreateModelList(string alias) { Infos infos = EnsureModels(); // fail fast if (infos == null) { return new List(); } if (!infos.ModelInfos.TryGetValue(alias, out ModelInfo modelInfo)) { return new List(); } Func ctor = modelInfo.ListCtor; if (ctor != null) { return ctor(); } Type listType = typeof(List<>).MakeGenericType(modelInfo.ModelType); ctor = modelInfo.ListCtor = ReflectionUtilities.EmitConstructor>(declaring: listType); return ctor(); } /// public void Reset() { if (_config.Enable) { ResetModels(); } } #region Compilation // deadlock note // // when RazorBuildProvider_CodeGenerationStarted runs, the thread has Monitor.Enter-ed the BuildManager // singleton instance, through a call to CompilationLock.GetLock in BuildManager.GetVPathBuildResultInternal, // and now wants to lock _locker. // when EnsureModels runs, the thread locks _locker and then wants BuildManager to compile, which in turns // requires that the BuildManager can Monitor.Enter-ed itself. // so: // // T1 - needs to ensure models, locks _locker // T2 - needs to compile a view, locks BuildManager // hits RazorBuildProvider_CodeGenerationStarted // wants to lock _locker, wait // T1 - needs to compile models, using BuildManager // wants to lock itself, wait // // // until ASP.NET kills the long-running request (thread abort) // // problem is, we *want* to suspend views compilation while the models assembly is being changed else we // end up with views compiled and cached with the old assembly, while models come from the new assembly, // which gives more YSOD. so we *have* to lock _locker in RazorBuildProvider_CodeGenerationStarted. // // one "easy" solution consists in locking the BuildManager *before* _locker in EnsureModels, thus ensuring // we always lock in the same order, and getting rid of deadlocks - but that requires having access to the // current BuildManager instance, which is BuildManager.TheBuildManager, which is an internal property. // // well, that's what we are doing in this class' TheBuildManager property, using reflection. // private void RazorBuildProvider_CodeGenerationStarted(object sender, EventArgs e) // { // try // { // _locker.EnterReadLock(); // // // just be safe - can happen if the first view is not an Umbraco view, // // or if something went wrong and we don't have an assembly at all // if (_modelsAssembly == null) return; // // if (_debugLevel > 0) // _logger.Debug("RazorBuildProvider.CodeGenerationStarted"); // if (!(sender is RazorBuildProvider provider)) return; // // // add the assembly, and add a dependency to a text file that will change on each // // compilation as in some environments (could not figure which/why) the BuildManager // // would not re-compile the views when the models assembly is rebuilt. // provider.AssemblyBuilder.AddAssemblyReference(_modelsAssembly); // provider.AddVirtualPathDependency(ProjVirt); // } // finally // { // if (_locker.IsReadLockHeld) // _locker.ExitReadLock(); // } // } // tells the factory that it should build a new generation of models private void ResetModels() { _logger.LogDebug("Resetting models."); try { _locker.EnterWriteLock(); _hasModels = false; _pendingRebuild = true; var modelsDirectory = _config.ModelsDirectoryAbsolute(_hostingEnvironment); if (!Directory.Exists(modelsDirectory)) { Directory.CreateDirectory(modelsDirectory); } // clear stuff var modelsHashFile = Path.Combine(modelsDirectory, "models.hash"); var dllPathFile = Path.Combine(modelsDirectory, "all.dll.path"); if (File.Exists(dllPathFile)) { File.Delete(dllPathFile); } if (File.Exists(modelsHashFile)) { File.Delete(modelsHashFile); } } finally { if (_locker.IsWriteLockHeld) { _locker.ExitWriteLock(); } } } // ensure that the factory is running with the lastest generation of models internal Infos EnsureModels() { if (_debugLevel > 0) { _logger.LogDebug("Ensuring models."); } // don't use an upgradeable lock here because only 1 thread at a time could enter it try { _locker.EnterReadLock(); if (_hasModels) { return _infos; } } finally { if (_locker.IsReadLockHeld) { _locker.ExitReadLock(); } } var roslynLocked = false; try { // TODO: DO NOT LOCK ON RoslynCompiler // always take the BuildManager lock *before* taking the _locker lock // to avoid possible deadlock situations (see notes above) Monitor.Enter(RoslynCompiler, ref roslynLocked); _locker.EnterUpgradeableReadLock(); if (_hasModels) { return _infos; } _locker.EnterWriteLock(); // we don't have models, // either they haven't been loaded from the cache yet // or they have been reseted and are pending a rebuild using (_profilingLogger.DebugDuration("Get models.", "Got models.")) { try { Assembly assembly = GetModelsAssembly(_pendingRebuild); // the one below can be used to simulate an issue with BuildManager, ie it will register // the models with the factory but NOT with the BuildManager, which will not recompile views. // this is for U4-8043 which is an obvious issue but I cannot replicate // _modelsAssembly = _modelsAssembly ?? assembly; IEnumerable types = assembly.ExportedTypes.Where(x => x.Inherits() || x.Inherits()); _infos = RegisterModels(types); _errors.Clear(); } catch (Exception e) { try { _logger.LogError(e, "Failed to build models."); _logger.LogWarning("Running without models."); // be explicit _errors.Report("Failed to build PureLive models.", e); } finally { _infos = new Infos { ModelInfos = null, ModelTypeMap = new Dictionary() }; } } // don't even try again _hasModels = true; } return _infos; } finally { if (_locker.IsWriteLockHeld) { _locker.ExitWriteLock(); } if (_locker.IsUpgradeableReadLockHeld) { _locker.ExitUpgradeableReadLock(); } if (roslynLocked) { Monitor.Exit(RoslynCompiler); } } } private Assembly ReloadAssembly(string pathToAssembly) { // If there's a current AssemblyLoadContext, unload it before creating a new one. if (!(_currentAssemblyLoadContext is null)) { _currentAssemblyLoadContext.Unload(); GC.Collect(); GC.WaitForPendingFinalizers(); } // We must create a new assembly load context // as long as theres a reference to the assembly load context we can't delete the assembly it loaded _currentAssemblyLoadContext = new UmbracoAssemblyLoadContext(); // Use filestream to load in the new assembly, otherwise it'll be locked // See https://www.strathweb.com/2019/01/collectible-assemblies-in-net-core-3-0/ for more info using (var fs = new FileStream(pathToAssembly, FileMode.Open, FileAccess.Read)) { return _currentAssemblyLoadContext.LoadFromStream(fs); } } private Assembly GetModelsAssembly(bool forceRebuild) { var modelsDirectory = _config.ModelsDirectoryAbsolute(_hostingEnvironment); if (!Directory.Exists(modelsDirectory)) { Directory.CreateDirectory(modelsDirectory); } IList typeModels = UmbracoServices.GetAllTypes(); var currentHash = TypeModelHasher.Hash(typeModels); var modelsHashFile = Path.Combine(modelsDirectory, "models.hash"); var modelsSrcFile = Path.Combine(modelsDirectory, "models.generated.cs"); var projFile = Path.Combine(modelsDirectory, "all.generated.cs"); var dllPathFile = Path.Combine(modelsDirectory, "all.dll.path"); // caching the generated models speeds up booting // currentHash hashes both the types & the user's partials if (!forceRebuild) { _logger.LogDebug("Looking for cached models."); if (File.Exists(modelsHashFile) && File.Exists(projFile)) { var cachedHash = File.ReadAllText(modelsHashFile); if (currentHash != cachedHash) { _logger.LogDebug("Found obsolete cached models."); forceRebuild = true; } // else cachedHash matches currentHash, we can try to load an existing dll } else { _logger.LogDebug("Could not find cached models."); forceRebuild = true; } } Assembly assembly; if (!forceRebuild) { // try to load the dll directly (avoid rebuilding) // // ensure that the .dll file does not have a corresponding .dll.delete file // as that would mean the the .dll file is going to be deleted and should not // be re-used - that should not happen in theory, but better be safe if (File.Exists(dllPathFile)) { var dllPath = File.ReadAllText(dllPathFile); _logger.LogDebug($"Cached models dll at {dllPath}."); if (File.Exists(dllPath) && !File.Exists(dllPath + ".delete")) { assembly = ReloadAssembly(dllPath); ModelsBuilderAssemblyAttribute attr = assembly.GetCustomAttribute(); if (attr != null && attr.PureLive && attr.SourceHash == currentHash) { // if we were to resume at that revision, then _ver would keep increasing // and that is probably a bad idea - so, we'll always rebuild starting at // ver 1, but we remember we want to skip that one - so we never end up // with the "same but different" version of the assembly in memory _skipver = assembly.GetName().Version.Revision; _logger.LogDebug("Loading cached models (dll)."); return assembly; } _logger.LogDebug("Cached models dll cannot be loaded (invalid assembly)."); } else if (!File.Exists(dllPath)) { _logger.LogDebug("Cached models dll does not exist."); } else if (File.Exists(dllPath + ".delete")) { _logger.LogDebug("Cached models dll is marked for deletion."); } else { _logger.LogDebug("Cached models dll cannot be loaded (why?)."); } } // must reset the version in the file else it would keep growing // loading cached modules only happens when the app restarts var text = File.ReadAllText(projFile); Match match = s_assemblyVersionRegex.Match(text); if (match.Success) { text = text.Replace(match.Value, "AssemblyVersion(\"0.0.0." + _ver + "\")"); File.WriteAllText(projFile, text); } _ver++; try { var assemblyPath = GetOutputAssemblyPath(currentHash); RoslynCompiler.CompileToFile(projFile, assemblyPath); assembly = ReloadAssembly(assemblyPath); File.WriteAllText(dllPathFile, assembly.Location); File.WriteAllText(modelsHashFile, currentHash); TryDeleteUnusedAssemblies(dllPathFile); } catch { ClearOnFailingToCompile(dllPathFile, modelsHashFile, projFile); throw; } _logger.LogDebug("Loading cached models (source)."); return assembly; } // need to rebuild _logger.LogDebug("Rebuilding models."); // generate code, save var code = GenerateModelsCode(typeModels); // add extra attributes, // PureLiveAssembly helps identifying Assemblies that contain PureLive models // AssemblyVersion is so that we have a different version for each rebuild var ver = _ver == _skipver ? ++_ver : _ver; _ver++; string mbAssemblyDirective = $@"[assembly:ModelsBuilderAssembly(PureLive = true, SourceHash = ""{currentHash}"")] [assembly:System.Reflection.AssemblyVersion(""0.0.0.{ver}"")]"; code = code.Replace("//ASSATTR", mbAssemblyDirective); File.WriteAllText(modelsSrcFile, code); // generate proj, save var projFiles = new Dictionary { { "models.generated.cs", code } }; var proj = GenerateModelsProj(projFiles); File.WriteAllText(projFile, proj); // compile and register try { var assemblyPath = GetOutputAssemblyPath(currentHash); RoslynCompiler.CompileToFile(projFile, assemblyPath); assembly = ReloadAssembly(assemblyPath); File.WriteAllText(dllPathFile, assemblyPath); File.WriteAllText(modelsHashFile, currentHash); TryDeleteUnusedAssemblies(dllPathFile); } catch { ClearOnFailingToCompile(dllPathFile, modelsHashFile, projFile); throw; } _logger.LogDebug("Done rebuilding."); return assembly; } private void TryDeleteUnusedAssemblies(string dllPathFile) { if (File.Exists(dllPathFile)) { var dllPath = File.ReadAllText(dllPathFile); DirectoryInfo dirInfo = new DirectoryInfo(dllPath).Parent; IEnumerable files = dirInfo.GetFiles().Where(f => f.FullName != dllPath); foreach (FileInfo file in files) { try { File.Delete(file.FullName); } catch (UnauthorizedAccessException) { // The file is in use, we'll try again next time... // This shouldn't happen anymore. } } } } private string GetOutputAssemblyPath(string currentHash) { var dirInfo = new DirectoryInfo(Path.Combine(_config.ModelsDirectoryAbsolute(_hostingEnvironment), "Compiled")); if (!dirInfo.Exists) { Directory.CreateDirectory(dirInfo.FullName); } return Path.Combine(dirInfo.FullName, $"generated.cs{currentHash}.dll"); } private void ClearOnFailingToCompile(string dllPathFile, string modelsHashFile, string projFile) { _logger.LogDebug("Failed to compile."); // the dll file reference still points to the previous dll, which is obsolete // now and will be deleted by ASP.NET eventually, so better clear that reference. // also touch the proj file to force views to recompile - don't delete as it's // useful to have the source around for debugging. try { if (File.Exists(dllPathFile)) { File.Delete(dllPathFile); } if (File.Exists(modelsHashFile)) { File.Delete(modelsHashFile); } if (File.Exists(projFile)) { File.SetLastWriteTime(projFile, DateTime.Now); } } catch { /* enough */ } } private static Infos RegisterModels(IEnumerable types) { Type[] ctorArgTypes = new[] { typeof(IPublishedElement), typeof(IPublishedValueFallback) }; var modelInfos = new Dictionary(StringComparer.InvariantCultureIgnoreCase); var map = new Dictionary(); foreach (Type type in types) { ConstructorInfo constructor = null; Type parameterType = null; foreach (ConstructorInfo ctor in type.GetConstructors()) { ParameterInfo[] parms = ctor.GetParameters(); if (parms.Length == 2 && typeof(IPublishedElement).IsAssignableFrom(parms[0].ParameterType) && typeof(IPublishedValueFallback).IsAssignableFrom(parms[1].ParameterType)) { if (constructor != null) { throw new InvalidOperationException($"Type {type.FullName} has more than one public constructor with one argument of type, or implementing, IPropertySet."); } constructor = ctor; parameterType = parms[0].ParameterType; } } if (constructor == null) { throw new InvalidOperationException($"Type {type.FullName} is missing a public constructor with one argument of type, or implementing, IPropertySet."); } PublishedModelAttribute attribute = type.GetCustomAttribute(false); var typeName = attribute == null ? type.Name : attribute.ContentTypeAlias; if (modelInfos.TryGetValue(typeName, out var modelInfo)) { throw new InvalidOperationException($"Both types {type.FullName} and {modelInfo.ModelType.FullName} want to be a model type for content type with alias \"{typeName}\"."); } // TODO: use Core's ReflectionUtilities.EmitCtor !! // Yes .. DynamicMethod is uber slow // TODO: But perhaps https://docs.microsoft.com/en-us/dotnet/api/system.reflection.emit.constructorbuilder?view=netcore-3.1 is better still? // See CtorInvokeBenchmarks var meth = new DynamicMethod(string.Empty, typeof(IPublishedElement), ctorArgTypes, type.Module, true); ILGenerator gen = meth.GetILGenerator(); gen.Emit(OpCodes.Ldarg_0); gen.Emit(OpCodes.Ldarg_1); gen.Emit(OpCodes.Newobj, constructor); gen.Emit(OpCodes.Ret); var func = (Func)meth.CreateDelegate(typeof(Func)); modelInfos[typeName] = new ModelInfo { ParameterType = parameterType, Ctor = func, ModelType = type }; map[typeName] = type; } return new Infos { ModelInfos = modelInfos.Count > 0 ? modelInfos : null, ModelTypeMap = map }; } private string GenerateModelsCode(IList typeModels) { var modelsDirectory = _config.ModelsDirectoryAbsolute(_hostingEnvironment); if (!Directory.Exists(modelsDirectory)) { Directory.CreateDirectory(modelsDirectory); } foreach (var file in Directory.GetFiles(modelsDirectory, "*.generated.cs")) { File.Delete(file); } var builder = new TextBuilder(_config, typeModels); var codeBuilder = new StringBuilder(); builder.Generate(codeBuilder, builder.GetModelsToGenerate()); var code = codeBuilder.ToString(); return code; } private static string GenerateModelsProj(IDictionary files) { // ideally we would generate a CSPROJ file but then we'd need a BuildProvider for csproj // trying to keep things simple for the time being, just write everything to one big file // group all 'using' at the top of the file (else fails) var usings = new List(); foreach (string k in files.Keys.ToList()) { files[k] = s_usingRegex.Replace(files[k], m => { usings.Add(m.Groups[1].Value); return string.Empty; }); } // group all '[assembly:...]' at the top of the file (else fails) var aattrs = new List(); foreach (string k in files.Keys.ToList()) { files[k] = s_aattrRegex.Replace(files[k], m => { aattrs.Add(m.Groups[1].Value); return string.Empty; }); } var text = new StringBuilder(); foreach (var u in usings.Distinct()) { text.Append("using "); text.Append(u); text.Append(";\r\n"); } foreach (var a in aattrs) { text.Append("[assembly:"); text.Append(a); text.Append("]\r\n"); } text.Append("\r\n\r\n"); foreach (KeyValuePair f in files) { text.Append("// FILE: "); text.Append(f.Key); text.Append("\r\n\r\n"); text.Append(f.Value); text.Append("\r\n\r\n\r\n"); } text.Append("// EOF\r\n"); return text.ToString(); } private void WatcherOnChanged(object sender, FileSystemEventArgs args) { var changed = args.Name; // don't reset when our files change because we are building! // // comment it out, and always ignore our files, because it seems that some // race conditions can occur on slow Cloud filesystems and then we keep // rebuilding //if (_building && OurFiles.Contains(changed)) //{ // //_logger.LogInformation("Ignoring files self-changes."); // return; //} // always ignore our own file changes if (s_ourFiles.Contains(changed)) return; _logger.LogInformation("Detected files changes."); lock (SyncRoot) // don't reset while being locked ResetModels(); } public void Stop(bool immediate) { _watcher.EnableRaisingEvents = false; _watcher.Dispose(); _hostingLifetime.UnregisterObject(this); } internal class Infos { public Dictionary ModelTypeMap { get; set; } public Dictionary ModelInfos { get; set; } } internal class ModelInfo { public Type ParameterType { get; set; } public Func Ctor { get; set; } public Type ModelType { get; set; } public Func ListCtor { get; set; } } #endregion } }