using System.ComponentModel.DataAnnotations; using System.Globalization; using System.IO.Compression; using System.Xml.Linq; using Microsoft.Extensions.Options; using NPoco; using Umbraco.Cms.Core; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Hosting; using Umbraco.Cms.Core.IO; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Packaging; using Umbraco.Cms.Core.Services; using Umbraco.Cms.Infrastructure.Persistence.Dtos; using Umbraco.Extensions; using File = System.IO.File; namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement; /// public class CreatedPackageSchemaRepository : ICreatedPackagesRepository { private readonly IContentService _contentService; private readonly IContentTypeService _contentTypeService; private readonly string _createdPackagesFolderPath; private readonly IDataTypeService _dataTypeService; private readonly IFileService _fileService; private readonly FileSystems _fileSystems; private readonly IHostingEnvironment _hostingEnvironment; private readonly ILocalizationService _localizationService; private readonly IMacroService _macroService; private readonly MediaFileManager _mediaFileManager; private readonly IMediaService _mediaService; private readonly IMediaTypeService _mediaTypeService; private readonly IEntityXmlSerializer _serializer; private readonly string _tempFolderPath; private readonly IUmbracoDatabase? _umbracoDatabase; private readonly PackageDefinitionXmlParser _xmlParser; /// /// Initializes a new instance of the class. /// public CreatedPackageSchemaRepository( IUmbracoDatabaseFactory umbracoDatabaseFactory, IHostingEnvironment hostingEnvironment, IOptions globalSettings, FileSystems fileSystems, IEntityXmlSerializer serializer, IDataTypeService dataTypeService, ILocalizationService localizationService, IFileService fileService, IMediaService mediaService, IMediaTypeService mediaTypeService, IContentService contentService, MediaFileManager mediaFileManager, IMacroService macroService, IContentTypeService contentTypeService, string? mediaFolderPath = null, string? tempFolderPath = null) { _umbracoDatabase = umbracoDatabaseFactory.CreateDatabase(); _hostingEnvironment = hostingEnvironment; _fileSystems = fileSystems; _serializer = serializer; _dataTypeService = dataTypeService; _localizationService = localizationService; _fileService = fileService; _mediaService = mediaService; _mediaTypeService = mediaTypeService; _contentService = contentService; _mediaFileManager = mediaFileManager; _macroService = macroService; _contentTypeService = contentTypeService; _xmlParser = new PackageDefinitionXmlParser(); _createdPackagesFolderPath = mediaFolderPath ?? Constants.SystemDirectories.CreatedPackages; _tempFolderPath = tempFolderPath ?? Constants.SystemDirectories.TempData + "/PackageFiles"; } public IEnumerable GetAll() { Sql query = new Sql(_umbracoDatabase!.SqlContext) .Select() .From() .OrderBy(x => x.Id); var packageDefinitions = new List(); List xmlSchemas = _umbracoDatabase.Fetch(query); foreach (CreatedPackageSchemaDto packageSchema in xmlSchemas) { var packageDefinition = _xmlParser.ToPackageDefinition(XElement.Parse(packageSchema.Value)); if (packageDefinition is not null) { packageDefinition.Id = packageSchema.Id; packageDefinition.Name = packageSchema.Name; packageDefinition.PackageId = packageSchema.PackageId; packageDefinitions.Add(packageDefinition); } } return packageDefinitions; } public PackageDefinition? GetById(int id) { Sql query = new Sql(_umbracoDatabase!.SqlContext) .Select() .From() .Where(x => x.Id == id); List schemaDtos = _umbracoDatabase.Fetch(query); if (schemaDtos.IsCollectionEmpty()) { return null; } CreatedPackageSchemaDto packageSchema = schemaDtos.First(); var packageDefinition = _xmlParser.ToPackageDefinition(XElement.Parse(packageSchema.Value)); if (packageDefinition is not null) { packageDefinition.Id = packageSchema.Id; packageDefinition.Name = packageSchema.Name; packageDefinition.PackageId = packageSchema.PackageId; } return packageDefinition; } public void Delete(int id) { // Delete package snapshot PackageDefinition? packageDef = GetById(id); if (File.Exists(packageDef?.PackagePath)) { File.Delete(packageDef.PackagePath); } Sql query = new Sql(_umbracoDatabase!.SqlContext) .Delete() .Where(x => x.Id == id); _umbracoDatabase.Execute(query); } public bool SavePackage(PackageDefinition? definition) { if (definition == null) { throw new NullReferenceException("PackageDefinition cannot be null when saving"); } if (string.IsNullOrEmpty(definition.Name) || definition.PackagePath == null) { return false; } // Ensure it's valid ValidatePackage(definition); if (definition.Id == default) { // Create dto from definition var dto = new CreatedPackageSchemaDto { Name = definition.Name, Value = _xmlParser.ToXml(definition).ToString(), UpdateDate = DateTime.Now, PackageId = Guid.NewGuid(), }; // Set the ids, we have to save in database first to get the Id _umbracoDatabase!.Insert(dto); definition.Id = dto.Id; } // Save snapshot locally, we do this to the updated packagePath ExportPackage(definition); // Create dto from definition var updatedDto = new CreatedPackageSchemaDto { Name = definition.Name, Value = _xmlParser.ToXml(definition).ToString(), Id = definition.Id, PackageId = definition.PackageId, UpdateDate = DateTime.Now, }; _umbracoDatabase?.Update(updatedDto); return true; } public string ExportPackage(PackageDefinition definition) { // Ensure it's valid ValidatePackage(definition); // Create a folder for building this package var temporaryPath = _hostingEnvironment.MapPathContentRoot(Path.Combine(_tempFolderPath, Guid.NewGuid().ToString())); Directory.CreateDirectory(temporaryPath); try { // Init package file XDocument compiledPackageXml = CreateCompiledPackageXml(out XElement root); // Info section root.Add(GetPackageInfoXml(definition)); PackageDocumentsAndTags(definition, root); PackageDocumentTypes(definition, root); PackageMediaTypes(definition, root); PackageTemplates(definition, root); PackageStylesheets(definition, root); PackageStaticFiles(definition.Scripts, root, "Scripts", "Script", _fileSystems.ScriptsFileSystem!); PackageStaticFiles(definition.PartialViews, root, "PartialViews", "View", _fileSystems.PartialViewsFileSystem!); PackageMacros(definition, root); PackageDictionaryItems(definition, root); PackageLanguages(definition, root); PackageDataTypes(definition, root); Dictionary mediaFiles = PackageMedia(definition, root); string fileName; string tempPackagePath; if (mediaFiles.Count > 0) { fileName = "package.zip"; tempPackagePath = Path.Combine(temporaryPath, fileName); using (FileStream fileStream = File.OpenWrite(tempPackagePath)) using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) { ZipArchiveEntry packageXmlEntry = archive.CreateEntry("package.xml"); using (Stream entryStream = packageXmlEntry.Open()) { compiledPackageXml.Save(entryStream); } foreach (KeyValuePair mediaFile in mediaFiles) { var entryPath = $"media{mediaFile.Key.EnsureStartsWith('/')}"; ZipArchiveEntry mediaEntry = archive.CreateEntry(entryPath); using (Stream entryStream = mediaEntry.Open()) using (mediaFile.Value) { mediaFile.Value.Seek(0, SeekOrigin.Begin); mediaFile.Value.CopyTo(entryStream); } } } } else { fileName = "package.xml"; tempPackagePath = Path.Combine(temporaryPath, fileName); using (FileStream fileStream = File.OpenWrite(tempPackagePath)) { compiledPackageXml.Save(fileStream); } } var directoryName = _hostingEnvironment.MapPathContentRoot(Path.Combine( _createdPackagesFolderPath, definition.Name.Replace(' ', '_'))); Directory.CreateDirectory(directoryName); var finalPackagePath = Path.Combine(directoryName, fileName); // Clean existing files foreach (var packagePath in new[] { definition.PackagePath, finalPackagePath }) { if (File.Exists(packagePath)) { File.Delete(packagePath); } } // Move to final package path File.Move(tempPackagePath, finalPackagePath); definition.PackagePath = finalPackagePath; return finalPackagePath; } finally { // Clean up Directory.Delete(temporaryPath, true); } } private static XElement GetPackageInfoXml(PackageDefinition definition) { var info = new XElement("info"); // Package info var package = new XElement("package"); package.Add(new XElement("name", definition.Name)); info.Add(package); return info; } private XDocument CreateCompiledPackageXml(out XElement root) { root = new XElement("umbPackage"); var compiledPackageXml = new XDocument(root); return compiledPackageXml; } private void ValidatePackage(PackageDefinition definition) { // Ensure it's valid var context = new ValidationContext(definition, null, null); var results = new List(); var isValid = Validator.TryValidateObject(definition, context, results); if (!isValid) { throw new InvalidOperationException("Validation failed, there is invalid data on the model: " + string.Join(", ", results.Select(x => x.ErrorMessage))); } } private void PackageDataTypes(PackageDefinition definition, XContainer root) { var dataTypes = new XElement("DataTypes"); foreach (var dtId in definition.DataTypes) { if (!int.TryParse(dtId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt)) { continue; } IDataType? dataType = _dataTypeService.GetDataType(outInt); if (dataType == null) { continue; } dataTypes.Add(_serializer.Serialize(dataType)); } root.Add(dataTypes); } private void PackageLanguages(PackageDefinition definition, XContainer root) { var languages = new XElement("Languages"); foreach (var langId in definition.Languages) { if (!int.TryParse(langId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt)) { continue; } ILanguage? lang = _localizationService.GetLanguageById(outInt); if (lang == null) { continue; } languages.Add(_serializer.Serialize(lang)); } root.Add(languages); } private void PackageDictionaryItems(PackageDefinition definition, XContainer root) { var rootDictionaryItems = new XElement("DictionaryItems"); var items = new Dictionary(); foreach (var dictionaryId in definition.DictionaryItems) { if (!int.TryParse(dictionaryId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt)) { continue; } IDictionaryItem? di = _localizationService.GetDictionaryItemById(outInt); if (di == null) { continue; } items[di.Key] = (di, _serializer.Serialize(di, false)); } // organize them in hierarchy ... var itemCount = items.Count; var processed = new Dictionary(); while (processed.Count < itemCount) { foreach (Guid key in items.Keys.ToList()) { (IDictionaryItem dictionaryItem, XElement serializedDictionaryValue) = items[key]; if (!dictionaryItem.ParentId.HasValue) { // if it has no parent, its definitely just at the root AppendDictionaryElement(rootDictionaryItems, items, processed, key, serializedDictionaryValue); } else { if (processed.ContainsKey(dictionaryItem.ParentId.Value)) { // we've processed this parent element already so we can just append this xml child to it AppendDictionaryElement(processed[dictionaryItem.ParentId.Value], items, processed, key, serializedDictionaryValue); } else if (items.ContainsKey(dictionaryItem.ParentId.Value)) { // we know the parent exists in the dictionary but // we haven't processed it yet so we'll leave it for the next loop } else { // in this case, the parent of this item doesn't exist in our collection, we have no // choice but to add it to the root. AppendDictionaryElement(rootDictionaryItems, items, processed, key, serializedDictionaryValue); } } } } root.Add(rootDictionaryItems); static void AppendDictionaryElement( XElement rootDictionaryItems, Dictionary items, Dictionary processed, Guid key, XElement serializedDictionaryValue) { // track it processed.Add(key, serializedDictionaryValue); // append it rootDictionaryItems.Add(serializedDictionaryValue); // remove it so its not re-processed items.Remove(key); } } private void PackageMacros(PackageDefinition definition, XContainer root) { var packagedMacros = new List(); var macros = new XElement("Macros"); foreach (var macroId in definition.Macros) { if (!int.TryParse(macroId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt)) { continue; } XElement? macroXml = GetMacroXml(outInt, out IMacro? macro); if (macroXml is null) { continue; } macros.Add(macroXml); packagedMacros.Add(macro!); } root.Add(macros); // Get the partial views for macros and package those (exclude views outside of the default directory, e.g. App_Plugins\*\Views) IEnumerable views = packagedMacros .Where(x => x.MacroSource.StartsWith(Constants.SystemDirectories.MacroPartials)) .Select(x => x.MacroSource[Constants.SystemDirectories.MacroPartials.Length..].Replace('/', '\\')); PackageStaticFiles(views, root, "MacroPartialViews", "View", _fileSystems.MacroPartialsFileSystem!); } private void PackageStylesheets(PackageDefinition definition, XContainer root) { var stylesheetsXml = new XElement("Stylesheets"); foreach (var stylesheet in definition.Stylesheets) { if (stylesheet.IsNullOrWhiteSpace()) { continue; } XElement? xml = GetStylesheetXml(stylesheet, true); if (xml != null) { stylesheetsXml.Add(xml); } } root.Add(stylesheetsXml); } private void PackageStaticFiles( IEnumerable filePaths, XContainer root, string containerName, string elementName, IFileSystem fileSystem) { var scriptsXml = new XElement(containerName); foreach (var file in filePaths) { if (file.IsNullOrWhiteSpace()) { continue; } if (!fileSystem.FileExists(file)) { throw new InvalidOperationException("No file found with path " + file); } using Stream stream = fileSystem.OpenFile(file); using (var reader = new StreamReader(stream)) { var fileContents = reader.ReadToEnd(); scriptsXml.Add( new XElement( elementName, new XAttribute("path", file), new XCData(fileContents))); } } root.Add(scriptsXml); } private void PackageTemplates(PackageDefinition definition, XContainer root) { var templatesXml = new XElement("Templates"); foreach (var templateId in definition.Templates) { if (!int.TryParse(templateId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt)) { continue; } ITemplate? template = _fileService.GetTemplate(outInt); if (template == null) { continue; } templatesXml.Add(_serializer.Serialize(template)); } root.Add(templatesXml); } private void PackageDocumentTypes(PackageDefinition definition, XContainer root) { var contentTypes = new HashSet(); var docTypesXml = new XElement("DocumentTypes"); foreach (var dtId in definition.DocumentTypes) { if (!int.TryParse(dtId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt)) { continue; } IContentType? contentType = _contentTypeService.Get(outInt); if (contentType == null) { continue; } AddDocumentType(contentType, contentTypes); } foreach (IContentType contentType in contentTypes) { docTypesXml.Add(_serializer.Serialize(contentType)); } root.Add(docTypesXml); } private void PackageMediaTypes(PackageDefinition definition, XContainer root) { var mediaTypes = new HashSet(); var mediaTypesXml = new XElement("MediaTypes"); foreach (var mediaTypeId in definition.MediaTypes) { if (!int.TryParse(mediaTypeId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt)) { continue; } IMediaType? mediaType = _mediaTypeService.Get(outInt); if (mediaType == null) { continue; } AddMediaType(mediaType, mediaTypes); } foreach (IMediaType mediaType in mediaTypes) { mediaTypesXml.Add(_serializer.Serialize(mediaType)); } root.Add(mediaTypesXml); } private void PackageDocumentsAndTags(PackageDefinition definition, XContainer root) { // Documents and tags if (string.IsNullOrEmpty(definition.ContentNodeId) == false && int.TryParse( definition.ContentNodeId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var contentNodeId)) { if (contentNodeId > 0) { // load content from umbraco. IContent? content = _contentService.GetById(contentNodeId); if (content != null) { XElement contentXml = definition.ContentLoadChildNodes ? content.ToDeepXml(_serializer) : content.ToXml(_serializer); // Create the Documents/DocumentSet node root.Add( new XElement( "Documents", new XElement( "DocumentSet", new XAttribute("importMode", "root"), contentXml))); } } } } private Dictionary PackageMedia(PackageDefinition definition, XElement root) { var mediaStreams = new Dictionary(); // callback that occurs on each serialized media item void OnSerializedMedia(IMedia media, XElement xmlMedia) { // get the media file path and store that separately in the XML. // the media file path is different from the URL and is specifically // extracted using the property editor for this media file and the current media file system. Stream mediaStream = _mediaFileManager.GetFile(media, out var mediaFilePath); xmlMedia.Add(new XAttribute("mediaFilePath", mediaFilePath!)); // add the stream to our outgoing stream mediaStreams.Add(mediaFilePath!, mediaStream); } IEnumerable medias = _mediaService.GetByIds(definition.MediaUdis); var mediaXml = new XElement( "MediaItems", medias.Select(media => { XElement serializedMedia = _serializer.Serialize( media, definition.MediaLoadChildNodes, OnSerializedMedia); return new XElement("MediaSet", serializedMedia); })); root.Add(mediaXml); return mediaStreams; } /// /// Gets a macros xml node /// private XElement? GetMacroXml(int macroId, out IMacro? macro) { macro = _macroService.GetById(macroId); if (macro == null) { return null; } XElement xml = _serializer.Serialize(macro); return xml; } /// /// Converts a umbraco stylesheet to a package xml node /// /// The path of the stylesheet. /// if set to true [include properties]. private XElement? GetStylesheetXml(string path, bool includeProperties) { if (string.IsNullOrWhiteSpace(path)) { throw new ArgumentException("Value cannot be null or whitespace.", nameof(path)); } IStylesheet? stylesheet = _fileService.GetStylesheet(path); if (stylesheet == null) { return null; } return _serializer.Serialize(stylesheet, includeProperties); } private void AddDocumentType(IContentType dt, HashSet dtl) { if (dt.ParentId > 0) { IContentType? parent = _contentTypeService.Get(dt.ParentId); if (parent != null) { AddDocumentType(parent, dtl); } } if (!dtl.Contains(dt)) { dtl.Add(dt); } } private void AddMediaType(IMediaType mediaType, HashSet mediaTypes) { if (mediaType.ParentId > 0) { IMediaType? parent = _mediaTypeService.Get(mediaType.ParentId); if (parent != null) { AddMediaType(parent, mediaTypes); } } if (!mediaTypes.Contains(mediaType)) { mediaTypes.Add(mediaType); } } }