Umbraco-CMS/src/Umbraco.Infrastructure/Persistence/Repositories/Implement/CreatedPackageSchemaRepository.cs

using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Xml.Linq;
using Microsoft.Extensions.Options;
using NPoco;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Packaging;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Infrastructure.Persistence.Dtos;
using Umbraco.Extensions;
using File = System.IO.File;

namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement
{
    /// <inheritdoc />
    public class CreatedPackageSchemaRepository : ICreatedPackagesRepository
    {
        private readonly PackageDefinitionXmlParser _xmlParser;
        private readonly IUmbracoDatabase? _umbracoDatabase;
        private readonly IHostingEnvironment _hostingEnvironment;
        private readonly FileSystems _fileSystems;
        private readonly IEntityXmlSerializer _serializer;
        private readonly IDataTypeService _dataTypeService;
        private readonly ILocalizationService _localizationService;
        private readonly IFileService _fileService;
        private readonly IMediaService _mediaService;
        private readonly IMediaTypeService _mediaTypeService;
        private readonly IContentService _contentService;
        private readonly MediaFileManager _mediaFileManager;
        private readonly IMacroService _macroService;
        private readonly IContentTypeService _contentTypeService;
        private readonly string _tempFolderPath;
        private readonly string _createdPackagesFolderPath;

        /// <summary>
        /// Initializes a new instance of the <see cref="CreatedPackageSchemaRepository"/> class.
        /// </summary>
        public CreatedPackageSchemaRepository(
            IUmbracoDatabaseFactory umbracoDatabaseFactory,
            IHostingEnvironment hostingEnvironment,
            IOptions<GlobalSettings> globalSettings,
            FileSystems fileSystems,
            IEntityXmlSerializer serializer,
            IDataTypeService dataTypeService,
            ILocalizationService localizationService,
            IFileService fileService,
            IMediaService mediaService,
            IMediaTypeService mediaTypeService,
            IContentService contentService,
            MediaFileManager mediaFileManager,
            IMacroService macroService,
            IContentTypeService contentTypeService,
            string? mediaFolderPath = null,
            string? tempFolderPath = null)
        {
            _umbracoDatabase = umbracoDatabaseFactory.CreateDatabase();
            _hostingEnvironment = hostingEnvironment;
            _fileSystems = fileSystems;
            _serializer = serializer;
            _dataTypeService = dataTypeService;
            _localizationService = localizationService;
            _fileService = fileService;
            _mediaService = mediaService;
            _mediaTypeService = mediaTypeService;
            _contentService = contentService;
            _mediaFileManager = mediaFileManager;
            _macroService = macroService;
            _contentTypeService = contentTypeService;
            _xmlParser = new PackageDefinitionXmlParser();
            _createdPackagesFolderPath = mediaFolderPath ?? Constants.SystemDirectories.CreatedPackages;
            _tempFolderPath = tempFolderPath ?? Constants.SystemDirectories.TempData + "/PackageFiles";
        }

        public IEnumerable<PackageDefinition> GetAll()
        {
            Sql<ISqlContext> query = new Sql<ISqlContext>(_umbracoDatabase!.SqlContext)
                .Select<CreatedPackageSchemaDto>()
                .From<CreatedPackageSchemaDto>()
                .OrderBy<CreatedPackageSchemaDto>(x => x.Id);

            var packageDefinitions = new List<PackageDefinition>();

            List<CreatedPackageSchemaDto> xmlSchemas = _umbracoDatabase.Fetch<CreatedPackageSchemaDto>(query);
            foreach (CreatedPackageSchemaDto packageSchema in xmlSchemas)
            {
                var packageDefinition = _xmlParser.ToPackageDefinition(XElement.Parse(packageSchema.Value));
                if (packageDefinition is not null)
                {
                    packageDefinition.Id = packageSchema.Id;
                    packageDefinition.Name = packageSchema.Name;
                    packageDefinition.PackageId = packageSchema.PackageId;
                    packageDefinitions.Add(packageDefinition);
                }
            }

            return packageDefinitions;
        }

        public PackageDefinition? GetById(int id)
        {
            Sql<ISqlContext> query = new Sql<ISqlContext>(_umbracoDatabase!.SqlContext)
                .Select<CreatedPackageSchemaDto>()
                .From<CreatedPackageSchemaDto>()
                .Where<CreatedPackageSchemaDto>(x => x.Id == id);
            List<CreatedPackageSchemaDto> schemaDtos = _umbracoDatabase.Fetch<CreatedPackageSchemaDto>(query);

            if (schemaDtos.IsCollectionEmpty())
            {
                return null;
            }

            var packageSchema = schemaDtos.First();
            var packageDefinition = _xmlParser.ToPackageDefinition(XElement.Parse(packageSchema.Value));
            if (packageDefinition is not null)
            {
                packageDefinition.Id = packageSchema.Id;
                packageDefinition.Name = packageSchema.Name;
                packageDefinition.PackageId = packageSchema.PackageId;
            }

            return packageDefinition;
        }

        public void Delete(int id)
        {
            // Delete package snapshot
            var packageDef = GetById(id);
            if (File.Exists(packageDef?.PackagePath))
            {
                File.Delete(packageDef.PackagePath);
            }

            Sql<ISqlContext> query = new Sql<ISqlContext>(_umbracoDatabase!.SqlContext)
                .Delete<CreatedPackageSchemaDto>()
                .Where<CreatedPackageSchemaDto>(x => x.Id == id);

            _umbracoDatabase.Execute(query);
        }

        public bool SavePackage(PackageDefinition definition)
        {
            if (definition == null)
            {
                throw new NullReferenceException("PackageDefinition cannot be null when saving");
            }

            if (definition.Name == null || string.IsNullOrEmpty(definition.Name) || definition.PackagePath == null)
            {
                return false;
            }

            // Ensure it's valid
            ValidatePackage(definition);


            if (definition.Id == default)
            {
                // Create dto from definition
                var dto = new CreatedPackageSchemaDto()
                {
                    Name = definition.Name,
                    Value = _xmlParser.ToXml(definition).ToString(),
                    UpdateDate = DateTime.Now,
                    PackageId = Guid.NewGuid()
                };

                // Set the ids, we have to save in database first to get the Id
                _umbracoDatabase!.Insert(dto);
                definition.Id = dto.Id;
            }

            // Save snapshot locally, we do this to the updated packagePath
            ExportPackage(definition);
            // Create dto from definition
            var updatedDto = new CreatedPackageSchemaDto()
            {
                Name = definition.Name,
                Value = _xmlParser.ToXml(definition).ToString(),
                Id = definition.Id,
                PackageId = definition.PackageId,
                UpdateDate = DateTime.Now
            };
            _umbracoDatabase?.Update(updatedDto);

            return true;
        }

        public string ExportPackage(PackageDefinition definition)
        {
            // Ensure it's valid
            ValidatePackage(definition);

            // Create a folder for building this package
            var temporaryPath = _hostingEnvironment.MapPathContentRoot(Path.Combine(_tempFolderPath, Guid.NewGuid().ToString()));
            Directory.CreateDirectory(temporaryPath);

            try
            {
                // Init package file
                XDocument compiledPackageXml = CreateCompiledPackageXml(out XElement root);

                // Info section
                root.Add(GetPackageInfoXml(definition));

                PackageDocumentsAndTags(definition, root);
                PackageDocumentTypes(definition, root);
                PackageMediaTypes(definition, root);
                PackageTemplates(definition, root);
                PackageStylesheets(definition, root);
                PackageStaticFiles(definition.Scripts, root, "Scripts", "Script", _fileSystems.ScriptsFileSystem!);
                PackageStaticFiles(definition.PartialViews, root, "PartialViews", "View", _fileSystems.PartialViewsFileSystem!);
                PackageMacros(definition, root);
                PackageDictionaryItems(definition, root);
                PackageLanguages(definition, root);
                PackageDataTypes(definition, root);
                Dictionary<string, Stream> mediaFiles = PackageMedia(definition, root);

                string fileName;
                string tempPackagePath;
                if (mediaFiles.Count > 0)
                {
                    fileName = "package.zip";
                    tempPackagePath = Path.Combine(temporaryPath, fileName);
                    using (FileStream fileStream = File.OpenWrite(tempPackagePath))
                    using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
                    {
                        ZipArchiveEntry packageXmlEntry = archive.CreateEntry("package.xml");
                        using (Stream entryStream = packageXmlEntry.Open())
                        {
                            compiledPackageXml.Save(entryStream);
                        }

                        foreach (KeyValuePair<string, Stream> mediaFile in mediaFiles)
                        {
                            var entryPath = $"media{mediaFile.Key.EnsureStartsWith('/')}";
                            ZipArchiveEntry mediaEntry = archive.CreateEntry(entryPath);
                            using (Stream entryStream = mediaEntry.Open())
                            using (mediaFile.Value)
                            {
                                mediaFile.Value.Seek(0, SeekOrigin.Begin);
                                mediaFile.Value.CopyTo(entryStream);
                            }
                        }
                    }
                }
                else
                {
                    fileName = "package.xml";
                    tempPackagePath = Path.Combine(temporaryPath, fileName);

                    using (FileStream fileStream = File.OpenWrite(tempPackagePath))
                    {
                        compiledPackageXml.Save(fileStream);
                    }
                }

                var directoryName = _hostingEnvironment.MapPathContentRoot(Path.Combine(_createdPackagesFolderPath, definition.Name.Replace(' ', '_')));
                Directory.CreateDirectory(directoryName);

                var finalPackagePath = Path.Combine(directoryName, fileName);

                // Clean existing files
                foreach (var packagePath in new[]
                {
                    definition.PackagePath,
                    finalPackagePath
                })
                {
                    if (File.Exists(packagePath))
                    {
                        File.Delete(packagePath);
                    }
                }

                // Move to final package path
                File.Move(tempPackagePath, finalPackagePath);

                definition.PackagePath = finalPackagePath;

                return finalPackagePath;
            }
            finally
            {
                // Clean up
                Directory.Delete(temporaryPath, true);
            }
        }

        private XDocument CreateCompiledPackageXml(out XElement root)
        {
            root = new XElement("umbPackage");
            var compiledPackageXml = new XDocument(root);
            return compiledPackageXml;
        }

        private void ValidatePackage(PackageDefinition definition)
        {
            // Ensure it's valid
            var context = new ValidationContext(definition, serviceProvider: null, items: null);
            var results = new List<ValidationResult>();
            var isValid = Validator.TryValidateObject(definition, context, results);
            if (!isValid)
            {
                throw new InvalidOperationException("Validation failed, there is invalid data on the model: " +
                                                    string.Join(", ", results.Select(x => x.ErrorMessage)));
            }
        }

        private void PackageDataTypes(PackageDefinition definition, XContainer root)
        {
            var dataTypes = new XElement("DataTypes");
            foreach (var dtId in definition.DataTypes)
            {
                if (!int.TryParse(dtId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt))
                {
                    continue;
                }

                IDataType? dataType = _dataTypeService.GetDataType(outInt);
                if (dataType == null)
                {
                    continue;
                }

                dataTypes.Add(_serializer.Serialize(dataType));
            }

            root.Add(dataTypes);
        }

        private void PackageLanguages(PackageDefinition definition, XContainer root)
        {
            var languages = new XElement("Languages");
            foreach (var langId in definition.Languages)
            {
                if (!int.TryParse(langId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt))
                {
                    continue;
                }

                ILanguage? lang = _localizationService.GetLanguageById(outInt);
                if (lang == null)
                {
                    continue;
                }

                languages.Add(_serializer.Serialize(lang));
            }

            root.Add(languages);
        }

        private void PackageDictionaryItems(PackageDefinition definition, XContainer root)
        {
            var rootDictionaryItems = new XElement("DictionaryItems");
            var items = new Dictionary<Guid, (IDictionaryItem dictionaryItem, XElement serializedDictionaryValue)>();

            foreach (var dictionaryId in definition.DictionaryItems)
            {
                if (!int.TryParse(dictionaryId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt))
                {
                    continue;
                }

                IDictionaryItem? di = _localizationService.GetDictionaryItemById(outInt);

                if (di == null)
                {
                    continue;
                }

                items[di.Key] = (di, _serializer.Serialize(di, false));
            }

            // organize them in hierarchy ...
            var itemCount = items.Count;
            var processed = new Dictionary<Guid, XElement>();
            while (processed.Count < itemCount)
            {
                foreach (Guid key in items.Keys.ToList())
                {
                    (IDictionaryItem dictionaryItem, XElement serializedDictionaryValue) = items[key];

                    if (!dictionaryItem.ParentId.HasValue)
                    {
                        // if it has no parent, its definitely just at the root
                        AppendDictionaryElement(rootDictionaryItems, items, processed, key, serializedDictionaryValue);
                    }
                    else
                    {
                        if (processed.ContainsKey(dictionaryItem.ParentId.Value))
                        {
                            // we've processed this parent element already so we can just append this xml child to it
                            AppendDictionaryElement(processed[dictionaryItem.ParentId.Value], items, processed, key,
                                serializedDictionaryValue);
                        }
                        else if (items.ContainsKey(dictionaryItem.ParentId.Value))
                        {
                            // we know the parent exists in the dictionary but
                            // we haven't processed it yet so we'll leave it for the next loop
                            continue;
                        }
                        else
                        {
                            // in this case, the parent of this item doesn't exist in our collection, we have no
                            // choice but to add it to the root.
                            AppendDictionaryElement(rootDictionaryItems, items, processed, key,
                                serializedDictionaryValue);
                        }
                    }
                }
            }

            root.Add(rootDictionaryItems);

            static void AppendDictionaryElement(XElement rootDictionaryItems,
                Dictionary<Guid, (IDictionaryItem dictionaryItem, XElement serializedDictionaryValue)> items,
                Dictionary<Guid, XElement> processed, Guid key, XElement serializedDictionaryValue)
            {
                // track it
                processed.Add(key, serializedDictionaryValue);

                // append it
                rootDictionaryItems.Add(serializedDictionaryValue);

                // remove it so its not re-processed
                items.Remove(key);
            }
        }

        private void PackageMacros(PackageDefinition definition, XContainer root)
        {
            var packagedMacros = new List<IMacro>();
            var macros = new XElement("Macros");
            foreach (var macroId in definition.Macros)
            {
                if (!int.TryParse(macroId, NumberStyles.Integer, CultureInfo.InvariantCulture, out int outInt))
                {
                    continue;
                }

                XElement? macroXml = GetMacroXml(outInt, out IMacro? macro);
                if (macroXml is null)
                {
                    continue;
                }

                macros.Add(macroXml);
                packagedMacros.Add(macro!);
            }

            root.Add(macros);

            // Get the partial views for macros and package those (exclude views outside of the default directory, e.g. App_Plugins\*\Views)
            IEnumerable<string> views = packagedMacros
                .Where(x => x.MacroSource.StartsWith(Constants.SystemDirectories.MacroPartials))
                .Select(x =>
                    x.MacroSource.Substring(Constants.SystemDirectories.MacroPartials.Length).Replace('/', '\\'));
            PackageStaticFiles(views, root, "MacroPartialViews", "View", _fileSystems.MacroPartialsFileSystem!);
        }

        private void PackageStylesheets(PackageDefinition definition, XContainer root)
        {
            var stylesheetsXml = new XElement("Stylesheets");
            foreach (var stylesheet in definition.Stylesheets)
            {
                if (stylesheet.IsNullOrWhiteSpace())
                {
                    continue;
                }

                XElement? xml = GetStylesheetXml(stylesheet, true);
                if (xml != null)
                {
                    stylesheetsXml.Add(xml);
                }
            }

            root.Add(stylesheetsXml);
        }

        private void PackageStaticFiles(
            IEnumerable<string> filePaths,
            XContainer root,
            string containerName,
            string elementName,
            IFileSystem fileSystem)
        {
            var scriptsXml = new XElement(containerName);
            foreach (var file in filePaths)
            {
                if (file.IsNullOrWhiteSpace())
                {
                    continue;
                }

                if (!fileSystem.FileExists(file))
                {
                    throw new InvalidOperationException("No file found with path " + file);
                }

                using Stream? stream = fileSystem.OpenFile(file);
                if (stream is not null)
                {
                    using (var reader = new StreamReader(stream))
                    {
                        var fileContents = reader.ReadToEnd();
                        scriptsXml.Add(
                            new XElement(
                                elementName,
                                new XAttribute("path", file),
                                new XCData(fileContents)));
                    }
                }
            }

            root.Add(scriptsXml);
        }

        private void PackageTemplates(PackageDefinition definition, XContainer root)
        {
            var templatesXml = new XElement("Templates");
            foreach (var templateId in definition.Templates)
            {
                if (!int.TryParse(templateId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt))
                {
                    continue;
                }

                ITemplate? template = _fileService.GetTemplate(outInt);
                if (template == null)
                {
                    continue;
                }

                templatesXml.Add(_serializer.Serialize(template));
            }

            root.Add(templatesXml);
        }

        private void PackageDocumentTypes(PackageDefinition definition, XContainer root)
        {
            var contentTypes = new HashSet<IContentType>();
            var docTypesXml = new XElement("DocumentTypes");
            foreach (var dtId in definition.DocumentTypes)
            {
                if (!int.TryParse(dtId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt))
                {
                    continue;
                }

                IContentType? contentType = _contentTypeService.Get(outInt);
                if (contentType == null)
                {
                    continue;
                }

                AddDocumentType(contentType, contentTypes);
            }

            foreach (IContentType contentType in contentTypes)
            {
                docTypesXml.Add(_serializer.Serialize(contentType));
            }

            root.Add(docTypesXml);
        }

        private void PackageMediaTypes(PackageDefinition definition, XContainer root)
        {
            var mediaTypes = new HashSet<IMediaType>();
            var mediaTypesXml = new XElement("MediaTypes");
            foreach (var mediaTypeId in definition.MediaTypes)
            {
                if (!int.TryParse(mediaTypeId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var outInt))
                {
                    continue;
                }

                IMediaType? mediaType = _mediaTypeService.Get(outInt);
                if (mediaType == null)
                {
                    continue;
                }

                AddMediaType(mediaType, mediaTypes);
            }

            foreach (IMediaType mediaType in mediaTypes)
            {
                mediaTypesXml.Add(_serializer.Serialize(mediaType));
            }

            root.Add(mediaTypesXml);
        }

        private void PackageDocumentsAndTags(PackageDefinition definition, XContainer root)
        {
            // Documents and tags
            if (string.IsNullOrEmpty(definition.ContentNodeId) == false && int.TryParse(definition.ContentNodeId,
                    NumberStyles.Integer, CultureInfo.InvariantCulture, out var contentNodeId))
            {
                if (contentNodeId > 0)
                {
                    // load content from umbraco.
                    IContent? content = _contentService.GetById(contentNodeId);
                    if (content != null)
                    {
                        var contentXml = definition.ContentLoadChildNodes
                            ? content.ToDeepXml(_serializer)
                            : content.ToXml(_serializer);

                        // Create the Documents/DocumentSet node

                        root.Add(
                            new XElement(
                                "Documents",
                                new XElement(
                                    "DocumentSet",
                                    new XAttribute("importMode", "root"),
                                    contentXml)));
                    }
                }
            }
        }

        private Dictionary<string, Stream> PackageMedia(PackageDefinition definition, XElement root)
        {
            var mediaStreams = new Dictionary<string, Stream>();

            // callback that occurs on each serialized media item
            void OnSerializedMedia(IMedia media, XElement xmlMedia)
            {
                // get the media file path and store that separately in the XML.
                // the media file path is different from the URL and is specifically
                // extracted using the property editor for this media file and the current media file system.
                Stream? mediaStream = _mediaFileManager.GetFile(media, out var mediaFilePath);
                if (mediaStream != null)
                {
                    xmlMedia.Add(new XAttribute("mediaFilePath", mediaFilePath!));

                    // add the stream to our outgoing stream
                    mediaStreams.Add(mediaFilePath!, mediaStream);
                }
            }

            IEnumerable<IMedia> medias = _mediaService.GetByIds(definition.MediaUdis);

            var mediaXml = new XElement(
                "MediaItems",
                medias.Select(media =>
                {
                    XElement serializedMedia = _serializer.Serialize(
                        media,
                        definition.MediaLoadChildNodes,
                        OnSerializedMedia);

                    return new XElement("MediaSet", serializedMedia);
                }));

            root.Add(mediaXml);

            return mediaStreams;
        }

        /// <summary>
        /// Gets a macros xml node
        /// </summary>
        private XElement? GetMacroXml(int macroId, out IMacro? macro)
        {
            macro = _macroService.GetById(macroId);
            if (macro == null)
            {
                return null;
            }

            XElement xml = _serializer.Serialize(macro);
            return xml;
        }

        /// <summary>
        /// Converts a umbraco stylesheet to a package xml node
        /// </summary>
        /// <param name="path">The path of the stylesheet.</param>
        /// <param name="includeProperties">if set to <c>true</c> [include properties].</param>
        private XElement? GetStylesheetXml(string path, bool includeProperties)
        {
            if (string.IsNullOrWhiteSpace(path))
            {
                throw new ArgumentException("Value cannot be null or whitespace.", nameof(path));
            }

            IStylesheet? stylesheet = _fileService.GetStylesheet(path);
            if (stylesheet == null)
            {
                return null;
            }

            return _serializer.Serialize(stylesheet, includeProperties);
        }

        private void AddDocumentType(IContentType dt, HashSet<IContentType> dtl)
        {
            if (dt.ParentId > 0)
            {
                IContentType? parent = _contentTypeService.Get(dt.ParentId);
                if (parent != null)
                {
                    AddDocumentType(parent, dtl);
                }
            }

            if (!dtl.Contains(dt))
            {
                dtl.Add(dt);
            }
        }

        private void AddMediaType(IMediaType mediaType, HashSet<IMediaType> mediaTypes)
        {
            if (mediaType.ParentId > 0)
            {
                IMediaType? parent = _mediaTypeService.Get(mediaType.ParentId);
                if (parent != null)
                {
                    AddMediaType(parent, mediaTypes);
                }
            }

            if (!mediaTypes.Contains(mediaType))
            {
                mediaTypes.Add(mediaType);
            }
        }

        private static XElement GetPackageInfoXml(PackageDefinition definition)
        {
            var info = new XElement("info");

            // Package info
            var package = new XElement("package");
            package.Add(new XElement("name", definition.Name));
            info.Add(package);
            return info;
        }
    }
}