diff --git a/src/Umbraco.Core/Packaging/PackageMigrationResource.cs b/src/Umbraco.Core/Packaging/PackageMigrationResource.cs index 2f407df88f..b9c0e99552 100644 --- a/src/Umbraco.Core/Packaging/PackageMigrationResource.cs +++ b/src/Umbraco.Core/Packaging/PackageMigrationResource.cs @@ -12,17 +12,52 @@ namespace Umbraco.Cms.Core.Packaging { public static class PackageMigrationResource { - private static Stream GetEmbeddedPackageStream(Type planType) + private static Stream GetEmbeddedPackageZipStream(Type planType) { // lookup the embedded resource by convention Assembly currentAssembly = planType.Assembly; var fileName = $"{planType.Namespace}.package.zip"; Stream stream = currentAssembly.GetManifestResourceStream(fileName); + + return stream; + } + + public static XDocument GetEmbeddedPackageDataManifest(Type planType, out ZipArchive zipArchive) + { + XDocument packageXml; + var zipStream = GetEmbeddedPackageZipStream(planType); + if (zipStream is not null) + { + zipArchive = GetPackageDataManifest(zipStream, out packageXml); + return packageXml; + } + + zipArchive = null; + packageXml = GetEmbeddedPackageXmlDoc(planType); + return packageXml; + } + + public static XDocument GetEmbeddedPackageDataManifest(Type planType) + { + return GetEmbeddedPackageDataManifest(planType, out _); + } + + private static XDocument GetEmbeddedPackageXmlDoc(Type planType) + { + // lookup the embedded resource by convention + Assembly currentAssembly = planType.Assembly; + var fileName = $"{planType.Namespace}.package.xml"; + Stream stream = currentAssembly.GetManifestResourceStream(fileName); if (stream == null) { - throw new FileNotFoundException("Cannot find the embedded file.", fileName); + return null; } - return stream; + XDocument xml; + using (stream) + { + xml = XDocument.Load(stream); + } + return xml; } public static string GetEmbeddedPackageDataManifestHash(Type planType) @@ -30,17 +65,46 @@ namespace Umbraco.Cms.Core.Packaging // SEE: HashFromStreams in the benchmarks project for how fast this is. It will run // on every startup for every embedded package.zip. The bigger the zip, the more time it takes. // But it is still very fast ~303ms for a 100MB file. This will only be an issue if there are - // several very large package.zips. + // several very large package.zips. - using Stream stream = GetEmbeddedPackageStream(planType); - return stream.GetStreamHash(); + using Stream stream = GetEmbeddedPackageZipStream(planType); + + if (stream is not null) + { + return stream.GetStreamHash(); + } + + var xml = GetEmbeddedPackageXmlDoc(planType); + + if (xml is not null) + { + return xml.ToString(); + } + + throw new IOException("Missing embedded files for planType: " + planType); } - public static ZipArchive GetEmbeddedPackageDataManifest(Type planType, out XDocument packageXml) - => GetPackageDataManifest(GetEmbeddedPackageStream(planType), out packageXml); + public static bool TryGetEmbeddedPackageDataManifest(Type planType, out XDocument packageXml, out ZipArchive zipArchive) + { + var zipStream = GetEmbeddedPackageZipStream(planType); + if (zipStream is not null) + { + zipArchive = GetPackageDataManifest(zipStream, out packageXml); + return true; + } + + zipArchive = null; + packageXml = GetEmbeddedPackageXmlDoc(planType); + return packageXml is not null; + } public static ZipArchive GetPackageDataManifest(Stream packageZipStream, out XDocument packageXml) { + if (packageZipStream == null) + { + throw new ArgumentNullException(nameof(packageZipStream)); + } + var zip = new ZipArchive(packageZipStream, ZipArchiveMode.Read); ZipArchiveEntry packageXmlEntry = zip.GetEntry("package.xml"); if (packageXmlEntry == null) diff --git a/src/Umbraco.Core/Packaging/PackagesRepository.cs b/src/Umbraco.Core/Packaging/PackagesRepository.cs index ffc67663cc..a24890d5f2 100644 --- a/src/Umbraco.Core/Packaging/PackagesRepository.cs +++ b/src/Umbraco.Core/Packaging/PackagesRepository.cs @@ -209,29 +209,46 @@ namespace Umbraco.Cms.Core.Packaging PackageDataTypes(definition, root); Dictionary mediaFiles = PackageMedia(definition, root); - var tempPackagePath = temporaryPath + "/package.zip"; - - using (FileStream fileStream = File.OpenWrite(tempPackagePath)) - using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) + string fileName; + string tempPackagePath; + if (mediaFiles.Count > 0) { - ZipArchiveEntry packageXmlEntry = archive.CreateEntry("package.xml"); - using (Stream entryStream = packageXmlEntry.Open()) + fileName = "package.zip"; + tempPackagePath = Path.Combine(temporaryPath, fileName); + using (FileStream fileStream = File.OpenWrite(tempPackagePath)) + using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) { - compiledPackageXml.Save(entryStream); - } - - foreach (KeyValuePair mediaFile in mediaFiles) - { - var entryPath = $"media{mediaFile.Key.EnsureStartsWith('/')}"; - ZipArchiveEntry mediaEntry = archive.CreateEntry(entryPath); - using (Stream entryStream = mediaEntry.Open()) - using (mediaFile.Value) + ZipArchiveEntry packageXmlEntry = archive.CreateEntry("package.xml"); + using (Stream entryStream = packageXmlEntry.Open()) { - mediaFile.Value.Seek(0, SeekOrigin.Begin); - mediaFile.Value.CopyTo(entryStream); + compiledPackageXml.Save(entryStream); + } + + foreach (KeyValuePair mediaFile in mediaFiles) + { + var entryPath = $"media{mediaFile.Key.EnsureStartsWith('/')}"; + ZipArchiveEntry mediaEntry = archive.CreateEntry(entryPath); + using (Stream entryStream = mediaEntry.Open()) + using (mediaFile.Value) + { + mediaFile.Value.Seek(0, SeekOrigin.Begin); + mediaFile.Value.CopyTo(entryStream); + } } } } + else + { + fileName = "package.xml"; + tempPackagePath = Path.Combine(temporaryPath, fileName); + + using (FileStream fileStream = File.OpenWrite(tempPackagePath)) + { + compiledPackageXml.Save(fileStream); + } + } + + var directoryName = _hostingEnvironment.MapPathWebRoot(Path.Combine(_mediaFolderPath, definition.Name.Replace(' ', '_'))); @@ -241,7 +258,7 @@ namespace Umbraco.Cms.Core.Packaging Directory.CreateDirectory(directoryName); } - var finalPackagePath = Path.Combine(directoryName, "package.zip"); + var finalPackagePath = Path.Combine(directoryName, fileName); if (File.Exists(finalPackagePath)) { @@ -347,7 +364,7 @@ namespace Umbraco.Cms.Core.Packaging } else if (items.ContainsKey(dictionaryItem.ParentId.Value)) { - // we know the parent exists in the dictionary but + // we know the parent exists in the dictionary but // we haven't processed it yet so we'll leave it for the next loop continue; } diff --git a/src/Umbraco.Infrastructure/Packaging/ImportPackageBuilderExpression.cs b/src/Umbraco.Infrastructure/Packaging/ImportPackageBuilderExpression.cs index b16326ea56..8eda0f0b45 100644 --- a/src/Umbraco.Infrastructure/Packaging/ImportPackageBuilderExpression.cs +++ b/src/Umbraco.Infrastructure/Packaging/ImportPackageBuilderExpression.cs @@ -19,12 +19,12 @@ namespace Umbraco.Cms.Infrastructure.Packaging { internal class ImportPackageBuilderExpression : MigrationExpressionBase { - private readonly IPackagingService _packagingService; - private readonly IMediaService _mediaService; - private readonly MediaFileManager _mediaFileManager; - private readonly MediaUrlGeneratorCollection _mediaUrlGenerators; - private readonly IShortStringHelper _shortStringHelper; private readonly IContentTypeBaseServiceProvider _contentTypeBaseServiceProvider; + private readonly MediaFileManager _mediaFileManager; + private readonly IMediaService _mediaService; + private readonly MediaUrlGeneratorCollection _mediaUrlGenerators; + private readonly IPackagingService _packagingService; + private readonly IShortStringHelper _shortStringHelper; private bool _executed; public ImportPackageBuilderExpression( @@ -45,7 +45,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging } /// - /// The type of the migration which dictates the namespace of the embedded resource + /// The type of the migration which dictates the namespace of the embedded resource /// public Type EmbeddedResourceMigrationType { get; set; } @@ -63,68 +63,77 @@ namespace Umbraco.Cms.Infrastructure.Packaging if (EmbeddedResourceMigrationType == null && PackageDataManifest == null) { - throw new InvalidOperationException($"Nothing to execute, neither {nameof(EmbeddedResourceMigrationType)} or {nameof(PackageDataManifest)} has been set."); + throw new InvalidOperationException( + $"Nothing to execute, neither {nameof(EmbeddedResourceMigrationType)} or {nameof(PackageDataManifest)} has been set."); } InstallationSummary installationSummary; if (EmbeddedResourceMigrationType != null) { - // get the embedded resource - using (ZipArchive zipPackage = PackageMigrationResource.GetEmbeddedPackageDataManifest( + if (PackageMigrationResource.TryGetEmbeddedPackageDataManifest( EmbeddedResourceMigrationType, - out XDocument xml)) + out XDocument xml, out ZipArchive zipPackage)) { // first install the package installationSummary = _packagingService.InstallCompiledPackageData(xml); - // then we need to save each file to the saved media items - var mediaWithFiles = xml.XPathSelectElements( - "./umbPackage/MediaItems/MediaSet//*[@id][@mediaFilePath]") - .ToDictionary( - x => x.AttributeValue("key"), - x => x.AttributeValue("mediaFilePath")); - - // Any existing media by GUID will not be installed by the package service, it will just be skipped - // so you cannot 'update' media (or content) using a package since those are not schema type items. - // This means you cannot 'update' the media file either. The installationSummary.MediaInstalled - // will be empty for any existing media which means that the files will also not be updated. - foreach (IMedia media in installationSummary.MediaInstalled) + if (zipPackage is not null) { - if (mediaWithFiles.TryGetValue(media.Key, out var mediaFilePath)) + // get the embedded resource + using (zipPackage) { - // this is a media item that has a file, so find that file in the zip - var entryPath = $"media{mediaFilePath.EnsureStartsWith('/')}"; - ZipArchiveEntry mediaEntry = zipPackage.GetEntry(entryPath); - if (mediaEntry == null) - { - throw new InvalidOperationException("No media file found in package zip for path " + entryPath); - } + // then we need to save each file to the saved media items + var mediaWithFiles = xml.XPathSelectElements( + "./umbPackage/MediaItems/MediaSet//*[@id][@mediaFilePath]") + .ToDictionary( + x => x.AttributeValue("key"), + x => x.AttributeValue("mediaFilePath")); - // read the media file and save it to the media item - // using the current file system provider. - using (Stream mediaStream = mediaEntry.Open()) + // Any existing media by GUID will not be installed by the package service, it will just be skipped + // so you cannot 'update' media (or content) using a package since those are not schema type items. + // This means you cannot 'update' the media file either. The installationSummary.MediaInstalled + // will be empty for any existing media which means that the files will also not be updated. + foreach (IMedia media in installationSummary.MediaInstalled) { - media.SetValue( - _mediaFileManager, - _mediaUrlGenerators, - _shortStringHelper, - _contentTypeBaseServiceProvider, - Constants.Conventions.Media.File, - Path.GetFileName(mediaFilePath), - mediaStream); - } + if (mediaWithFiles.TryGetValue(media.Key, out var mediaFilePath)) + { + // this is a media item that has a file, so find that file in the zip + var entryPath = $"media{mediaFilePath.EnsureStartsWith('/')}"; + ZipArchiveEntry mediaEntry = zipPackage.GetEntry(entryPath); + if (mediaEntry == null) + { + throw new InvalidOperationException( + "No media file found in package zip for path " + + entryPath); + } - _mediaService.Save(media); + // read the media file and save it to the media item + // using the current file system provider. + using (Stream mediaStream = mediaEntry.Open()) + { + media.SetValue( + _mediaFileManager, + _mediaUrlGenerators, + _shortStringHelper, + _contentTypeBaseServiceProvider, + Constants.Conventions.Media.File, + Path.GetFileName(mediaFilePath), + mediaStream); + } + + _mediaService.Save(media); + } + } } } } - } - else - { - installationSummary = _packagingService.InstallCompiledPackageData(PackageDataManifest); - } + else + { + installationSummary = _packagingService.InstallCompiledPackageData(PackageDataManifest); + } - Logger.LogInformation($"Package migration executed. Summary: {installationSummary}"); + Logger.LogInformation($"Package migration executed. Summary: {installationSummary}"); + } } } } diff --git a/src/Umbraco.Infrastructure/Packaging/PackageDataInstallation.cs b/src/Umbraco.Infrastructure/Packaging/PackageDataInstallation.cs index fcf4ada7c0..2511aab600 100644 --- a/src/Umbraco.Infrastructure/Packaging/PackageDataInstallation.cs +++ b/src/Umbraco.Infrastructure/Packaging/PackageDataInstallation.cs @@ -222,14 +222,12 @@ namespace Umbraco.Cms.Infrastructure.Packaging importedContentTypes.Add(contentTypeAlias, contentType); } - TContentBase content = CreateContentFromXml(root, importedContentTypes[contentTypeAlias], default, parentId, service); - if (content == null) + if (TryCreateContentFromXml(root, importedContentTypes[contentTypeAlias], default, parentId, service, + out TContentBase content)) { - continue; + contents.Add(content); } - contents.Add(content); - var children = root.Elements().Where(doc => (string)doc.Attribute("isDoc") == string.Empty) .ToList(); @@ -262,8 +260,10 @@ namespace Umbraco.Cms.Infrastructure.Packaging } //Create and add the child to the list - var content = CreateContentFromXml(child, importedContentTypes[contentTypeAlias], parent, default, service); - list.Add(content); + if (TryCreateContentFromXml(child, importedContentTypes[contentTypeAlias], parent, default, service, out var content)) + { + list.Add(content); + } //Recursive call var child1 = child; @@ -278,21 +278,24 @@ namespace Umbraco.Cms.Infrastructure.Packaging return list; } - private T CreateContentFromXml( + private bool TryCreateContentFromXml( XElement element, S contentType, T parent, int parentId, - IContentServiceBase service) + IContentServiceBase service, + out T output) where T : class, IContentBase where S : IContentTypeComposition { Guid key = element.RequiredAttributeValue("key"); // we need to check if the content already exists and if so we ignore the installation for this item - if (service.GetById(key) != null) + var value = service.GetById(key); + if (value != null) { - return null; + output = value; + return false; } var level = element.Attribute("level").Value; @@ -383,7 +386,8 @@ namespace Umbraco.Cms.Infrastructure.Packaging } } - return content; + output = content; + return true; } private T CreateContent(string name, T parent, int parentId, S contentType, Guid key, int level, int sortOrder, int? templateId) @@ -498,7 +502,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging //Iterate the sorted document types and create them as IContentType objects foreach (XElement documentType in documentTypes) { - var alias = documentType.Element("Info").Element("Alias").Value; + var alias = documentType.Element("Info").Element("Alias").Value; if (importedContentTypes.ContainsKey(alias) == false) { @@ -1142,7 +1146,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging IDictionaryItem dictionaryItem; var itemName = dictionaryItemElement.Attribute("Name").Value; Guid key = dictionaryItemElement.RequiredAttributeValue("Key"); - + dictionaryItem = _localizationService.GetDictionaryItemById(key); if (dictionaryItem != null) { @@ -1277,7 +1281,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging throw new InvalidOperationException("No path attribute found"); } var contents = element.Value ?? string.Empty; - + var physicalPath = _hostingEnvironment.MapPathContentRoot(path); // TODO: Do we overwrite? IMO I don't think so since these will be views a user will change. if (!System.IO.File.Exists(physicalPath)) @@ -1419,7 +1423,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging if (partialView == null) { var content = partialViewXml.Value ?? string.Empty; - + partialView = new PartialView(PartialViewType.PartialView, path) { Content = content }; _fileService.SavePartialView(partialView, userId); result.Add(partialView); diff --git a/src/Umbraco.Tests.Integration/Umbraco.Core/Packaging/CreatedPackagesRepositoryTests.cs b/src/Umbraco.Tests.Integration/Umbraco.Core/Packaging/CreatedPackagesRepositoryTests.cs index 3ee9b1375c..74d364256f 100644 --- a/src/Umbraco.Tests.Integration/Umbraco.Core/Packaging/CreatedPackagesRepositoryTests.cs +++ b/src/Umbraco.Tests.Integration/Umbraco.Core/Packaging/CreatedPackagesRepositoryTests.cs @@ -8,6 +8,7 @@ using System.IO.Compression; using System.Linq; using System.Xml.Linq; using NUnit.Framework; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Configuration; using Umbraco.Cms.Core.Configuration.Models; using Umbraco.Cms.Core.Hosting; @@ -36,7 +37,7 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging [TearDown] public void DeleteTestFolder() => Directory.Delete(HostingEnvironment.MapPathContentRoot("~/" + _testBaseFolder), true); - + private IContentService ContentService => GetRequiredService(); private IContentTypeService ContentTypeService => GetRequiredService(); @@ -164,7 +165,7 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging { var parent = new DictionaryItem("Parent") { - Key = Guid.NewGuid() + Key = Guid.NewGuid() }; LocalizationService.Save(parent); var child1 = new DictionaryItem(parent.Key, "Child1") @@ -204,12 +205,12 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging }; PackageBuilder.SavePackage(def); - + string packageXmlPath = PackageBuilder.ExportPackage(def); - using (var packageZipStream = File.OpenRead(packageXmlPath)) - using (ZipArchive zipArchive = PackageMigrationResource.GetPackageDataManifest(packageZipStream, out XDocument packageXml)) + using (var packageXmlStream = File.OpenRead(packageXmlPath)) { + var packageXml = XDocument.Load(packageXmlStream); var dictionaryItems = packageXml.Root.Element("DictionaryItems"); Assert.IsNotNull(dictionaryItems); var rootItems = dictionaryItems.Elements("DictionaryItem").ToList(); @@ -226,7 +227,53 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging } [Test] - public void Export() + public void Export_Zip() + { + var mt = MediaTypeBuilder.CreateImageMediaType("testImage"); + MediaTypeService.Save(mt); + var m1 = MediaBuilder.CreateMediaFile(mt, -1); + MediaService.Save(m1); + + //Ensure a file exist + var fullPath = HostingEnvironment.MapPathWebRoot(m1.Properties[Constants.Conventions.Media.File].GetValue().ToString()); + using (StreamWriter file1 = File.CreateText(fullPath)) + { + file1.WriteLine("hello"); + } + + var def = new PackageDefinition + { + Name = "test", + MediaUdis = new List(){m1.GetUdi()} + }; + + bool result = PackageBuilder.SavePackage(def); + Assert.IsTrue(result); + Assert.IsTrue(def.PackagePath.IsNullOrWhiteSpace()); + + string packageXmlPath = PackageBuilder.ExportPackage(def); + + def = PackageBuilder.GetById(def.Id); // re-get + Assert.IsNotNull(def.PackagePath); + + using (FileStream packageZipStream = File.OpenRead(packageXmlPath)) + using (ZipArchive zipArchive = PackageMigrationResource.GetPackageDataManifest(packageZipStream, out XDocument packageXml)) + { + Assert.AreEqual("umbPackage", packageXml.Root.Name.ToString()); + Assert.IsNotNull(zipArchive.GetEntry("media/media/test-file.txt")); + + Assert.AreEqual( + $"", + packageXml.Element("umbPackage").Element("MediaItems").ToString(SaveOptions.DisableFormatting)); + + // TODO: There's a whole lot more assertions to be done + + } + } + + + [Test] + public void Export_Xml() { var template = TemplateBuilder.CreateTextPageTemplate(); @@ -242,19 +289,17 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging Assert.IsTrue(result); Assert.IsTrue(def.PackagePath.IsNullOrWhiteSpace()); - string packageXmlPath = PackageBuilder.ExportPackage(def); + string packageXmlPath = PackageBuilder.ExportPackage(def); // Get def = PackageBuilder.GetById(def.Id); // re-get Assert.IsNotNull(def.PackagePath); - using (FileStream packageZipStream = File.OpenRead(packageXmlPath)) - using (ZipArchive zipArchive = PackageMigrationResource.GetPackageDataManifest(packageZipStream, out XDocument packageXml)) + using (var packageXmlStream = File.OpenRead(packageXmlPath)) { - Assert.AreEqual("umbPackage", packageXml.Root.Name.ToString()); + var xml = XDocument.Load(packageXmlStream); + Assert.AreEqual("umbPackage", xml.Root.Name.ToString()); - Assert.AreEqual( - $"", - packageXml.Element("umbPackage").Element("Templates").ToString(SaveOptions.DisableFormatting)); + Assert.AreEqual($"", xml.Element("umbPackage").Element("Templates").ToString(SaveOptions.DisableFormatting)); // TODO: There's a whole lot more assertions to be done