yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10823 from umbraco/v9/feature/package_export_and_import_both_xml_and_zip

Browse Source 
V9: Package export and import both xml and zip
This commit is contained in:
Warren Buckley
2021-08-11 11:10:28 +01:00
committed by GitHub
parent c2d54f508f 2b8256c576
commit 558352e3ac
5 changed files with 244 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/Umbraco.Core/Packaging/PackageMigrationResource.cs
View File

@@ -12,17 +12,52 @@ namespace Umbraco.Cms.Core.Packaging
{
public static class PackageMigrationResource
{
private static Stream GetEmbeddedPackageStream(Type planType)
private static Stream GetEmbeddedPackageZipStream(Type planType)
{
// lookup the embedded resource by convention
Assembly currentAssembly = planType.Assembly;
var fileName = $"{planType.Namespace}.package.zip";
Stream stream = currentAssembly.GetManifestResourceStream(fileName);
return stream;
}
public static XDocument GetEmbeddedPackageDataManifest(Type planType, out ZipArchive zipArchive)
{
XDocument packageXml;
var zipStream = GetEmbeddedPackageZipStream(planType);
if (zipStream is not null)
{
zipArchive = GetPackageDataManifest(zipStream, out packageXml);
return packageXml;
}
zipArchive = null;
packageXml = GetEmbeddedPackageXmlDoc(planType);
return packageXml;
}
public static XDocument GetEmbeddedPackageDataManifest(Type planType)
{
return GetEmbeddedPackageDataManifest(planType, out _);
}
private static XDocument GetEmbeddedPackageXmlDoc(Type planType)
{
// lookup the embedded resource by convention
Assembly currentAssembly = planType.Assembly;
var fileName = $"{planType.Namespace}.package.xml";
Stream stream = currentAssembly.GetManifestResourceStream(fileName);
if (stream == null)
{
throw new FileNotFoundException("Cannot find the embedded file.", fileName);
return null;
}
return stream;
XDocument xml;
using (stream)
{
xml = XDocument.Load(stream);
}
return xml;
}
public static string GetEmbeddedPackageDataManifestHash(Type planType)
@@ -30,17 +65,46 @@ namespace Umbraco.Cms.Core.Packaging
// SEE: HashFromStreams in the benchmarks project for how fast this is. It will run
// on every startup for every embedded package.zip. The bigger the zip, the more time it takes.
// But it is still very fast ~303ms for a 100MB file. This will only be an issue if there are
// several very large package.zips.
// several very large package.zips.
using Stream stream = GetEmbeddedPackageStream(planType);
return stream.GetStreamHash();
using Stream stream = GetEmbeddedPackageZipStream(planType);
if (stream is not null)
{
return stream.GetStreamHash();
}
var xml = GetEmbeddedPackageXmlDoc(planType);
if (xml is not null)
{
return xml.ToString();
}
throw new IOException("Missing embedded files for planType: " + planType);
}
public static ZipArchive GetEmbeddedPackageDataManifest(Type planType, out XDocument packageXml)
=> GetPackageDataManifest(GetEmbeddedPackageStream(planType), out packageXml);
public static bool TryGetEmbeddedPackageDataManifest(Type planType, out XDocument packageXml, out ZipArchive zipArchive)
{
var zipStream = GetEmbeddedPackageZipStream(planType);
if (zipStream is not null)
{
zipArchive = GetPackageDataManifest(zipStream, out packageXml);
return true;
}
zipArchive = null;
packageXml = GetEmbeddedPackageXmlDoc(planType);
return packageXml is not null;
}
public static ZipArchive GetPackageDataManifest(Stream packageZipStream, out XDocument packageXml)
{
if (packageZipStream == null)
{
throw new ArgumentNullException(nameof(packageZipStream));
}
var zip = new ZipArchive(packageZipStream, ZipArchiveMode.Read);
ZipArchiveEntry packageXmlEntry = zip.GetEntry("package.xml");
if (packageXmlEntry == null)

55
src/Umbraco.Core/Packaging/PackagesRepository.cs
View File

@@ -209,29 +209,46 @@ namespace Umbraco.Cms.Core.Packaging
PackageDataTypes(definition, root);
Dictionary<string, Stream> mediaFiles = PackageMedia(definition, root);
var tempPackagePath = temporaryPath + "/package.zip";
using (FileStream fileStream = File.OpenWrite(tempPackagePath))
using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
string fileName;
string tempPackagePath;
if (mediaFiles.Count > 0)
{
ZipArchiveEntry packageXmlEntry = archive.CreateEntry("package.xml");
using (Stream entryStream = packageXmlEntry.Open())
fileName = "package.zip";
tempPackagePath = Path.Combine(temporaryPath, fileName);
using (FileStream fileStream = File.OpenWrite(tempPackagePath))
using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
{
compiledPackageXml.Save(entryStream);
}
foreach (KeyValuePair<string, Stream> mediaFile in mediaFiles)
{
var entryPath = $"media{mediaFile.Key.EnsureStartsWith('/')}";
ZipArchiveEntry mediaEntry = archive.CreateEntry(entryPath);
using (Stream entryStream = mediaEntry.Open())
using (mediaFile.Value)
ZipArchiveEntry packageXmlEntry = archive.CreateEntry("package.xml");
using (Stream entryStream = packageXmlEntry.Open())
{
mediaFile.Value.Seek(0, SeekOrigin.Begin);
mediaFile.Value.CopyTo(entryStream);
compiledPackageXml.Save(entryStream);
}
foreach (KeyValuePair<string, Stream> mediaFile in mediaFiles)
{
var entryPath = $"media{mediaFile.Key.EnsureStartsWith('/')}";
ZipArchiveEntry mediaEntry = archive.CreateEntry(entryPath);
using (Stream entryStream = mediaEntry.Open())
using (mediaFile.Value)
{
mediaFile.Value.Seek(0, SeekOrigin.Begin);
mediaFile.Value.CopyTo(entryStream);
}
}
}
}
else
{
fileName = "package.xml";
tempPackagePath = Path.Combine(temporaryPath, fileName);
using (FileStream fileStream = File.OpenWrite(tempPackagePath))
{
compiledPackageXml.Save(fileStream);
}
}
var directoryName =
_hostingEnvironment.MapPathWebRoot(Path.Combine(_mediaFolderPath, definition.Name.Replace(' ', '_')));
@@ -241,7 +258,7 @@ namespace Umbraco.Cms.Core.Packaging
Directory.CreateDirectory(directoryName);
}
var finalPackagePath = Path.Combine(directoryName, "package.zip");
var finalPackagePath = Path.Combine(directoryName, fileName);
if (File.Exists(finalPackagePath))
{
@@ -347,7 +364,7 @@ namespace Umbraco.Cms.Core.Packaging
}
else if (items.ContainsKey(dictionaryItem.ParentId.Value))
{
// we know the parent exists in the dictionary but
// we know the parent exists in the dictionary but
// we haven't processed it yet so we'll leave it for the next loop
continue;
}

107
src/Umbraco.Infrastructure/Packaging/ImportPackageBuilderExpression.cs
View File

@@ -19,12 +19,12 @@ namespace Umbraco.Cms.Infrastructure.Packaging
{
internal class ImportPackageBuilderExpression : MigrationExpressionBase
{
private readonly IPackagingService _packagingService;
private readonly IMediaService _mediaService;
private readonly MediaFileManager _mediaFileManager;
private readonly MediaUrlGeneratorCollection _mediaUrlGenerators;
private readonly IShortStringHelper _shortStringHelper;
private readonly IContentTypeBaseServiceProvider _contentTypeBaseServiceProvider;
private readonly MediaFileManager _mediaFileManager;
private readonly IMediaService _mediaService;
private readonly MediaUrlGeneratorCollection _mediaUrlGenerators;
private readonly IPackagingService _packagingService;
private readonly IShortStringHelper _shortStringHelper;
private bool _executed;
public ImportPackageBuilderExpression(
@@ -45,7 +45,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging
}
/// <summary>
/// The type of the migration which dictates the namespace of the embedded resource
/// The type of the migration which dictates the namespace of the embedded resource
/// </summary>
public Type EmbeddedResourceMigrationType { get; set; }
@@ -63,68 +63,77 @@ namespace Umbraco.Cms.Infrastructure.Packaging
if (EmbeddedResourceMigrationType == null && PackageDataManifest == null)
{
throw new InvalidOperationException($"Nothing to execute, neither {nameof(EmbeddedResourceMigrationType)} or {nameof(PackageDataManifest)} has been set.");
throw new InvalidOperationException(
$"Nothing to execute, neither {nameof(EmbeddedResourceMigrationType)} or {nameof(PackageDataManifest)} has been set.");
}
InstallationSummary installationSummary;
if (EmbeddedResourceMigrationType != null)
{
// get the embedded resource
using (ZipArchive zipPackage = PackageMigrationResource.GetEmbeddedPackageDataManifest(
if (PackageMigrationResource.TryGetEmbeddedPackageDataManifest(
EmbeddedResourceMigrationType,
out XDocument xml))
out XDocument xml, out ZipArchive zipPackage))
{
// first install the package
installationSummary = _packagingService.InstallCompiledPackageData(xml);
// then we need to save each file to the saved media items
var mediaWithFiles = xml.XPathSelectElements(
"./umbPackage/MediaItems/MediaSet//*[@id][@mediaFilePath]")
.ToDictionary(
x => x.AttributeValue<Guid>("key"),
x => x.AttributeValue<string>("mediaFilePath"));
// Any existing media by GUID will not be installed by the package service, it will just be skipped
// so you cannot 'update' media (or content) using a package since those are not schema type items.
// This means you cannot 'update' the media file either. The installationSummary.MediaInstalled
// will be empty for any existing media which means that the files will also not be updated.
foreach (IMedia media in installationSummary.MediaInstalled)
if (zipPackage is not null)
{
if (mediaWithFiles.TryGetValue(media.Key, out var mediaFilePath))
// get the embedded resource
using (zipPackage)
{
// this is a media item that has a file, so find that file in the zip
var entryPath = $"media{mediaFilePath.EnsureStartsWith('/')}";
ZipArchiveEntry mediaEntry = zipPackage.GetEntry(entryPath);
if (mediaEntry == null)
{
throw new InvalidOperationException("No media file found in package zip for path " + entryPath);
}
// then we need to save each file to the saved media items
var mediaWithFiles = xml.XPathSelectElements(
"./umbPackage/MediaItems/MediaSet//*[@id][@mediaFilePath]")
.ToDictionary(
x => x.AttributeValue<Guid>("key"),
x => x.AttributeValue<string>("mediaFilePath"));
// read the media file and save it to the media item
// using the current file system provider.
using (Stream mediaStream = mediaEntry.Open())
// Any existing media by GUID will not be installed by the package service, it will just be skipped
// so you cannot 'update' media (or content) using a package since those are not schema type items.
// This means you cannot 'update' the media file either. The installationSummary.MediaInstalled
// will be empty for any existing media which means that the files will also not be updated.
foreach (IMedia media in installationSummary.MediaInstalled)
{
media.SetValue(
_mediaFileManager,
_mediaUrlGenerators,
_shortStringHelper,
_contentTypeBaseServiceProvider,
Constants.Conventions.Media.File,
Path.GetFileName(mediaFilePath),
mediaStream);
}
if (mediaWithFiles.TryGetValue(media.Key, out var mediaFilePath))
{
// this is a media item that has a file, so find that file in the zip
var entryPath = $"media{mediaFilePath.EnsureStartsWith('/')}";
ZipArchiveEntry mediaEntry = zipPackage.GetEntry(entryPath);
if (mediaEntry == null)
{
throw new InvalidOperationException(
"No media file found in package zip for path " +
entryPath);
}
_mediaService.Save(media);
// read the media file and save it to the media item
// using the current file system provider.
using (Stream mediaStream = mediaEntry.Open())
{
media.SetValue(
_mediaFileManager,
_mediaUrlGenerators,
_shortStringHelper,
_contentTypeBaseServiceProvider,
Constants.Conventions.Media.File,
Path.GetFileName(mediaFilePath),
mediaStream);
}
_mediaService.Save(media);
}
}
}
}
}
}
else
{
installationSummary = _packagingService.InstallCompiledPackageData(PackageDataManifest);
}
else
{
installationSummary = _packagingService.InstallCompiledPackageData(PackageDataManifest);
}
Logger.LogInformation($"Package migration executed. Summary: {installationSummary}");
Logger.LogInformation($"Package migration executed. Summary: {installationSummary}");
}
}
}
}

36
src/Umbraco.Infrastructure/Packaging/PackageDataInstallation.cs
View File

@@ -222,14 +222,12 @@ namespace Umbraco.Cms.Infrastructure.Packaging
importedContentTypes.Add(contentTypeAlias, contentType);
}
TContentBase content = CreateContentFromXml(root, importedContentTypes[contentTypeAlias], default, parentId, service);
if (content == null)
if (TryCreateContentFromXml(root, importedContentTypes[contentTypeAlias], default, parentId, service,
out TContentBase content))
{
continue;
contents.Add(content);
}
contents.Add(content);
var children = root.Elements().Where(doc => (string)doc.Attribute("isDoc") == string.Empty)
.ToList();
@@ -262,8 +260,10 @@ namespace Umbraco.Cms.Infrastructure.Packaging
}
//Create and add the child to the list
var content = CreateContentFromXml(child, importedContentTypes[contentTypeAlias], parent, default, service);
list.Add(content);
if (TryCreateContentFromXml(child, importedContentTypes[contentTypeAlias], parent, default, service, out var content))
{
list.Add(content);
}
//Recursive call
var child1 = child;
@@ -278,21 +278,24 @@ namespace Umbraco.Cms.Infrastructure.Packaging
return list;
}
private T CreateContentFromXml<T, S>(
private bool TryCreateContentFromXml<T, S>(
XElement element,
S contentType,
T parent,
int parentId,
IContentServiceBase<T> service)
IContentServiceBase<T> service,
out T output)
where T : class, IContentBase
where S : IContentTypeComposition
{
Guid key = element.RequiredAttributeValue<Guid>("key");
// we need to check if the content already exists and if so we ignore the installation for this item
if (service.GetById(key) != null)
var value = service.GetById(key);
if (value != null)
{
return null;
output = value;
return false;
}
var level = element.Attribute("level").Value;
@@ -383,7 +386,8 @@ namespace Umbraco.Cms.Infrastructure.Packaging
}
}
return content;
output = content;
return true;
}
private T CreateContent<T, S>(string name, T parent, int parentId, S contentType, Guid key, int level, int sortOrder, int? templateId)
@@ -498,7 +502,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging
//Iterate the sorted document types and create them as IContentType objects
foreach (XElement documentType in documentTypes)
{
var alias = documentType.Element("Info").Element("Alias").Value;
var alias = documentType.Element("Info").Element("Alias").Value;
if (importedContentTypes.ContainsKey(alias) == false)
{
@@ -1142,7 +1146,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging
IDictionaryItem dictionaryItem;
var itemName = dictionaryItemElement.Attribute("Name").Value;
Guid key = dictionaryItemElement.RequiredAttributeValue<Guid>("Key");
dictionaryItem = _localizationService.GetDictionaryItemById(key);
if (dictionaryItem != null)
{
@@ -1277,7 +1281,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging
throw new InvalidOperationException("No path attribute found");
}
var contents = element.Value ?? string.Empty;
var physicalPath = _hostingEnvironment.MapPathContentRoot(path);
// TODO: Do we overwrite? IMO I don't think so since these will be views a user will change.
if (!System.IO.File.Exists(physicalPath))
@@ -1419,7 +1423,7 @@ namespace Umbraco.Cms.Infrastructure.Packaging
if (partialView == null)
{
var content = partialViewXml.Value ?? string.Empty;
partialView = new PartialView(PartialViewType.PartialView, path) { Content = content };
_fileService.SavePartialView(partialView, userId);
result.Add(partialView);

71
src/Umbraco.Tests.Integration/Umbraco.Core/Packaging/CreatedPackagesRepositoryTests.cs
View File

@@ -8,6 +8,7 @@ using System.IO.Compression;
using System.Linq;
using System.Xml.Linq;
using NUnit.Framework;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Configuration;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Hosting;
@@ -36,7 +37,7 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging
[TearDown]
public void DeleteTestFolder() =>
Directory.Delete(HostingEnvironment.MapPathContentRoot("~/" + _testBaseFolder), true);
private IContentService ContentService => GetRequiredService<IContentService>();
private IContentTypeService ContentTypeService => GetRequiredService<IContentTypeService>();
@@ -164,7 +165,7 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging
{
var parent = new DictionaryItem("Parent")
{
Key = Guid.NewGuid()
Key = Guid.NewGuid()
};
LocalizationService.Save(parent);
var child1 = new DictionaryItem(parent.Key, "Child1")
@@ -204,12 +205,12 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging
};
PackageBuilder.SavePackage(def);
string packageXmlPath = PackageBuilder.ExportPackage(def);
using (var packageZipStream = File.OpenRead(packageXmlPath))
using (ZipArchive zipArchive = PackageMigrationResource.GetPackageDataManifest(packageZipStream, out XDocument packageXml))
using (var packageXmlStream = File.OpenRead(packageXmlPath))
{
var packageXml = XDocument.Load(packageXmlStream);
var dictionaryItems = packageXml.Root.Element("DictionaryItems");
Assert.IsNotNull(dictionaryItems);
var rootItems = dictionaryItems.Elements("DictionaryItem").ToList();
@@ -226,7 +227,53 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging
}
[Test]
public void Export()
public void Export_Zip()
{
var mt = MediaTypeBuilder.CreateImageMediaType("testImage");
MediaTypeService.Save(mt);
var m1 = MediaBuilder.CreateMediaFile(mt, -1);
MediaService.Save(m1);
//Ensure a file exist
var fullPath = HostingEnvironment.MapPathWebRoot(m1.Properties[Constants.Conventions.Media.File].GetValue().ToString());
using (StreamWriter file1 = File.CreateText(fullPath))
{
file1.WriteLine("hello");
}
var def = new PackageDefinition
{
Name = "test",
MediaUdis = new List<GuidUdi>(){m1.GetUdi()}
};
bool result = PackageBuilder.SavePackage(def);
Assert.IsTrue(result);
Assert.IsTrue(def.PackagePath.IsNullOrWhiteSpace());
string packageXmlPath = PackageBuilder.ExportPackage(def);
def = PackageBuilder.GetById(def.Id); // re-get
Assert.IsNotNull(def.PackagePath);
using (FileStream packageZipStream = File.OpenRead(packageXmlPath))
using (ZipArchive zipArchive = PackageMigrationResource.GetPackageDataManifest(packageZipStream, out XDocument packageXml))
{
Assert.AreEqual("umbPackage", packageXml.Root.Name.ToString());
Assert.IsNotNull(zipArchive.GetEntry("media/media/test-file.txt"));
Assert.AreEqual(
$"<MediaItems><MediaSet><testImage id=\"{m1.Id}\" key=\"{m1.Key}\" parentID=\"-1\" level=\"1\" creatorID=\"-1\" sortOrder=\"0\" createDate=\"{m1.CreateDate.ToString("s")}\" updateDate=\"{m1.UpdateDate.ToString("s")}\" nodeName=\"Test File\" urlName=\"test-file\" path=\"{m1.Path}\" isDoc=\"\" nodeType=\"{mt.Id}\" nodeTypeAlias=\"testImage\" writerName=\"\" writerID=\"0\" udi=\"{m1.GetUdi()}\" mediaFilePath=\"/media/test-file.txt\"><umbracoFile><![CDATA[/media/test-file.txt]]></umbracoFile><umbracoBytes><![CDATA[100]]></umbracoBytes><umbracoExtension><![CDATA[png]]></umbracoExtension></testImage></MediaSet></MediaItems>",
packageXml.Element("umbPackage").Element("MediaItems").ToString(SaveOptions.DisableFormatting));
// TODO: There's a whole lot more assertions to be done
}
}
[Test]
public void Export_Xml()
{
var template = TemplateBuilder.CreateTextPageTemplate();
@@ -242,19 +289,17 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging
Assert.IsTrue(result);
Assert.IsTrue(def.PackagePath.IsNullOrWhiteSpace());
string packageXmlPath = PackageBuilder.ExportPackage(def);
string packageXmlPath = PackageBuilder.ExportPackage(def); // Get
def = PackageBuilder.GetById(def.Id); // re-get
Assert.IsNotNull(def.PackagePath);
using (FileStream packageZipStream = File.OpenRead(packageXmlPath))
using (ZipArchive zipArchive = PackageMigrationResource.GetPackageDataManifest(packageZipStream, out XDocument packageXml))
using (var packageXmlStream = File.OpenRead(packageXmlPath))
{
Assert.AreEqual("umbPackage", packageXml.Root.Name.ToString());
var xml = XDocument.Load(packageXmlStream);
Assert.AreEqual("umbPackage", xml.Root.Name.ToString());
Assert.AreEqual(
$"<Templates><Template><Name>Text page</Name><Alias>textPage</Alias><Design><![CDATA[@using Umbraco.Cms.Web.Common.PublishedModels;{Environment.NewLine}@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage{Environment.NewLine}@{{{Environment.NewLine}\tLayout = null;{Environment.NewLine}}}]]></Design></Template></Templates>",
packageXml.Element("umbPackage").Element("Templates").ToString(SaveOptions.DisableFormatting));
Assert.AreEqual($"<Templates><Template><Name>Text page</Name><Alias>textPage</Alias><Design><![CDATA[@using Umbraco.Cms.Web.Common.PublishedModels;{Environment.NewLine}@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage{Environment.NewLine}@{{{Environment.NewLine}\tLayout = null;{Environment.NewLine}}}]]></Design></Template></Templates>", xml.Element("umbPackage").Element("Templates").ToString(SaveOptions.DisableFormatting));
// TODO: There's a whole lot more assertions to be done