init commit for changing packages back to xml format so we can more easily deal with media.

This commit is contained in:
Shannon
2021-07-06 15:09:56 -06:00
parent 9178e28485
commit f7123b355d
15 changed files with 188 additions and 51 deletions

View File

@@ -224,6 +224,9 @@ namespace Umbraco.Extensions
string oldpath = null;
if (property.GetValue(culture, segment) is string svalue)
{
// TODO: This is still horrible! We are not delagating to the actual property editors on how to handle this
// which we know how to do.
if (svalue.DetectIsJson())
{
// the property value is a JSON serialized image crop data set - grab the "src" property as the file source
@@ -233,6 +236,10 @@ namespace Umbraco.Extensions
}
var filepath = mediaFileManager.StoreFile(content, property.PropertyType, filename, filestream, oldpath);
// TODO: And here we are just setting the value to a string which means that any file based editor
// will need to handle the raw string value and progressively save it to it's correct (i.e. JSON)
// format.
property.SetValue(mediaFileManager.FileSystem.GetUrl(filepath), culture, segment);
}

View File

@@ -3,8 +3,12 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Strings;
using Umbraco.Extensions;
@@ -15,6 +19,9 @@ namespace Umbraco.Cms.Core.IO
private readonly IMediaPathScheme _mediaPathScheme;
private readonly ILogger<MediaFileManager> _logger;
private readonly IShortStringHelper _shortStringHelper;
private readonly IServiceProvider _serviceProvider;
private MediaUrlGeneratorCollection _mediaUrlGenerators;
private readonly ContentSettings _contentSettings;
/// <summary>
/// Gets the media filesystem.
@@ -25,11 +32,15 @@ namespace Umbraco.Cms.Core.IO
IFileSystem fileSystem,
IMediaPathScheme mediaPathScheme,
ILogger<MediaFileManager> logger,
IShortStringHelper shortStringHelper)
IShortStringHelper shortStringHelper,
IServiceProvider serviceProvider,
IOptions<ContentSettings> contentSettings)
{
_mediaPathScheme = mediaPathScheme;
_logger = logger;
_shortStringHelper = shortStringHelper;
_serviceProvider = serviceProvider;
_contentSettings = contentSettings.Value;
FileSystem = fileSystem;
}
@@ -118,11 +129,36 @@ namespace Umbraco.Cms.Core.IO
return _mediaPathScheme.GetFilePath(this, cuid, puid, filename, prevpath);
}
#endregion
#region Associated Media Files
public Stream GetFile(IMedia media, out string mediaFilePath)
{
// TODO: If collections were lazy we could just inject them
if (_mediaUrlGenerators == null)
{
_mediaUrlGenerators = _serviceProvider.GetRequiredService<MediaUrlGeneratorCollection>();
}
// This is how we get all URLs based on the defined aliases,
// then we just try each to get a file until one works.
var urls = media.GetUrls(_contentSettings, _mediaUrlGenerators);
foreach(var url in urls)
{
Stream stream = FileSystem.OpenFile(url);
if (stream != null)
{
mediaFilePath = url;
return stream;
}
}
mediaFilePath = null;
return null;
}
/// <summary>
/// Stores a media file associated to a property of a content item.
/// </summary>

View File

@@ -1,4 +1,4 @@
namespace Umbraco.Cms.Core.Models
namespace Umbraco.Cms.Core.Models
{
/// <summary>
/// Used to generate paths to media items for a specified property editor alias
@@ -8,11 +8,11 @@
/// <summary>
/// Tries to get a media path for a given property editor alias
/// </summary>
/// <param name="alias">The property editor alias</param>
/// <param name="propertyEditorAlias">The property editor alias</param>
/// <param name="value">The value of the property</param>
/// <returns>
/// True if a media path was returned
/// </returns>
bool TryGetMediaPath(string alias, object value, out string mediaPath);
bool TryGetMediaPath(string propertyEditorAlias, object value, out string mediaPath);
}
}

View File

@@ -1,4 +1,4 @@
using System.Linq;
using System.Linq;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
@@ -29,11 +29,9 @@ namespace Umbraco.Extensions
/// Gets the URLs of a media item.
/// </summary>
public static string[] GetUrls(this IMedia media, ContentSettings contentSettings, MediaUrlGeneratorCollection mediaUrlGenerators)
{
return contentSettings.Imaging.AutoFillImageProperties
=> contentSettings.Imaging.AutoFillImageProperties
.Select(field => media.GetUrl(field.Alias, mediaUrlGenerators))
.Where(link => string.IsNullOrWhiteSpace(link) == false)
.ToArray();
}
}
}

View File

@@ -2,12 +2,15 @@ using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Xml.Linq;
using System.Xml.XPath;
using Microsoft.Extensions.Options;
using Umbraco.Cms.Core.Configuration;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Services;
using Umbraco.Extensions;
@@ -35,6 +38,7 @@ namespace Umbraco.Cms.Core.Packaging
private readonly PackageDefinitionXmlParser _parser;
private readonly IMediaService _mediaService;
private readonly IMediaTypeService _mediaTypeService;
private readonly MediaFileManager _mediaFileManager;
/// <summary>
/// Constructor
@@ -66,6 +70,7 @@ namespace Umbraco.Cms.Core.Packaging
IOptions<GlobalSettings> globalSettings,
IMediaService mediaService,
IMediaTypeService mediaTypeService,
MediaFileManager mediaFileManager,
string packageRepositoryFileName,
string tempFolderPath = null,
string packagesFolderPath = null,
@@ -90,6 +95,7 @@ namespace Umbraco.Cms.Core.Packaging
_parser = new PackageDefinitionXmlParser();
_mediaService = mediaService;
_mediaTypeService = mediaTypeService;
_mediaFileManager = mediaFileManager;
}
private string CreatedPackagesFile => _packagesFolderPath.EnsureEndsWith('/') + _packageRepositoryFileName;
@@ -197,19 +203,32 @@ namespace Umbraco.Cms.Core.Packaging
PackageDictionaryItems(definition, root);
PackageLanguages(definition, root);
PackageDataTypes(definition, root);
PackageMedia(definition, root);
Dictionary<string, Stream> mediaFiles = PackageMedia(definition, root);
var packageXmlFileName = temporaryPath + "/package.xml";
var tempPackagePath = temporaryPath + "/package.zip";
if (File.Exists(packageXmlFileName))
using (FileStream fileStream = File.OpenWrite(tempPackagePath))
using (var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
{
File.Delete(packageXmlFileName);
ZipArchiveEntry packageXmlEntry = archive.CreateEntry("package.xml");
using (Stream entryStream = packageXmlEntry.Open())
{
compiledPackageXml.Save(entryStream);
}
foreach (KeyValuePair<string, Stream> mediaFile in mediaFiles)
{
// must ensure the path is relative, doesn't start with /
ZipArchiveEntry mediaEntry = archive.CreateEntry(mediaFile.Key.TrimStart('/'));
using (Stream entryStream = mediaEntry.Open())
using (mediaFile.Value)
{
mediaFile.Value.Seek(0, SeekOrigin.Begin);
mediaFile.Value.CopyTo(entryStream);
}
}
}
compiledPackageXml.Save(packageXmlFileName);
// check if there's a packages directory below media
var directoryName =
_hostingEnvironment.MapPathWebRoot(Path.Combine(_mediaFolderPath, definition.Name.Replace(' ', '_')));
@@ -218,18 +237,19 @@ namespace Umbraco.Cms.Core.Packaging
Directory.CreateDirectory(directoryName);
}
var packPath = Path.Combine(directoryName, "package.xml");
var finalPackagePath = Path.Combine(directoryName, "package.zip");
if (File.Exists(packPath))
if (File.Exists(finalPackagePath))
{
File.Delete(packPath);
File.Delete(finalPackagePath);
}
File.Move(packageXmlFileName, packPath);
definition.PackagePath = packPath;
File.Move(tempPackagePath, finalPackagePath);
definition.PackagePath = finalPackagePath;
SavePackage(definition);
return packPath;
return finalPackagePath;
}
finally
{
@@ -305,7 +325,7 @@ namespace Umbraco.Cms.Core.Packaging
var processed = new Dictionary<Guid, XElement>();
while (processed.Count < itemCount)
{
foreach(Guid key in items.Keys.ToList())
foreach (Guid key in items.Keys.ToList())
{
(IDictionaryItem dictionaryItem, XElement serializedDictionaryValue) = items[key];
@@ -518,14 +538,43 @@ namespace Umbraco.Cms.Core.Packaging
}
private void PackageMedia(PackageDefinition definition, XElement root)
private Dictionary<string, Stream> PackageMedia(PackageDefinition definition, XElement root)
{
var mediaStreams = new Dictionary<string, Stream>();
// callback that occurs on each serialized media item
void OnSerializedMedia(IMedia media, XElement xmlMedia)
{
// get the media file path and store that separately in the XML.
// the media file path is different from the URL and is specifically
// extracted using the property editor for this media file and the current media file system.
Stream mediaStream = _mediaFileManager.GetFile(media, out var mediaFilePath);
if (mediaStream != null)
{
xmlMedia.Add(new XAttribute("mediaFilePath", mediaFilePath));
// add the stream to our outgoing stream
mediaStreams.Add(mediaFilePath, mediaStream);
}
}
IEnumerable<IMedia> medias = _mediaService.GetByIds(definition.MediaUdis);
root.Add(
new XElement(
"MediaItems",
medias.Select(x => new XElement("MediaSet", _serializer.Serialize(x, definition.MediaLoadChildNodes)))));
var mediaXml = new XElement(
"MediaItems",
medias.Select(media =>
{
XElement serializedMedia = _serializer.Serialize(
media,
definition.MediaLoadChildNodes,
OnSerializedMedia);
return new XElement("MediaSet", serializedMedia);
}));
root.Add(mediaXml);
return mediaStreams;
}
// TODO: Delete this

View File

@@ -10,11 +10,11 @@ namespace Umbraco.Cms.Core.PropertyEditors
{
}
public bool TryGetMediaPath(string alias, object value, out string mediaPath)
public bool TryGetMediaPath(string propertyEditorAlias, object value, out string mediaPath)
{
foreach(var generator in this)
foreach(IMediaUrlGenerator generator in this)
{
if (generator.TryGetMediaPath(alias, value, out var mp))
if (generator.TryGetMediaPath(propertyEditorAlias, value, out var mp))
{
mediaPath = mp;
return true;

View File

@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Xml.Linq;
using Umbraco.Cms.Core.Models;
@@ -22,7 +23,8 @@ namespace Umbraco.Cms.Core.Services
/// </summary>
XElement Serialize(
IMedia media,
bool withDescendants = false);
bool withDescendants = false,
Action<IMedia, XElement> onMediaItemSerialized = null);
/// <summary>
/// Exports an IMember item as an XElement.

View File

@@ -10,6 +10,7 @@ using Umbraco.Cms.Core.Configuration;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.DependencyInjection;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Packaging;
using Umbraco.Cms.Core.Routing;
using Umbraco.Cms.Core.Services;
@@ -93,6 +94,7 @@ namespace Umbraco.Cms.Infrastructure.DependencyInjection
factory.GetRequiredService<IOptions<GlobalSettings>>(),
factory.GetRequiredService<IMediaService>(),
factory.GetRequiredService<IMediaTypeService>(),
factory.GetRequiredService<MediaFileManager>(),
packageRepoFileName);
private static LocalizedTextServiceFileSources SourcesFactory(IServiceProvider container)

View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using Examine;
@@ -44,6 +44,9 @@ namespace Umbraco.Cms.Infrastructure.Examine
var umbracoFilePath = string.Empty;
var umbracoFile = string.Empty;
// TODO: To get the file path we should/could use the MediaUrlGeneratorCollection instead of this
// just like we are doing in the MediaFileManager?
var umbracoFileSource = m.GetValue<string>(Constants.Conventions.Media.File);
if (umbracoFileSource.DetectIsJson())

View File

@@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Xml.Linq;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Serialization;
@@ -98,7 +100,8 @@ namespace Umbraco.Cms.Core.Services.Implement
/// </summary>
public XElement Serialize(
IMedia media,
bool withDescendants = false)
bool withDescendants = false,
Action<IMedia, XElement> onMediaItemSerialized = null)
{
if (_mediaService == null) throw new ArgumentNullException(nameof(_mediaService));
if (_dataTypeService == null) throw new ArgumentNullException(nameof(_dataTypeService));
@@ -110,7 +113,9 @@ namespace Umbraco.Cms.Core.Services.Implement
var nodeName = media.ContentType.Alias.ToSafeAlias(_shortStringHelper);
const bool published = false; // always false for media
var xml = SerializeContentBase(media, media.GetUrlSegment(_shortStringHelper, _urlSegmentProviders), nodeName, published);
string urlValue = media.GetUrlSegment(_shortStringHelper, _urlSegmentProviders);
XElement xml = SerializeContentBase(media, urlValue, nodeName, published);
xml.Add(new XAttribute("nodeType", media.ContentType.Id));
xml.Add(new XAttribute("nodeTypeAlias", media.ContentType.Alias));
@@ -123,6 +128,8 @@ namespace Umbraco.Cms.Core.Services.Implement
//xml.Add(new XAttribute("template", 0)); // no template for media
onMediaItemSerialized?.Invoke(media, xml);
if (withDescendants)
{
const int pageSize = 500;
@@ -131,7 +138,7 @@ namespace Umbraco.Cms.Core.Services.Implement
while (page * pageSize < total)
{
var children = _mediaService.GetPagedChildren(media.Id, page++, pageSize, out total);
SerializeChildren(children, xml);
SerializeChildren(children, xml, onMediaItemSerialized);
}
}
@@ -619,12 +626,12 @@ namespace Umbraco.Cms.Core.Services.Implement
}
// exports an IMedia item descendants.
private void SerializeChildren(IEnumerable<IMedia> children, XElement xml)
private void SerializeChildren(IEnumerable<IMedia> children, XElement xml, Action<IMedia, XElement> onMediaItemSerialized)
{
foreach (var child in children)
{
// add the child xml
var childXml = Serialize(child);
var childXml = Serialize(child, onMediaItemSerialized: onMediaItemSerialized);
xml.Add(childXml);
const int pageSize = 500;
@@ -634,7 +641,7 @@ namespace Umbraco.Cms.Core.Services.Implement
{
var grandChildren = _mediaService.GetPagedChildren(child.Id, page++, pageSize, out total);
// recurse
SerializeChildren(grandChildren, childXml);
SerializeChildren(grandChildren, childXml, onMediaItemSerialized);
}
}
}

View File

@@ -11,6 +11,7 @@ using NUnit.Framework;
using Umbraco.Cms.Core.Configuration;
using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Packaging;
using Umbraco.Cms.Core.Services;
@@ -56,6 +57,8 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging
private IMediaTypeService MediaTypeService => GetRequiredService<IMediaTypeService>();
private MediaFileManager MediaFileManager => GetRequiredService<MediaFileManager>();
public ICreatedPackagesRepository PackageBuilder => new PackagesRepository(
ContentService,
ContentTypeService,
@@ -68,6 +71,7 @@ namespace Umbraco.Cms.Tests.Integration.Umbraco.Core.Packaging
Microsoft.Extensions.Options.Options.Create(new GlobalSettings()),
MediaService,
MediaTypeService,
MediaFileManager,
"createdPackages.config",
// temp paths

View File

@@ -22,6 +22,8 @@ using Umbraco.Cms.Core.Events;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Logging;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Scoping;
using Umbraco.Cms.Core.Strings;
using Umbraco.Cms.Infrastructure.Migrations.Install;
@@ -60,17 +62,22 @@ namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Core.Components
var fs = new FileSystems(loggerFactory, IOHelper, Options.Create(globalSettings), Mock.Of<IHostingEnvironment>());
var coreDebug = new CoreDebugSettings();
MediaFileManager mediaFileManager = new MediaFileManager(Mock.Of<IFileSystem>(),
Mock.Of<IMediaPathScheme>(), Mock.Of<ILogger<MediaFileManager>>(), Mock.Of<IShortStringHelper>());
var mediaFileManager = new MediaFileManager(
Mock.Of<IFileSystem>(),
Mock.Of<IMediaPathScheme>(),
Mock.Of<ILogger<MediaFileManager>>(),
Mock.Of<IShortStringHelper>(),
Mock.Of<IServiceProvider>(),
Options.Create(new ContentSettings()));
IEventAggregator eventAggregator = Mock.Of<IEventAggregator>();
var p = new ScopeProvider(f, fs, Options.Create(coreDebug), mediaFileManager, loggerFactory.CreateLogger<ScopeProvider>(), loggerFactory, NoAppCache.Instance, eventAggregator);
var scopeProvider = new ScopeProvider(f, fs, Options.Create(coreDebug), mediaFileManager, loggerFactory.CreateLogger<ScopeProvider>(), loggerFactory, NoAppCache.Instance, eventAggregator);
mock.Setup(x => x.GetService(typeof(ILogger))).Returns(logger);
mock.Setup(x => x.GetService(typeof(ILogger<ComponentCollection>))).Returns(loggerFactory.CreateLogger<ComponentCollection>);
mock.Setup(x => x.GetService(typeof(ILoggerFactory))).Returns(loggerFactory);
mock.Setup(x => x.GetService(typeof(IProfilingLogger))).Returns(new ProfilingLogger(loggerFactory.CreateLogger<ProfilingLogger>(), Mock.Of<IProfiler>()));
mock.Setup(x => x.GetService(typeof(IUmbracoDatabaseFactory))).Returns(f);
mock.Setup(x => x.GetService(typeof(IScopeProvider))).Returns(p);
mock.Setup(x => x.GetService(typeof(IScopeProvider))).Returns(scopeProvider);
setup?.Invoke(mock);
return mock.Object;

View File

@@ -11,6 +11,7 @@ using Umbraco.Cms.Core.Events;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Scoping;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Strings;
@@ -79,8 +80,13 @@ namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Core.Scoping
Options.Create(new GlobalSettings()),
Mock.Of<IHostingEnvironment>());
var mediaFileManager = new MediaFileManager(Mock.Of<IFileSystem>(), Mock.Of<IMediaPathScheme>(),
instance.CreateLogger<MediaFileManager>(), Mock.Of<IShortStringHelper>());
var mediaFileManager = new MediaFileManager(
Mock.Of<IFileSystem>(),
Mock.Of<IMediaPathScheme>(),
instance.CreateLogger<MediaFileManager>(),
Mock.Of<IShortStringHelper>(),
Mock.Of<IServiceProvider>(),
Options.Create(new ContentSettings()));
return new ScopeProvider(
Mock.Of<IUmbracoDatabaseFactory>(),

View File

@@ -1,4 +1,5 @@
using System;
using System.Linq;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
@@ -9,7 +10,9 @@ using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Events;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Notifications;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Scoping;
using Umbraco.Cms.Core.Strings;
using Umbraco.Cms.Infrastructure.Persistence;
@@ -76,8 +79,13 @@ namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Core.Scoping
Options.Create(new GlobalSettings()),
Mock.Of<IHostingEnvironment>());
var mediaFileManager = new MediaFileManager(Mock.Of<IFileSystem>(), Mock.Of<IMediaPathScheme>(),
loggerFactory.CreateLogger<MediaFileManager>(), Mock.Of<IShortStringHelper>());
var mediaFileManager = new MediaFileManager(
Mock.Of<IFileSystem>(),
Mock.Of<IMediaPathScheme>(),
loggerFactory.CreateLogger<MediaFileManager>(),
Mock.Of<IShortStringHelper>(),
Mock.Of<IServiceProvider>(),
Options.Create(new ContentSettings()));
eventAggregatorMock = new Mock<IEventAggregator>();

View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using CSharpTest.Net.Interfaces;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
@@ -13,6 +14,8 @@ using Umbraco.Cms.Core.Configuration.Models;
using Umbraco.Cms.Core.Events;
using Umbraco.Cms.Core.Hosting;
using Umbraco.Cms.Core.IO;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.PropertyEditors;
using Umbraco.Cms.Core.Scoping;
using Umbraco.Cms.Core.Strings;
using Umbraco.Cms.Infrastructure.Persistence;
@@ -33,8 +36,13 @@ namespace Umbraco.Cms.Tests.UnitTests.Umbraco.Infrastructure.Scoping
var loggerFactory = NullLoggerFactory.Instance;
var fileSystems = new FileSystems(loggerFactory,
Mock.Of<IIOHelper>(), Mock.Of<IOptions<GlobalSettings>>(), Mock.Of<IHostingEnvironment>());
var mediaFileManager = new MediaFileManager(Mock.Of<IFileSystem>(), Mock.Of<IMediaPathScheme>(),
loggerFactory.CreateLogger<MediaFileManager>(), Mock.Of<IShortStringHelper>());
var mediaFileManager = new MediaFileManager(
Mock.Of<IFileSystem>(),
Mock.Of<IMediaPathScheme>(),
loggerFactory.CreateLogger<MediaFileManager>(),
Mock.Of<IShortStringHelper>(),
Mock.Of<IServiceProvider>(),
Options.Create(new ContentSettings()));
var databaseFactory = new Mock<IUmbracoDatabaseFactory>();
var database = new Mock<IUmbracoDatabase>();
var sqlContext = new Mock<ISqlContext>();