Migrate old locallinks formats (#17307)

* Initial working localLinks migration

* Cleanup

* Refactor for extendability part 1

* Refactor part 2

* Fixed circular dependency

* Made sure all the extendable logic for the migration is marked as obsolete for v18

* Refactor to use Interface and non circular references instead

* Use Npco for SQLserver compatibility and include media properties too

---------

Co-authored-by: nikolajlauridsen <nikolajlauridsen@protonmail.ch>
This commit is contained in:
Sven Geusens
2024-11-04 12:29:55 +01:00
committed by nikolajlauridsen
parent 6417948a58
commit 691ca2827d
10 changed files with 458 additions and 3 deletions

View File

@@ -118,8 +118,8 @@ public sealed class HtmlLocalLinkParser
}
}
// todo remove at some point?
private IEnumerable<LocalLinkTag> FindLegacyLocalLinkIds(string text)
[Obsolete("This is a temporary method to support legacy formats until we are sure all data has been migration. Scheduled for removal in v17")]
public IEnumerable<LocalLinkTag> FindLegacyLocalLinkIds(string text)
{
// Parse internal links
MatchCollection tags = LocalLinkPattern.Matches(text);
@@ -148,7 +148,8 @@ public sealed class HtmlLocalLinkParser
}
}
private class LocalLinkTag
[Obsolete("This is a temporary method to support legacy formats until we are sure all data has been migration. Scheduled for removal in v17")]
public class LocalLinkTag
{
public LocalLinkTag(int? intId, GuidUdi? udi, string tagHref)
{

View File

@@ -103,5 +103,6 @@ public class UmbracoPlan : MigrationPlan
To<V_15_0_0.ConvertBlockListEditorProperties>("{6C04B137-0097-4938-8C6A-276DF1A0ECA8}");
To<V_15_0_0.ConvertBlockGridEditorProperties>("{9D3CE7D4-4884-41D4-98E8-302EB6CB0CF6}");
To<V_15_0_0.ConvertRichTextEditorProperties>("{37875E80-5CDD-42FF-A21A-7D4E3E23E0ED}");
To<V_15_0_0.ConvertLocalLinks>("{42E44F9E-7262-4269-922D-7310CB48E724}");
}
}

View File

@@ -0,0 +1,188 @@
using Microsoft.Extensions.Logging;
using NPoco;
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Models.Editors;
using Umbraco.Cms.Core.Serialization;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Web;
using Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks;
using Umbraco.Cms.Infrastructure.Persistence;
using Umbraco.Cms.Infrastructure.Persistence.Dtos;
using Umbraco.Extensions;
namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0;
public class ConvertLocalLinks : MigrationBase
{
private readonly IUmbracoContextFactory _umbracoContextFactory;
private readonly IContentTypeService _contentTypeService;
private readonly ILogger<ConvertLocalLinks> _logger;
private readonly IDataTypeService _dataTypeService;
private readonly ILanguageService _languageService;
private readonly IJsonSerializer _jsonSerializer;
private readonly LocalLinkProcessor _localLinkProcessor;
private readonly IMediaTypeService _mediaTypeService;
public ConvertLocalLinks(
IMigrationContext context,
IUmbracoContextFactory umbracoContextFactory,
IContentTypeService contentTypeService,
ILogger<ConvertLocalLinks> logger,
IDataTypeService dataTypeService,
ILanguageService languageService,
IJsonSerializer jsonSerializer,
LocalLinkProcessor localLinkProcessor,
IMediaTypeService mediaTypeService)
: base(context)
{
_umbracoContextFactory = umbracoContextFactory;
_contentTypeService = contentTypeService;
_logger = logger;
_dataTypeService = dataTypeService;
_languageService = languageService;
_jsonSerializer = jsonSerializer;
_localLinkProcessor = localLinkProcessor;
_mediaTypeService = mediaTypeService;
}
protected override void Migrate()
{
IEnumerable<string> propertyEditorAliases = _localLinkProcessor.GetSupportedPropertyEditorAliases();
using UmbracoContextReference umbracoContextReference = _umbracoContextFactory.EnsureUmbracoContext();
var languagesById = _languageService.GetAllAsync().GetAwaiter().GetResult()
.ToDictionary(language => language.Id);
IEnumerable<IContentType> allContentTypes = _contentTypeService.GetAll();
IEnumerable<IPropertyType> contentPropertyTypes = allContentTypes
.SelectMany(ct => ct.PropertyTypes);
IMediaType[] allMediaTypes = _mediaTypeService.GetAll().ToArray();
IEnumerable<IPropertyType> mediaPropertyTypes = allMediaTypes
.SelectMany(ct => ct.PropertyTypes);
var relevantPropertyEditors =
contentPropertyTypes.Concat(mediaPropertyTypes).DistinctBy(pt => pt.Id)
.Where(pt => propertyEditorAliases.Contains(pt.PropertyEditorAlias))
.GroupBy(pt => pt.PropertyEditorAlias)
.ToDictionary(group => group.Key, group => group.ToArray());
foreach (var propertyEditorAlias in propertyEditorAliases)
{
if (relevantPropertyEditors.TryGetValue(propertyEditorAlias, out IPropertyType[]? propertyTypes) is false)
{
continue;
}
_logger.LogInformation(
"Migration starting for all properties of type: {propertyEditorAlias}",
propertyEditorAlias);
if (ProcessPropertyTypes(propertyTypes, languagesById))
{
_logger.LogInformation(
"Migration succeeded for all properties of type: {propertyEditorAlias}",
propertyEditorAlias);
}
else
{
_logger.LogError(
"Migration failed for one or more properties of type: {propertyEditorAlias}",
propertyEditorAlias);
}
}
}
private bool ProcessPropertyTypes(IPropertyType[] propertyTypes, IDictionary<int, ILanguage> languagesById)
{
foreach (IPropertyType propertyType in propertyTypes)
{
IDataType dataType = _dataTypeService.GetAsync(propertyType.DataTypeKey).GetAwaiter().GetResult()
?? throw new InvalidOperationException("The data type could not be fetched.");
IDataValueEditor valueEditor = dataType.Editor?.GetValueEditor()
?? throw new InvalidOperationException(
"The data type value editor could not be fetched.");
Sql<ISqlContext> sql = Sql()
.Select<PropertyDataDto>()
.From<PropertyDataDto>()
.InnerJoin<ContentVersionDto>()
.On<PropertyDataDto, ContentVersionDto>((propertyData, contentVersion) =>
propertyData.VersionId == contentVersion.Id)
.LeftJoin<DocumentVersionDto>()
.On<ContentVersionDto, DocumentVersionDto>((contentVersion, documentVersion) =>
contentVersion.Id == documentVersion.Id)
.Where<PropertyDataDto, ContentVersionDto, DocumentVersionDto>(
(propertyData, contentVersion, documentVersion) =>
(contentVersion.Current == true || documentVersion.Published == true)
&& propertyData.PropertyTypeId == propertyType.Id);
List<PropertyDataDto> propertyDataDtos = Database.Fetch<PropertyDataDto>(sql);
if (propertyDataDtos.Any() is false)
{
continue;
}
foreach (PropertyDataDto propertyDataDto in propertyDataDtos)
{
if (ProcessPropertyDataDto(propertyDataDto, propertyType, languagesById, valueEditor))
{
Database.Update(propertyDataDto);
}
}
}
return true;
}
private bool ProcessPropertyDataDto(PropertyDataDto propertyDataDto, IPropertyType propertyType,
IDictionary<int, ILanguage> languagesById, IDataValueEditor valueEditor)
{
// NOTE: some old property data DTOs can have variance defined, even if the property type no longer varies
var culture = propertyType.VariesByCulture()
&& propertyDataDto.LanguageId.HasValue
&& languagesById.TryGetValue(propertyDataDto.LanguageId.Value, out ILanguage? language)
? language.IsoCode
: null;
if (culture is null && propertyType.VariesByCulture())
{
// if we end up here, the property DTO is bound to a language that no longer exists. this is an error scenario,
// and we can't really handle it in any other way than logging; in all likelihood this is an old property version,
// and it won't cause any runtime issues
_logger.LogWarning(
" - property data with id: {propertyDataId} references a language that does not exist - language id: {languageId} (property type: {propertyTypeName}, id: {propertyTypeId}, alias: {propertyTypeAlias})",
propertyDataDto.Id,
propertyDataDto.LanguageId,
propertyType.Name,
propertyType.Id,
propertyType.Alias);
return false;
}
var segment = propertyType.VariesBySegment() ? propertyDataDto.Segment : null;
var property = new Property(propertyType);
property.SetValue(propertyDataDto.Value, culture, segment);
var toEditorValue = valueEditor.ToEditor(property, culture, segment);
_localLinkProcessor.ProcessToEditorValue(toEditorValue);
var editorValue = _jsonSerializer.Serialize(toEditorValue);
var dbValue = valueEditor.FromEditor(new ContentPropertyData(editorValue, null), null);
if (dbValue is not string stringValue || stringValue.DetectIsJson() is false)
{
_logger.LogError(
" - value editor did not yield a valid JSON string as FromEditor value property data with id: {propertyDataId} (property type: {propertyTypeName}, id: {propertyTypeId}, alias: {propertyTypeAlias})",
propertyDataDto.Id,
propertyType.Name,
propertyType.Id,
propertyType.Alias);
return false;
}
propertyDataDto.TextValue = stringValue;
return true;
}
}

View File

@@ -0,0 +1,17 @@
using Microsoft.Extensions.DependencyInjection;
using Umbraco.Cms.Core.Composing;
using Umbraco.Cms.Core.DependencyInjection;
namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks;
[Obsolete("Will be removed in V18")]
public class ConvertLocalLinkComposer : IComposer
{
public void Compose(IUmbracoBuilder builder)
{
builder.Services.AddSingleton<ITypedLocalLinkProcessor, LocalLinkBlockListProcessor>();
builder.Services.AddSingleton<ITypedLocalLinkProcessor, LocalLinkBlockGridProcessor>();
builder.Services.AddSingleton<ITypedLocalLinkProcessor, LocalLinkRteProcessor>();
builder.Services.AddSingleton<LocalLinkProcessor>();
}
}

View File

@@ -0,0 +1,11 @@
namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks;
[Obsolete("Will be removed in V18")]
public interface ITypedLocalLinkProcessor
{
public Type PropertyEditorValueType { get; }
public IEnumerable<string> PropertyEditorAliases { get; }
public Func<object?, Func<object?, bool>, Func<string, string>, bool> Process { get; }
}

View File

@@ -0,0 +1,54 @@
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Models.Blocks;
namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks;
[Obsolete("Will be removed in V18")]
public abstract class LocalLinkBlocksProcessor
{
public bool ProcessBlocks(
object? value,
Func<object?, bool> processNested,
Func<string, string> processStringValue)
{
if (value is not BlockValue blockValue)
{
return false;
}
bool hasChanged = false;
foreach (BlockItemData blockItemData in blockValue.ContentData)
{
foreach (BlockPropertyValue blockPropertyValue in blockItemData.Values)
{
if (processNested.Invoke(blockPropertyValue.Value))
{
hasChanged = true;
}
}
}
return hasChanged;
}
}
[Obsolete("Will be removed in V18")]
public class LocalLinkBlockListProcessor : LocalLinkBlocksProcessor, ITypedLocalLinkProcessor
{
public Type PropertyEditorValueType => typeof(BlockListValue);
public IEnumerable<string> PropertyEditorAliases => [Constants.PropertyEditors.Aliases.BlockList];
public Func<object?, Func<object?, bool>, Func<string, string>, bool> Process => ProcessBlocks;
}
[Obsolete("Will be removed in V18")]
public class LocalLinkBlockGridProcessor : LocalLinkBlocksProcessor, ITypedLocalLinkProcessor
{
public Type PropertyEditorValueType => typeof(BlockGridValue);
public IEnumerable<string> PropertyEditorAliases => [Constants.PropertyEditors.Aliases.BlockGrid];
public Func<object?, Func<object?, bool>, Func<string, string>, bool> Process => ProcessBlocks;
}

View File

@@ -0,0 +1,90 @@
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Models;
using Umbraco.Cms.Core.Services;
using Umbraco.Cms.Core.Templates;
namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks;
[Obsolete("Will be removed in V18")]
public class LocalLinkProcessor
{
private readonly HtmlLocalLinkParser _localLinkParser;
private readonly IIdKeyMap _idKeyMap;
private readonly IEnumerable<ITypedLocalLinkProcessor> _localLinkProcessors;
public LocalLinkProcessor(
HtmlLocalLinkParser localLinkParser,
IIdKeyMap idKeyMap,
IEnumerable<ITypedLocalLinkProcessor> localLinkProcessors)
{
_localLinkParser = localLinkParser;
_idKeyMap = idKeyMap;
_localLinkProcessors = localLinkProcessors;
}
public IEnumerable<string> GetSupportedPropertyEditorAliases() =>
_localLinkProcessors.SelectMany(p => p.PropertyEditorAliases);
public bool ProcessToEditorValue(object? editorValue)
{
ITypedLocalLinkProcessor? processor =
_localLinkProcessors.FirstOrDefault(p => p.PropertyEditorValueType == editorValue?.GetType());
return processor is not null && processor.Process.Invoke(editorValue, ProcessToEditorValue, ProcessStringValue);
}
public string ProcessStringValue(string input)
{
// find all legacy tags
var tags = _localLinkParser.FindLegacyLocalLinkIds(input).ToList();
foreach (HtmlLocalLinkParser.LocalLinkTag tag in tags)
{
string newTagHref;
if (tag.Udi is not null)
{
newTagHref = $" type=\"{tag.Udi.EntityType}\" "
+ tag.TagHref.Replace(tag.Udi.ToString(), tag.Udi.Guid.ToString());
}
else if (tag.IntId is not null)
{
// try to get the key and type from the int, else do nothing
(Guid Key, string EntityType)? conversionResult = CreateIntBasedKeyType(tag.IntId.Value);
if (conversionResult is null)
{
continue;
}
newTagHref = $" type=\"{conversionResult.Value.EntityType}\" "
+ tag.TagHref.Replace(tag.IntId.Value.ToString(), conversionResult.Value.Key.ToString());
}
else
{
// tag does not contain enough information to convert
continue;
}
input = input.Replace(tag.TagHref, newTagHref);
}
return input;
}
private (Guid Key, string EntityType)? CreateIntBasedKeyType(int id)
{
// very old data, best effort replacement
Attempt<Guid> documentAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Document);
if (documentAttempt.Success)
{
return (Key: documentAttempt.Result, EntityType: UmbracoObjectTypes.Document.ToString());
}
Attempt<Guid> mediaAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Media);
if (mediaAttempt.Success)
{
return (Key: mediaAttempt.Result, EntityType: UmbracoObjectTypes.Media.ToString());
}
return null;
}
}

View File

@@ -0,0 +1,55 @@
using Umbraco.Cms.Core;
using Umbraco.Cms.Core.Models.Blocks;
namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks;
[Obsolete("Will be removed in V18")]
public class LocalLinkRteProcessor : ITypedLocalLinkProcessor
{
public Type PropertyEditorValueType => typeof(RichTextEditorValue);
public IEnumerable<string> PropertyEditorAliases =>
[
Constants.PropertyEditors.Aliases.TinyMce, Constants.PropertyEditors.Aliases.RichText
];
public Func<object?, Func<object?, bool>, Func<string, string>, bool> Process => ProcessRichText;
public bool ProcessRichText(
object? value,
Func<object?, bool> processNested,
Func<string, string> processStringValue)
{
if (value is not RichTextEditorValue richTextValue)
{
return false;
}
bool hasChanged = false;
var newMarkup = processStringValue.Invoke(richTextValue.Markup);
if (newMarkup.Equals(richTextValue.Markup) == false)
{
hasChanged = true;
richTextValue.Markup = newMarkup;
}
if (richTextValue.Blocks is null)
{
return hasChanged;
}
foreach (BlockItemData blockItemData in richTextValue.Blocks.ContentData)
{
foreach (BlockPropertyValue blockPropertyValue in blockItemData.Values)
{
if (processNested.Invoke(blockPropertyValue.Value))
{
hasChanged = true;
}
}
}
return hasChanged;
}
}

View File

@@ -45,6 +45,24 @@ namespace Umbraco.Extensions
return sql.Where(s, a);
}
/// <summary>
/// Appends a WHERE clause to the Sql statement.
/// </summary>
/// <typeparam name="TDto1">The type of Dto 1.</typeparam>
/// <typeparam name="TDto2">The type of Dto 2.</typeparam>
/// <typeparam name="TDto3">The type of Dto 3.</typeparam>
/// <param name="sql">The Sql statement.</param>
/// <param name="predicate">A predicate to transform and append to the Sql statement.</param>
/// <param name="alias1">An optional alias for Dto 1 table.</param>
/// <param name="alias2">An optional alias for Dto 2 table.</param>
/// <param name="alias3">An optional alias for Dto 3 table.</param>
/// <returns>The Sql statement.</returns>
public static Sql<ISqlContext> Where<TDto1, TDto2, TDto3>(this Sql<ISqlContext> sql, Expression<Func<TDto1, TDto2, TDto3, bool>> predicate, string? alias1 = null, string? alias2 = null, string? alias3 = null)
{
var (s, a) = sql.SqlContext.VisitDto(predicate, alias1, alias2, alias3);
return sql.Where(s, a);
}
/// <summary>
/// Appends a WHERE IN clause to the Sql statement.
/// </summary>

View File

@@ -57,6 +57,26 @@ public static class SqlContextExtensions
return (visited, visitor.GetSqlParameters());
}
/// <summary>
/// Visit an expression.
/// </summary>
/// <typeparam name="TDto1">The type of the first DTO.</typeparam>
/// <typeparam name="TDto2">The type of the second DTO.</typeparam>
/// <typeparam name="TDto3">The type of the third DTO.</typeparam>
/// <typeparam name="TOut">The type returned by the expression.</typeparam>
/// <param name="sqlContext">An <see cref="ISqlContext" />.</param>
/// <param name="expression">An expression to visit.</param>
/// <param name="alias1">An optional table alias for the first DTO.</param>
/// <param name="alias2">An optional table alias for the second DTO.</param>
/// <param name="alias3">An optional table alias for the third DTO.</param>
/// <returns>A SQL statement, and arguments, corresponding to the expression.</returns>
public static (string Sql, object[] Args) VisitDto<TDto1, TDto2, TDto3, TOut>(this ISqlContext sqlContext, Expression<Func<TDto1, TDto2, TDto3, TOut>> expression, string? alias1 = null, string? alias2 = null, string? alias3 = null)
{
var visitor = new PocoToSqlExpressionVisitor<TDto1, TDto2, TDto3>(sqlContext, alias1, alias2, alias3);
var visited = visitor.Visit(expression);
return (visited, visitor.GetSqlParameters());
}
/// <summary>
/// Visit an expression.
/// </summary>