Backport relation tracking fixes and get references from recursive (nested/block) properties (#15593)

* Include automatic relation type aliases from factory and fix SQL parameter overflow (#15141)

* Include automatic relation type aliases from factory

* Remove unnessecary distinct and fix SQL parameter overflow issue

* Fixed assertions and test distinct aliases

* Simplified collection assertions

* Improve logging of invalid reference relations (#15160)

* Include automatic relation type aliases from factory

* Remove unnessecary distinct and fix SQL parameter overflow issue

* Fixed assertions and test distinct aliases

* Simplified collection assertions

* Improve logging of invalid reference relations

* Always get all automatic relation type aliases

* Do not set relation type alias for unknown entity types

* Get references from recursive (nested/block) properties

(cherry picked from commit 5198e7c52d)
This commit is contained in:
Ronald Barendse
2024-01-19 20:02:57 +01:00
committed by Bjarke Berg
parent a70c606c27
commit ce22315520
11 changed files with 336 additions and 234 deletions

View File

@@ -1083,91 +1083,59 @@ namespace Umbraco.Cms.Infrastructure.Persistence.Repositories.Implement
protected void PersistRelations(TEntity entity)
{
// Get all references from our core built in DataEditors/Property Editors
// Along with seeing if deverlopers want to collect additional references from the DataValueReferenceFactories collection
var trackedRelations = new List<UmbracoEntityReference>();
trackedRelations.AddRange(_dataValueReferenceFactories.GetAllReferences(entity.Properties, PropertyEditors));
var relationTypeAliases = GetAutomaticRelationTypesAliases(entity.Properties, PropertyEditors).ToList();
// At this point we potentially have a problem (see for example: https://github.com/umbraco/Umbraco.Forms.Issues/issues/1129).
// If we have a custom relation type (i.e. not document or media) and use of this within a block grid/list property editor,
// we'll get an error when saving the relations.
// It happens because the block grid doesn't expose the automatic relation type aliases for all of it's nested properties, and
// as such relationTypeAliases does not contain the custom relation type alias.
// Auto-relations of that type are then then not deleted, and we get duplicate error on insert.
// To resolve we can look at the relations we are going to be saving, and if they include any relation type aliases we haven't
// already identified, we'll add them in, so they will also be removed along with the other auto-relations.
relationTypeAliases.AddRange(trackedRelations.Select(x => x.RelationTypeAlias).Distinct().Except(relationTypeAliases));
// Along with seeing if developers want to collect additional references from the DataValueReferenceFactories collection
ISet<UmbracoEntityReference> references = _dataValueReferenceFactories.GetAllReferences(entity.Properties, PropertyEditors);
// First delete all auto-relations for this entity
RelationRepository.DeleteByParent(entity.Id, relationTypeAliases.ToArray());
ISet<string> automaticRelationTypeAliases = _dataValueReferenceFactories.GetAllAutomaticRelationTypesAliases(PropertyEditors);
RelationRepository.DeleteByParent(entity.Id, automaticRelationTypeAliases.ToArray());
if (trackedRelations.Count == 0)
if (references.Count == 0)
{
// No need to add new references/relations
return;
}
trackedRelations = trackedRelations.Distinct().ToList();
var udiToGuids = trackedRelations.Select(x => x.Udi as GuidUdi)
.ToDictionary(x => (Udi)x!, x => x!.Guid);
// lookup in the DB all INT ids for the GUIDs and chuck into a dictionary
var keyToIds = Database.Fetch<NodeIdKey>(Sql()
.Select<NodeDto>(x => x.NodeId, x => x.UniqueId)
.From<NodeDto>()
.WhereIn<NodeDto>(x => x.UniqueId, udiToGuids.Values))
.ToDictionary(x => x.UniqueId, x => x.NodeId);
var allRelationTypes = RelationTypeRepository.GetMany(Array.Empty<int>())?
.ToDictionary(x => x.Alias, x => x);
IEnumerable<ReadOnlyRelation> toSave = trackedRelations.Select(rel =>
{
if (allRelationTypes is null || !allRelationTypes.TryGetValue(rel.RelationTypeAlias, out IRelationType? relationType))
{
throw new InvalidOperationException($"The relation type {rel.RelationTypeAlias} does not exist");
}
if (!udiToGuids.TryGetValue(rel.Udi, out Guid guid))
{
return null; // This shouldn't happen!
}
if (!keyToIds.TryGetValue(guid, out var id))
{
return null; // This shouldn't happen!
}
return new ReadOnlyRelation(entity.Id, id, relationType.Id);
}).WhereNotNull();
// Save bulk relations
RelationRepository.SaveBulk(toSave);
}
private IEnumerable<string> GetAutomaticRelationTypesAliases(
IPropertyCollection properties,
PropertyEditorCollection propertyEditors)
{
var automaticRelationTypesAliases = new HashSet<string>(Constants.Conventions.RelationTypes.AutomaticRelationTypes);
foreach (IProperty property in properties)
// Lookup node IDs for all GUID based UDIs
IEnumerable<Guid> keys = references.Select(x => x.Udi).OfType<GuidUdi>().Select(x => x.Guid);
var keysLookup = Database.FetchByGroups<NodeIdKey, Guid>(keys, Constants.Sql.MaxParameterCount, guids =>
{
if (propertyEditors.TryGet(property.PropertyType.PropertyEditorAlias, out IDataEditor? editor) is false )
{
continue;
}
return Sql()
.Select<NodeDto>(x => x.NodeId, x => x.UniqueId)
.From<NodeDto>()
.WhereIn<NodeDto>(x => x.UniqueId, guids);
}).ToDictionary(x => x.UniqueId, x => x.NodeId);
if (editor.GetValueEditor() is IDataValueReference reference)
// Lookup all relation type IDs
var relationTypeLookup = RelationTypeRepository.GetMany(Array.Empty<int>()).ToDictionary(x => x.Alias, x => x.Id);
// Get all valid relations
var relations = new List<ReadOnlyRelation>(references.Count);
foreach (UmbracoEntityReference reference in references)
{
if (!automaticRelationTypeAliases.Contains(reference.RelationTypeAlias))
{
foreach (var alias in reference.GetAutomaticRelationTypesAliases())
{
automaticRelationTypesAliases.Add(alias);
}
// Returning a reference that doesn't use an automatic relation type is an issue that should be fixed in code
Logger.LogError("The reference to {Udi} uses a relation type {RelationTypeAlias} that is not an automatic relation type.", reference.Udi, reference.RelationTypeAlias);
}
else if (!relationTypeLookup.TryGetValue(reference.RelationTypeAlias, out int relationTypeId))
{
// A non-existent relation type could be caused by an environment issue (e.g. it was manually removed)
Logger.LogWarning("The reference to {Udi} uses a relation type {RelationTypeAlias} that does not exist.", reference.Udi, reference.RelationTypeAlias);
}
else if (reference.Udi is not GuidUdi udi || !keysLookup.TryGetValue(udi.Guid, out var id))
{
// Relations only support references to items that are stored in the NodeDto table (because of foreign key constraints)
Logger.LogInformation("The reference to {Udi} can not be saved as relation, because doesn't have a node ID.", reference.Udi);
}
else
{
relations.Add(new ReadOnlyRelation(entity.Id, id, relationTypeId));
}
}
return automaticRelationTypesAliases;
// Save bulk relations
RelationRepository.SaveBulk(relations);
}
/// <summary>

View File

@@ -17,12 +17,14 @@ internal abstract class BlockEditorPropertyValueEditor : DataValueEditor, IDataV
{
private BlockEditorValues? _blockEditorValues;
private readonly IDataTypeService _dataTypeService;
private readonly ILogger<BlockEditorPropertyValueEditor> _logger;
private readonly PropertyEditorCollection _propertyEditors;
private readonly DataValueReferenceFactoryCollection _dataValueReferenceFactories;
private readonly ILogger<BlockEditorPropertyValueEditor> _logger;
protected BlockEditorPropertyValueEditor(
DataEditorAttribute attribute,
PropertyEditorCollection propertyEditors,
DataValueReferenceFactoryCollection dataValueReferenceFactories,
IDataTypeService dataTypeService,
ILocalizedTextService textService,
ILogger<BlockEditorPropertyValueEditor> logger,
@@ -32,6 +34,7 @@ internal abstract class BlockEditorPropertyValueEditor : DataValueEditor, IDataV
: base(textService, shortStringHelper, jsonSerializer, ioHelper, attribute)
{
_propertyEditors = propertyEditors;
_dataValueReferenceFactories = dataValueReferenceFactories;
_dataTypeService = dataTypeService;
_logger = logger;
}
@@ -42,73 +45,58 @@ internal abstract class BlockEditorPropertyValueEditor : DataValueEditor, IDataV
set => _blockEditorValues = value;
}
/// <inheritdoc />
public IEnumerable<UmbracoEntityReference> GetReferences(object? value)
{
var rawJson = value == null ? string.Empty : value is string str ? str : value.ToString();
var result = new List<UmbracoEntityReference>();
BlockEditorData? blockEditorData = BlockEditorValues.DeserializeAndClean(rawJson);
if (blockEditorData == null)
foreach (BlockItemData.BlockPropertyValue propertyValue in GetAllPropertyValues(value))
{
return Enumerable.Empty<UmbracoEntityReference>();
}
// loop through all content and settings data
foreach (BlockItemData row in blockEditorData.BlockValue.ContentData.Concat(blockEditorData.BlockValue.SettingsData))
{
foreach (KeyValuePair<string, BlockItemData.BlockPropertyValue> prop in row.PropertyValues)
if (!_propertyEditors.TryGet(propertyValue.PropertyType.PropertyEditorAlias, out IDataEditor? dataEditor))
{
IDataEditor? propEditor = _propertyEditors[prop.Value.PropertyType.PropertyEditorAlias];
continue;
}
IDataValueEditor? valueEditor = propEditor?.GetValueEditor();
if (!(valueEditor is IDataValueReference reference))
{
continue;
}
var val = prop.Value.Value?.ToString();
IEnumerable<UmbracoEntityReference> refs = reference.GetReferences(val);
result.AddRange(refs);
foreach (UmbracoEntityReference reference in _dataValueReferenceFactories.GetReferences(dataEditor, propertyValue.Value))
{
yield return reference;
}
}
return result;
}
/// <inheritdoc />
public IEnumerable<ITag> GetTags(object? value, object? dataTypeConfiguration, int? languageId)
{
foreach (BlockItemData.BlockPropertyValue propertyValue in GetAllPropertyValues(value))
{
if (!_propertyEditors.TryGet(propertyValue.PropertyType.PropertyEditorAlias, out IDataEditor? dataEditor) ||
dataEditor.GetValueEditor() is not IDataValueTags dataValueTags)
{
continue;
}
object? configuration = _dataTypeService.GetDataType(propertyValue.PropertyType.DataTypeKey)?.Configuration;
foreach (ITag tag in dataValueTags.GetTags(propertyValue.Value, configuration, languageId))
{
yield return tag;
}
}
}
private IEnumerable<BlockItemData.BlockPropertyValue> GetAllPropertyValues(object? value)
{
var rawJson = value == null ? string.Empty : value is string str ? str : value.ToString();
BlockEditorData? blockEditorData = BlockEditorValues.DeserializeAndClean(rawJson);
if (blockEditorData == null)
if (blockEditorData is null)
{
return Enumerable.Empty<ITag>();
yield break;
}
var result = new List<ITag>();
// loop through all content and settings data
foreach (BlockItemData row in blockEditorData.BlockValue.ContentData.Concat(blockEditorData.BlockValue.SettingsData))
// Return all property values from the content and settings data
IEnumerable<BlockItemData> data = blockEditorData.BlockValue.ContentData.Concat(blockEditorData.BlockValue.SettingsData);
foreach (BlockItemData.BlockPropertyValue propertyValue in data.SelectMany(x => x.PropertyValues.Select(x => x.Value)))
{
foreach (KeyValuePair<string, BlockItemData.BlockPropertyValue> prop in row.PropertyValues)
{
IDataEditor? propEditor = _propertyEditors[prop.Value.PropertyType.PropertyEditorAlias];
IDataValueEditor? valueEditor = propEditor?.GetValueEditor();
if (valueEditor is not IDataValueTags tagsProvider)
{
continue;
}
object? configuration = _dataTypeService.GetDataType(prop.Value.PropertyType.DataTypeKey)?.Configuration;
result.AddRange(tagsProvider.GetTags(prop.Value.Value, configuration, languageId));
}
yield return propertyValue;
}
return result;
}
#region Convert database // editor

View File

@@ -50,6 +50,7 @@ public abstract class BlockGridPropertyEditorBase : DataEditor
public BlockGridEditorPropertyValueEditor(
DataEditorAttribute attribute,
PropertyEditorCollection propertyEditors,
DataValueReferenceFactoryCollection dataValueReferenceFactories,
IDataTypeService dataTypeService,
ILocalizedTextService textService,
ILogger<BlockEditorPropertyValueEditor> logger,
@@ -58,7 +59,7 @@ public abstract class BlockGridPropertyEditorBase : DataEditor
IIOHelper ioHelper,
IContentTypeService contentTypeService,
IPropertyValidationService propertyValidationService)
: base(attribute, propertyEditors, dataTypeService, textService, logger, shortStringHelper, jsonSerializer, ioHelper)
: base(attribute, propertyEditors, dataValueReferenceFactories, dataTypeService, textService, logger, shortStringHelper, jsonSerializer, ioHelper)
{
BlockEditorValues = new BlockEditorValues(new BlockGridEditorDataConverter(jsonSerializer), contentTypeService, logger);
Validators.Add(new BlockEditorValidator(propertyValidationService, BlockEditorValues, contentTypeService));

View File

@@ -46,6 +46,7 @@ public abstract class BlockListPropertyEditorBase : DataEditor
public BlockListEditorPropertyValueEditor(
DataEditorAttribute attribute,
PropertyEditorCollection propertyEditors,
DataValueReferenceFactoryCollection dataValueReferenceFactories,
IDataTypeService dataTypeService,
IContentTypeService contentTypeService,
ILocalizedTextService textService,
@@ -54,7 +55,7 @@ public abstract class BlockListPropertyEditorBase : DataEditor
IJsonSerializer jsonSerializer,
IIOHelper ioHelper,
IPropertyValidationService propertyValidationService) :
base(attribute, propertyEditors, dataTypeService, textService, logger, shortStringHelper, jsonSerializer, ioHelper)
base(attribute, propertyEditors, dataValueReferenceFactories,dataTypeService, textService, logger, shortStringHelper, jsonSerializer, ioHelper)
{
BlockEditorValues = new BlockEditorValues(new BlockListEditorDataConverter(), contentTypeService, logger);
Validators.Add(new BlockEditorValidator(propertyValidationService, BlockEditorValues, contentTypeService));

View File

@@ -91,9 +91,10 @@ public class NestedContentPropertyEditor : DataEditor
internal class NestedContentPropertyValueEditor : DataValueEditor, IDataValueReference, IDataValueTags
{
private readonly IDataTypeService _dataTypeService;
private readonly PropertyEditorCollection _propertyEditors;
private readonly DataValueReferenceFactoryCollection _dataValueReferenceFactories;
private readonly ILogger<NestedContentPropertyEditor> _logger;
private readonly NestedContentValues _nestedContentValues;
private readonly PropertyEditorCollection _propertyEditors;
public NestedContentPropertyValueEditor(
IDataTypeService dataTypeService,
@@ -102,16 +103,19 @@ public class NestedContentPropertyEditor : DataEditor
IShortStringHelper shortStringHelper,
DataEditorAttribute attribute,
PropertyEditorCollection propertyEditors,
DataValueReferenceFactoryCollection dataValueReferenceFactories,
ILogger<NestedContentPropertyEditor> logger,
IJsonSerializer jsonSerializer,
IIOHelper ioHelper,
IPropertyValidationService propertyValidationService)
: base(localizedTextService, shortStringHelper, jsonSerializer, ioHelper, attribute)
{
_propertyEditors = propertyEditors;
_dataTypeService = dataTypeService;
_propertyEditors = propertyEditors;
_dataValueReferenceFactories = dataValueReferenceFactories;
_logger = logger;
_nestedContentValues = new NestedContentValues(contentTypeService);
Validators.Add(new NestedContentValidator(propertyValidationService, _nestedContentValues, contentTypeService));
}
@@ -139,66 +143,45 @@ public class NestedContentPropertyEditor : DataEditor
}
}
/// <inheritdoc />
public IEnumerable<UmbracoEntityReference> GetReferences(object? value)
{
var rawJson = value == null ? string.Empty : value is string str ? str : value.ToString();
var result = new List<UmbracoEntityReference>();
foreach (NestedContentValues.NestedContentRowValue row in _nestedContentValues.GetPropertyValues(rawJson))
foreach (NestedContentValues.NestedContentPropertyValue propertyValue in GetAllPropertyValues(value))
{
foreach (KeyValuePair<string, NestedContentValues.NestedContentPropertyValue> prop in
row.PropertyValues)
if (!_propertyEditors.TryGet(propertyValue.PropertyType.PropertyEditorAlias, out IDataEditor? dataEditor))
{
IDataEditor? propEditor = _propertyEditors[prop.Value.PropertyType.PropertyEditorAlias];
continue;
}
IDataValueEditor? valueEditor = propEditor?.GetValueEditor();
if (!(valueEditor is IDataValueReference reference))
{
continue;
}
var val = prop.Value.Value?.ToString();
IEnumerable<UmbracoEntityReference> refs = reference.GetReferences(val);
result.AddRange(refs);
foreach (UmbracoEntityReference reference in _dataValueReferenceFactories.GetReferences(dataEditor, propertyValue.Value))
{
yield return reference;
}
}
return result;
}
/// <inheritdoc />
public IEnumerable<ITag> GetTags(object? value, object? dataTypeConfiguration, int? languageId)
{
IReadOnlyList<NestedContentValues.NestedContentRowValue> rows =
_nestedContentValues.GetPropertyValues(value);
var result = new List<ITag>();
foreach (NestedContentValues.NestedContentRowValue row in rows.ToList())
foreach (NestedContentValues.NestedContentPropertyValue propertyValue in GetAllPropertyValues(value))
{
foreach (KeyValuePair<string, NestedContentValues.NestedContentPropertyValue> prop in row.PropertyValues
.ToList())
if (!_propertyEditors.TryGet(propertyValue.PropertyType.PropertyEditorAlias, out IDataEditor? dataEditor) ||
dataEditor.GetValueEditor() is not IDataValueTags dataValueTags)
{
IDataEditor? propEditor = _propertyEditors[prop.Value.PropertyType.PropertyEditorAlias];
continue;
}
IDataValueEditor? valueEditor = propEditor?.GetValueEditor();
if (valueEditor is not IDataValueTags tagsProvider)
{
continue;
}
object? configuration = _dataTypeService.GetDataType(prop.Value.PropertyType.DataTypeKey)?.Configuration;
result.AddRange(tagsProvider.GetTags(prop.Value.Value, configuration, languageId));
object? configuration = _dataTypeService.GetDataType(propertyValue.PropertyType.DataTypeKey)?.Configuration;
foreach (ITag tag in dataValueTags.GetTags(propertyValue.Value, configuration, languageId))
{
yield return tag;
}
}
return result;
}
private IEnumerable<NestedContentValues.NestedContentPropertyValue> GetAllPropertyValues(object? value)
=> _nestedContentValues.GetPropertyValues(value).SelectMany(x => x.PropertyValues.Values);
#region DB to String
public override string ConvertDbToString(IPropertyType propertyType, object? propertyValue)
@@ -424,7 +407,8 @@ public class NestedContentPropertyEditor : DataEditor
// set values to null
row.PropertyValues[elementTypeProp.Alias] = new NestedContentValues.NestedContentPropertyValue
{
PropertyType = elementTypeProp, Value = null,
PropertyType = elementTypeProp,
Value = null,
};
row.RawPropertyValues[elementTypeProp.Alias] = null;
}