yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11369 from umbraco/v8/bugfix/maxparametercount

Browse Source 
Add Constants.Sql.MaxParameterCount and update query logic
This commit is contained in:
Paul Johnson
2021-10-22 21:08:55 +01:00
committed by GitHub
parent 3fb150dda3 60be71d1a0
commit c61eaa2257
16 changed files with 80 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/Umbraco.Core/Constants-Sql.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace Umbraco.Core
{
public static partial class Constants
{
public static class Sql
{
/// <summary>
/// The maximum amount of parameters that can be used in a query.
/// </summary>
/// <remarks>
/// The actual limit is 2100 (https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server),
/// but we want to ensure there's room for additional parameters if this value is used to create groups/batches.
/// </remarks>
public const int MaxParameterCount = 2000;
}
}
}

2
src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs
View File

@@ -137,7 +137,7 @@ namespace Umbraco.Core.Persistence
// Math.Floor(2100 / 8) = 262 record per command
// 4168 / 262 = 15.908... = there will be 16 command in total
// (if we have disabled db parameters, then all records will be included, in only one command)
var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor(2000.00 / paramsPerRecord));
var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor((double)Constants.Sql.MaxParameterCount / paramsPerRecord));
var commandsCount = Convert.ToInt32(Math.Ceiling((double)records.Length / recordsPerCommand));
var commands = new IDbCommand[commandsCount];

2
src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs
View File

@@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
var entries = new List<IAuditEntry>();
foreach (var group in ids.InGroupsOf(2000))
foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
{
var sql = Sql()
.Select<AuditEntryDto>()

4
src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs
View File

@@ -638,7 +638,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// in the table?
// get all PropertyDataDto for all definitions / versions
var allPropertyDataDtos = Database.FetchByGroups<PropertyDataDto, int>(versions, 2000, batch =>
var allPropertyDataDtos = Database.FetchByGroups<PropertyDataDto, int>(versions, Constants.Sql.MaxParameterCount, batch =>
SqlContext.Sql()
.Select<PropertyDataDto>()
.From<PropertyDataDto>()
@@ -647,7 +647,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// get PropertyDataDto distinct PropertyTypeDto
var allPropertyTypeIds = allPropertyDataDtos.Select(x => x.PropertyTypeId).Distinct().ToList();
var allPropertyTypeDtos = Database.FetchByGroups<PropertyTypeDto, int>(allPropertyTypeIds, 2000, batch =>
var allPropertyTypeDtos = Database.FetchByGroups<PropertyTypeDto, int>(allPropertyTypeIds, Constants.Sql.MaxParameterCount, batch =>
SqlContext.Sql()
.Select<PropertyTypeDto>(r => r.Select(x => x.DataTypeDto))
.From<PropertyTypeDto>()

28
src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs
View File

@@ -768,7 +768,7 @@ AND umbracoNode.id <> @id",
// note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
if (whereInArgsCount > 2000)
if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
// delete existing relations (for target language)
@@ -906,7 +906,7 @@ AND umbracoNode.id <> @id",
// note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
//
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
if (whereInArgsCount > 2000)
if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
//first clear out any existing property data that might already exists under the target language
@@ -1005,7 +1005,7 @@ AND umbracoNode.id <> @id",
//based on the current variance of each item to see if it's 'edited' value should be true/false.
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
if (whereInArgsCount > 2000)
if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
var propertySql = Sql()
@@ -1094,14 +1094,20 @@ AND umbracoNode.id <> @id",
}
}
//lookup all matching rows in umbracoDocumentCultureVariation
var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(2000)
.SelectMany(_ => Database.Fetch<DocumentCultureVariationDto>(
Sql().Select<DocumentCultureVariationDto>().From<DocumentCultureVariationDto>()
.WhereIn<DocumentCultureVariationDto>(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList())
.WhereIn<DocumentCultureVariationDto>(x => x.NodeId, editedLanguageVersions.Keys.Select(x => x.nodeId))))
//convert to dictionary with the same key type
.ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x);
// lookup all matching rows in umbracoDocumentCultureVariation
// fetch in batches to account for maximum parameter count (distinct languages can't exceed 2000)
var languageIds = editedLanguageVersions.Keys.Select(x => x.langId).Distinct().ToArray();
var nodeIds = editedLanguageVersions.Keys.Select(x => x.nodeId).Distinct();
var docCultureVariationsToUpdate = nodeIds.InGroupsOf(Constants.Sql.MaxParameterCount - languageIds.Length)
.SelectMany(group =>
{
var sql = Sql().Select<DocumentCultureVariationDto>().From<DocumentCultureVariationDto>()
.WhereIn<DocumentCultureVariationDto>(x => x.LanguageId, languageIds)
.WhereIn<DocumentCultureVariationDto>(x => x.NodeId, group);
return Database.Fetch<DocumentCultureVariationDto>(sql);
})
.ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); //convert to dictionary with the same key type
var toUpdate = new List<DocumentCultureVariationDto>();
foreach (var ev in editedLanguageVersions)

7
src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs
View File

@@ -260,13 +260,12 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
Func<Guid[], IEnumerable<IEnumerable<IDictionaryItem>>> getItemsFromParents = guids =>
{
//needs to be in groups of 2000 because we are doing an IN clause and there's a max parameter count that can be used.
return guids.InGroupsOf(2000)
.Select(@group =>
return guids.InGroupsOf(Constants.Sql.MaxParameterCount)
.Select(group =>
{
var sqlClause = GetBaseQuery(false)
.Where<DictionaryDto>(x => x.Parent != null)
.Where($"{SqlSyntax.GetQuotedColumnName("parent")} IN (@parentIds)", new { parentIds = @group });
.WhereIn<DictionaryDto>(x => x.Parent, group);
var translator = new SqlTranslator<IDictionaryItem>(sqlClause, Query<IDictionaryItem>());
var sql = translator.Translate();

6
src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs
View File

@@ -1342,7 +1342,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
var result = new Dictionary<int, ContentScheduleCollection>();
var scheduleDtos = Database.FetchByGroups<ContentScheduleDto, int>(contentIds, 2000, batch => Sql()
var scheduleDtos = Database.FetchByGroups<ContentScheduleDto, int>(contentIds, Constants.Sql.MaxParameterCount, batch => Sql()
.Select<ContentScheduleDto>()
.From<ContentScheduleDto>()
.WhereIn<ContentScheduleDto>(x => x.NodeId, batch));
@@ -1391,7 +1391,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
}
if (versions.Count == 0) return new Dictionary<int, List<ContentVariation>>();
var dtos = Database.FetchByGroups<ContentVersionCultureVariationDto, int>(versions, 2000, batch
var dtos = Database.FetchByGroups<ContentVersionCultureVariationDto, int>(versions, Constants.Sql.MaxParameterCount, batch
=> Sql()
.Select<ContentVersionCultureVariationDto>()
.From<ContentVersionCultureVariationDto>()
@@ -1420,7 +1420,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
var ids = temps.Select(x => x.Id);
var dtos = Database.FetchByGroups<DocumentCultureVariationDto, int>(ids, 2000, batch =>
var dtos = Database.FetchByGroups<DocumentCultureVariationDto, int>(ids, Constants.Sql.MaxParameterCount, batch =>
Sql()
.Select<DocumentCultureVariationDto>()
.From<DocumentCultureVariationDto>()

2
src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs
View File

@@ -60,7 +60,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
if (ids.Any())
{
return Database.FetchByGroups<NodeDto, int>(ids, 2000, batch =>
return Database.FetchByGroups<NodeDto, int>(ids, Constants.Sql.MaxParameterCount, batch =>
GetBaseQuery(false)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId)
.WhereIn<NodeDto>(x => x.NodeId, batch))

2
src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs
View File

@@ -281,7 +281,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
if (v == null) return entitiesList;
// fetch all variant info dtos
var dtos = Database.FetchByGroups<VariantInfoDto, int>(v.Select(x => x.Id), 2000, GetVariantInfos);
var dtos = Database.FetchByGroups<VariantInfoDto, int>(v.Select(x => x.Id), Constants.Sql.MaxParameterCount, GetVariantInfos);
// group by node id (each group contains all languages)
var xdtos = dtos.GroupBy(x => x.NodeId).ToDictionary(x => x.Key, x => x);

11
src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs
View File

@@ -431,16 +431,13 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
var matchedMembers = Get(query).ToArray();
var membersInGroup = new List<IMember>();
//then we need to filter the matched members that are in the role
//since the max sql params are 2100 on sql server, we'll reduce that to be safe for potentially other servers and run the queries in batches
var inGroups = matchedMembers.InGroupsOf(1000);
foreach (var batch in inGroups)
{
var memberIdBatch = batch.Select(x => x.Id);
//then we need to filter the matched members that are in the role
foreach (var group in matchedMembers.Select(x => x.Id).InGroupsOf(Constants.Sql.MaxParameterCount))
{
var sql = Sql().SelectAll().From<Member2MemberGroupDto>()
.Where<Member2MemberGroupDto>(dto => dto.MemberGroup == memberGroup.Id)
.WhereIn<Member2MemberGroupDto>(dto => dto.Member, memberIdBatch);
.WhereIn<Member2MemberGroupDto>(dto => dto.Member, group);
var memberIdsInGroup = Database.Fetch<Member2MemberGroupDto>(sql)
.Select(x => x.Member).ToArray();

42
src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs
View File

@@ -38,44 +38,41 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
/// <param name="entityIds"></param>
/// <returns></returns>
/// <remarks>
/// This method will not support passing in more than 2000 group Ids
/// This method will not support passing in more than 2000 group IDs when also passing in entity IDs.
/// </remarks>
public EntityPermissionCollection GetPermissionsForEntities(int[] groupIds, params int[] entityIds)
{
var result = new EntityPermissionCollection();
foreach (var groupOfGroupIds in groupIds.InGroupsOf(2000))
if (entityIds.Length == 0)
{
//copy local
var localIds = groupOfGroupIds.ToArray();
if (entityIds.Length == 0)
foreach (var group in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
var sql = Sql()
.SelectAll()
.From<UserGroup2NodePermissionDto>()
.Where<UserGroup2NodePermissionDto>(dto => localIds.Contains(dto.UserGroupId));
.Where<UserGroup2NodePermissionDto>(dto => group.Contains(dto.UserGroupId));
var permissions = AmbientScope.Database.Fetch<UserGroup2NodePermissionDto>(sql);
foreach (var permission in ConvertToPermissionList(permissions))
{
result.Add(permission);
}
}
else
}
else
{
foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount - groupIds.Length))
{
//iterate in groups of 2000 since we don't want to exceed the max SQL param count
foreach (var groupOfEntityIds in entityIds.InGroupsOf(2000))
var sql = Sql()
.SelectAll()
.From<UserGroup2NodePermissionDto>()
.Where<UserGroup2NodePermissionDto>(dto => groupIds.Contains(dto.UserGroupId) && group.Contains(dto.NodeId));
var permissions = AmbientScope.Database.Fetch<UserGroup2NodePermissionDto>(sql);
foreach (var permission in ConvertToPermissionList(permissions))
{
var ids = groupOfEntityIds;
var sql = Sql()
.SelectAll()
.From<UserGroup2NodePermissionDto>()
.Where<UserGroup2NodePermissionDto>(dto => localIds.Contains(dto.UserGroupId) && ids.Contains(dto.NodeId));
var permissions = AmbientScope.Database.Fetch<UserGroup2NodePermissionDto>(sql);
foreach (var permission in ConvertToPermissionList(permissions))
{
result.Add(permission);
}
result.Add(permission);
}
}
}
@@ -133,11 +130,10 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
var db = AmbientScope.Database;
//we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
var sql = "DELETE FROM umbracoUserGroup2NodePermission WHERE userGroupId = @groupId AND nodeId in (@nodeIds)";
foreach (var idGroup in entityIds.InGroupsOf(2000))
foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
db.Execute(sql, new { groupId, nodeIds = idGroup });
db.Execute(sql, new { groupId, nodeIds = group });
}
var toInsert = new List<UserGroup2NodePermissionDto>();

5
src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs
View File

@@ -37,8 +37,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected override IEnumerable<IRedirectUrl> PerformGetAll(params Guid[] ids)
{
if (ids.Length > 2000)
throw new NotSupportedException("This repository does not support more than 2000 ids.");
if (ids.Length > Constants.Sql.MaxParameterCount)
throw new NotSupportedException($"This repository does not support more than {Constants.Sql.MaxParameterCount} ids.");
var sql = GetBaseQuery(false).WhereIn<RedirectUrlDto>(x => x.Id, ids);
var dtos = Database.Fetch<RedirectUrlDto>(sql);
return dtos.WhereNotNull().Select(Map);

8
src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs
View File

@@ -186,17 +186,17 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// can't query more than 2000 ids at a time... but if someone is really querying 2000+ entities,
// the additional overhead of fetching them in groups is minimal compared to the lookup time of each group
const int maxParams = 2000;
if (ids.Length <= maxParams)
if (ids.Length <= Constants.Sql.MaxParameterCount)
{
return CachePolicy.GetAll(ids, PerformGetAll);
}
var entities = new List<TEntity>();
foreach (var groupOfIds in ids.InGroupsOf(maxParams))
foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
{
entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll));
entities.AddRange(CachePolicy.GetAll(group.ToArray(), PerformGetAll));
}
return entities;
}

2
src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs
View File

@@ -39,7 +39,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
var dtos = ids.Length == 0
? Database.Fetch<TagDto>(Sql().Select<TagDto>().From<TagDto>())
: Database.FetchByGroups<TagDto, int>(ids, 2000, batch => Sql().Select<TagDto>().From<TagDto>().WhereIn<TagDto>(x => x.Id, batch));
: Database.FetchByGroups<TagDto, int>(ids, Constants.Sql.MaxParameterCount, batch => Sql().Select<TagDto>().From<TagDto>().WhereIn<TagDto>(x => x.Id, batch));
return dtos.Select(TagFactory.BuildEntity).ToList();
}

3
src/Umbraco.Core/Umbraco.Core.csproj
View File

@@ -132,6 +132,7 @@
<Compile Include="Configuration\ICoreDebug.cs" />
<Compile Include="Constants-CharArrays.cs" />
<Compile Include="Collections\EventClearingObservableCollection.cs" />
<Compile Include="Constants-Sql.cs" />
<Compile Include="Constants-SqlTemplates.cs" />
<Compile Include="Events\UnattendedInstallEventArgs.cs" />
<Compile Include="Logging\ILogger2.cs" />
@@ -1665,4 +1666,4 @@
<Folder Include="Auditing\" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>
</Project>

2
src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs
View File

@@ -240,7 +240,7 @@ namespace Umbraco.Web.Models.Mapping
// the entity service due to too many Sql parameters.
var list = new List<IEntitySlim>();
foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000))
foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount))
list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray()));
contentEntities = list.ToArray();
}