yv01p/Umbraco-CMS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace arbitrary group size with constant

Browse Source
This commit is contained in:
Ronald Barendse
2021-10-13 13:42:18 +02:00
parent c5ba23a15c
commit a6b04a941c
13 changed files with 25 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs
View File

@@ -137,7 +137,7 @@ namespace Umbraco.Core.Persistence
// Math.Floor(2100 / 8) = 262 record per command
// 4168 / 262 = 15.908... = there will be 16 command in total
// (if we have disabled db parameters, then all records will be included, in only one command)
var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor(2000.00 / paramsPerRecord));
var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor((double)Constants.Sql.MaxParameterCount / paramsPerRecord));
var commandsCount = Convert.ToInt32(Math.Ceiling((double)records.Length / recordsPerCommand));
var commands = new IDbCommand[commandsCount];

2
src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs
View File

@@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
var entries = new List<IAuditEntry>();
foreach (var group in ids.InGroupsOf(2000))
foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
{
var sql = Sql()
.Select<AuditEntryDto>()

4
src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs
View File

@@ -638,7 +638,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// in the table?
// get all PropertyDataDto for all definitions / versions
var allPropertyDataDtos = Database.FetchByGroups<PropertyDataDto, int>(versions, 2000, batch =>
var allPropertyDataDtos = Database.FetchByGroups<PropertyDataDto, int>(versions, Constants.Sql.MaxParameterCount, batch =>
SqlContext.Sql()
.Select<PropertyDataDto>()
.From<PropertyDataDto>()
@@ -647,7 +647,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// get PropertyDataDto distinct PropertyTypeDto
var allPropertyTypeIds = allPropertyDataDtos.Select(x => x.PropertyTypeId).Distinct().ToList();
var allPropertyTypeDtos = Database.FetchByGroups<PropertyTypeDto, int>(allPropertyTypeIds, 2000, batch =>
var allPropertyTypeDtos = Database.FetchByGroups<PropertyTypeDto, int>(allPropertyTypeIds, Constants.Sql.MaxParameterCount, batch =>
SqlContext.Sql()
.Select<PropertyTypeDto>(r => r.Select(x => x.DataTypeDto))
.From<PropertyTypeDto>()

8
src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs
View File

@@ -768,7 +768,7 @@ AND umbracoNode.id <> @id",
// note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
if (whereInArgsCount > 2000)
if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
// delete existing relations (for target language)
@@ -906,7 +906,7 @@ AND umbracoNode.id <> @id",
// note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers
//
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
if (whereInArgsCount > 2000)
if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
//first clear out any existing property data that might already exists under the target language
@@ -1005,7 +1005,7 @@ AND umbracoNode.id <> @id",
//based on the current variance of each item to see if it's 'edited' value should be true/false.
var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0);
if (whereInArgsCount > 2000)
if (whereInArgsCount > Constants.Sql.MaxParameterCount)
throw new NotSupportedException("Too many property/content types.");
var propertySql = Sql()
@@ -1095,7 +1095,7 @@ AND umbracoNode.id <> @id",
}
//lookup all matching rows in umbracoDocumentCultureVariation
var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(2000)
var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(Constants.Sql.MaxParameterCount)
.SelectMany(_ => Database.Fetch<DocumentCultureVariationDto>(
Sql().Select<DocumentCultureVariationDto>().From<DocumentCultureVariationDto>()
.WhereIn<DocumentCultureVariationDto>(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList())

3
src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs
View File

@@ -260,8 +260,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
Func<Guid[], IEnumerable<IEnumerable<IDictionaryItem>>> getItemsFromParents = guids =>
{
//needs to be in groups of 2000 because we are doing an IN clause and there's a max parameter count that can be used.
return guids.InGroupsOf(2000)
return guids.InGroupsOf(Constants.Sql.MaxParameterCount)
.Select(@group =>
{
var sqlClause = GetBaseQuery(false)

6
src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs
View File

@@ -1342,7 +1342,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
var result = new Dictionary<int, ContentScheduleCollection>();
var scheduleDtos = Database.FetchByGroups<ContentScheduleDto, int>(contentIds, 2000, batch => Sql()
var scheduleDtos = Database.FetchByGroups<ContentScheduleDto, int>(contentIds, Constants.Sql.MaxParameterCount, batch => Sql()
.Select<ContentScheduleDto>()
.From<ContentScheduleDto>()
.WhereIn<ContentScheduleDto>(x => x.NodeId, batch));
@@ -1391,7 +1391,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
}
if (versions.Count == 0) return new Dictionary<int, List<ContentVariation>>();
var dtos = Database.FetchByGroups<ContentVersionCultureVariationDto, int>(versions, 2000, batch
var dtos = Database.FetchByGroups<ContentVersionCultureVariationDto, int>(versions, Constants.Sql.MaxParameterCount, batch
=> Sql()
.Select<ContentVersionCultureVariationDto>()
.From<ContentVersionCultureVariationDto>()
@@ -1420,7 +1420,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
var ids = temps.Select(x => x.Id);
var dtos = Database.FetchByGroups<DocumentCultureVariationDto, int>(ids, 2000, batch =>
var dtos = Database.FetchByGroups<DocumentCultureVariationDto, int>(ids, Constants.Sql.MaxParameterCount, batch =>
Sql()
.Select<DocumentCultureVariationDto>()
.From<DocumentCultureVariationDto>()

2
src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs
View File

@@ -60,7 +60,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
if (ids.Any())
{
return Database.FetchByGroups<NodeDto, int>(ids, 2000, batch =>
return Database.FetchByGroups<NodeDto, int>(ids, Constants.Sql.MaxParameterCount, batch =>
GetBaseQuery(false)
.Where<NodeDto>(x => x.NodeObjectType == NodeObjectTypeId)
.WhereIn<NodeDto>(x => x.NodeId, batch))

2
src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs
View File

@@ -281,7 +281,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
if (v == null) return entitiesList;
// fetch all variant info dtos
var dtos = Database.FetchByGroups<VariantInfoDto, int>(v.Select(x => x.Id), 2000, GetVariantInfos);
var dtos = Database.FetchByGroups<VariantInfoDto, int>(v.Select(x => x.Id), Constants.Sql.MaxParameterCount, GetVariantInfos);
// group by node id (each group contains all languages)
var xdtos = dtos.GroupBy(x => x.NodeId).ToDictionary(x => x.Key, x => x);

9
src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs
View File

@@ -44,7 +44,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
var result = new EntityPermissionCollection();
foreach (var groupOfGroupIds in groupIds.InGroupsOf(2000))
foreach (var groupOfGroupIds in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
//copy local
var localIds = groupOfGroupIds.ToArray();
@@ -64,7 +64,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
else
{
//iterate in groups of 2000 since we don't want to exceed the max SQL param count
foreach (var groupOfEntityIds in entityIds.InGroupsOf(2000))
foreach (var groupOfEntityIds in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
var ids = groupOfEntityIds;
var sql = Sql()
@@ -133,11 +133,10 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
var db = AmbientScope.Database;
//we need to batch these in groups of 2000 so we don't exceed the max 2100 limit
var sql = "DELETE FROM umbracoUserGroup2NodePermission WHERE userGroupId = @groupId AND nodeId in (@nodeIds)";
foreach (var idGroup in entityIds.InGroupsOf(2000))
foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount))
{
db.Execute(sql, new { groupId, nodeIds = idGroup });
db.Execute(sql, new { groupId, nodeIds = group });
}
var toInsert = new List<UserGroup2NodePermissionDto>();

5
src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs
View File

@@ -37,8 +37,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
protected override IEnumerable<IRedirectUrl> PerformGetAll(params Guid[] ids)
{
if (ids.Length > 2000)
throw new NotSupportedException("This repository does not support more than 2000 ids.");
if (ids.Length > Constants.Sql.MaxParameterCount)
throw new NotSupportedException($"This repository does not support more than {Constants.Sql.MaxParameterCount} ids.");
var sql = GetBaseQuery(false).WhereIn<RedirectUrlDto>(x => x.Id, ids);
var dtos = Database.Fetch<RedirectUrlDto>(sql);
return dtos.WhereNotNull().Select(Map);

5
src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs
View File

@@ -186,14 +186,13 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
// can't query more than 2000 ids at a time... but if someone is really querying 2000+ entities,
// the additional overhead of fetching them in groups is minimal compared to the lookup time of each group
const int maxParams = 2000;
if (ids.Length <= maxParams)
if (ids.Length <= Constants.Sql.MaxParameterCount)
{
return CachePolicy.GetAll(ids, PerformGetAll);
}
var entities = new List<TEntity>();
foreach (var groupOfIds in ids.InGroupsOf(maxParams))
foreach (var groupOfIds in ids.InGroupsOf(Constants.Sql.MaxParameterCount))
{
entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll));
}

2
src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs
View File

@@ -39,7 +39,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement
{
var dtos = ids.Length == 0
? Database.Fetch<TagDto>(Sql().Select<TagDto>().From<TagDto>())
: Database.FetchByGroups<TagDto, int>(ids, 2000, batch => Sql().Select<TagDto>().From<TagDto>().WhereIn<TagDto>(x => x.Id, batch));
: Database.FetchByGroups<TagDto, int>(ids, Constants.Sql.MaxParameterCount, batch => Sql().Select<TagDto>().From<TagDto>().WhereIn<TagDto>(x => x.Id, batch));
return dtos.Select(TagFactory.BuildEntity).ToList();
}

2
src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs
View File

@@ -240,7 +240,7 @@ namespace Umbraco.Web.Models.Mapping
// the entity service due to too many Sql parameters.
var list = new List<IEntitySlim>();
foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000))
foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount))
list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray()));
contentEntities = list.ToArray();
}