From a6b04a941ced30381230c3d37e0c667960a2bfda Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Wed, 13 Oct 2021 13:42:18 +0200 Subject: [PATCH] Replace arbitrary group size with constant --- .../Persistence/NPocoDatabaseExtensions-Bulk.cs | 2 +- .../Repositories/Implement/AuditEntryRepository.cs | 2 +- .../Repositories/Implement/ContentRepositoryBase.cs | 4 ++-- .../Repositories/Implement/ContentTypeRepositoryBase.cs | 8 ++++---- .../Repositories/Implement/DictionaryRepository.cs | 3 +-- .../Repositories/Implement/DocumentRepository.cs | 6 +++--- .../Repositories/Implement/EntityContainerRepository.cs | 2 +- .../Repositories/Implement/EntityRepository.cs | 2 +- .../Repositories/Implement/PermissionRepository.cs | 9 ++++----- .../Repositories/Implement/RedirectUrlRepository.cs | 5 +++-- .../Repositories/Implement/RepositoryBaseOfTIdTEntity.cs | 5 ++--- .../Persistence/Repositories/Implement/TagRepository.cs | 2 +- src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs | 2 +- 13 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs index 77cc0d6601..a50e3c2aaa 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs @@ -137,7 +137,7 @@ namespace Umbraco.Core.Persistence // Math.Floor(2100 / 8) = 262 record per command // 4168 / 262 = 15.908... = there will be 16 command in total // (if we have disabled db parameters, then all records will be included, in only one command) - var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor(2000.00 / paramsPerRecord)); + var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor((double)Constants.Sql.MaxParameterCount / paramsPerRecord)); var commandsCount = Convert.ToInt32(Math.Ceiling((double)records.Length / recordsPerCommand)); var commands = new IDbCommand[commandsCount]; diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs index c3d34cc3e9..d921bb6d51 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs @@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement var entries = new List(); - foreach (var group in ids.InGroupsOf(2000)) + foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) { var sql = Sql() .Select() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs index 85814ef681..a6b08c2bfd 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs @@ -638,7 +638,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // in the table? // get all PropertyDataDto for all definitions / versions - var allPropertyDataDtos = Database.FetchByGroups(versions, 2000, batch => + var allPropertyDataDtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch => SqlContext.Sql() .Select() .From() @@ -647,7 +647,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // get PropertyDataDto distinct PropertyTypeDto var allPropertyTypeIds = allPropertyDataDtos.Select(x => x.PropertyTypeId).Distinct().ToList(); - var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, 2000, batch => + var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, Constants.Sql.MaxParameterCount, batch => SqlContext.Sql() .Select(r => r.Select(x => x.DataTypeDto)) .From() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs index aff53e44b0..8835f8c647 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs @@ -768,7 +768,7 @@ AND umbracoNode.id <> @id", // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); // delete existing relations (for target language) @@ -906,7 +906,7 @@ AND umbracoNode.id <> @id", // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers // var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); //first clear out any existing property data that might already exists under the target language @@ -1005,7 +1005,7 @@ AND umbracoNode.id <> @id", //based on the current variance of each item to see if it's 'edited' value should be true/false. var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); var propertySql = Sql() @@ -1095,7 +1095,7 @@ AND umbracoNode.id <> @id", } //lookup all matching rows in umbracoDocumentCultureVariation - var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(2000) + var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(Constants.Sql.MaxParameterCount) .SelectMany(_ => Database.Fetch( Sql().Select().From() .WhereIn(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList()) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs index ac1f7c3f2a..7895b7aa8f 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs @@ -260,8 +260,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement Func>> getItemsFromParents = guids => { - //needs to be in groups of 2000 because we are doing an IN clause and there's a max parameter count that can be used. - return guids.InGroupsOf(2000) + return guids.InGroupsOf(Constants.Sql.MaxParameterCount) .Select(@group => { var sqlClause = GetBaseQuery(false) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs index 5716fbe129..753630d186 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs @@ -1342,7 +1342,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var result = new Dictionary(); - var scheduleDtos = Database.FetchByGroups(contentIds, 2000, batch => Sql() + var scheduleDtos = Database.FetchByGroups(contentIds, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() .WhereIn(x => x.NodeId, batch)); @@ -1391,7 +1391,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement } if (versions.Count == 0) return new Dictionary>(); - var dtos = Database.FetchByGroups(versions, 2000, batch + var dtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() @@ -1420,7 +1420,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var ids = temps.Select(x => x.Id); - var dtos = Database.FetchByGroups(ids, 2000, batch => + var dtos = Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs index 505cbfc816..07b82190f5 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs @@ -60,7 +60,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { if (ids.Any()) { - return Database.FetchByGroups(ids, 2000, batch => + return Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => GetBaseQuery(false) .Where(x => x.NodeObjectType == NodeObjectTypeId) .WhereIn(x => x.NodeId, batch)) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs index a7502a338a..7bcc5f6b2d 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs @@ -281,7 +281,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement if (v == null) return entitiesList; // fetch all variant info dtos - var dtos = Database.FetchByGroups(v.Select(x => x.Id), 2000, GetVariantInfos); + var dtos = Database.FetchByGroups(v.Select(x => x.Id), Constants.Sql.MaxParameterCount, GetVariantInfos); // group by node id (each group contains all languages) var xdtos = dtos.GroupBy(x => x.NodeId).ToDictionary(x => x.Key, x => x); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs index 259f0b89c0..ce324df685 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs @@ -44,7 +44,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var result = new EntityPermissionCollection(); - foreach (var groupOfGroupIds in groupIds.InGroupsOf(2000)) + foreach (var groupOfGroupIds in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { //copy local var localIds = groupOfGroupIds.ToArray(); @@ -64,7 +64,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement else { //iterate in groups of 2000 since we don't want to exceed the max SQL param count - foreach (var groupOfEntityIds in entityIds.InGroupsOf(2000)) + foreach (var groupOfEntityIds in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { var ids = groupOfEntityIds; var sql = Sql() @@ -133,11 +133,10 @@ namespace Umbraco.Core.Persistence.Repositories.Implement var db = AmbientScope.Database; - //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit var sql = "DELETE FROM umbracoUserGroup2NodePermission WHERE userGroupId = @groupId AND nodeId in (@nodeIds)"; - foreach (var idGroup in entityIds.InGroupsOf(2000)) + foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { - db.Execute(sql, new { groupId, nodeIds = idGroup }); + db.Execute(sql, new { groupId, nodeIds = group }); } var toInsert = new List(); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs index 24c1e31c20..099d49fbf8 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs @@ -37,8 +37,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement protected override IEnumerable PerformGetAll(params Guid[] ids) { - if (ids.Length > 2000) - throw new NotSupportedException("This repository does not support more than 2000 ids."); + if (ids.Length > Constants.Sql.MaxParameterCount) + throw new NotSupportedException($"This repository does not support more than {Constants.Sql.MaxParameterCount} ids."); + var sql = GetBaseQuery(false).WhereIn(x => x.Id, ids); var dtos = Database.Fetch(sql); return dtos.WhereNotNull().Select(Map); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs index 69e4db5940..c2aec05fa7 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs @@ -186,14 +186,13 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // can't query more than 2000 ids at a time... but if someone is really querying 2000+ entities, // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group - const int maxParams = 2000; - if (ids.Length <= maxParams) + if (ids.Length <= Constants.Sql.MaxParameterCount) { return CachePolicy.GetAll(ids, PerformGetAll); } var entities = new List(); - foreach (var groupOfIds in ids.InGroupsOf(maxParams)) + foreach (var groupOfIds in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) { entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll)); } diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs index 279e03b19e..83170d9dbc 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs @@ -39,7 +39,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var dtos = ids.Length == 0 ? Database.Fetch(Sql().Select().From()) - : Database.FetchByGroups(ids, 2000, batch => Sql().Select().From().WhereIn(x => x.Id, batch)); + : Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => Sql().Select().From().WhereIn(x => x.Id, batch)); return dtos.Select(TagFactory.BuildEntity).ToList(); } diff --git a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs index 6198630898..f66a1ee934 100644 --- a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs @@ -240,7 +240,7 @@ namespace Umbraco.Web.Models.Mapping // the entity service due to too many Sql parameters. var list = new List(); - foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000)) + foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount)) list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray())); contentEntities = list.ToArray(); }