From c5ba23a15c360eada010553c39809d8dbf26ba1e Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Wed, 13 Oct 2021 13:32:30 +0200 Subject: [PATCH 1/5] Add Constants.Sql.MaxParameterCount --- src/Umbraco.Core/Constants-Sql.cs | 17 +++++++++++++++++ src/Umbraco.Core/Umbraco.Core.csproj | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 src/Umbraco.Core/Constants-Sql.cs diff --git a/src/Umbraco.Core/Constants-Sql.cs b/src/Umbraco.Core/Constants-Sql.cs new file mode 100644 index 0000000000..366dfbe44a --- /dev/null +++ b/src/Umbraco.Core/Constants-Sql.cs @@ -0,0 +1,17 @@ +namespace Umbraco.Core +{ + public static partial class Constants + { + public static class Sql + { + /// + /// The maximum amount of parameters that can be used in a query. + /// + /// + /// The actual limit is 2100 (https://docs.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server), + /// but we want to ensure there's room for additional parameters if this value is used to create groups/batches. + /// + public const int MaxParameterCount = 2000; + } + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 77eeaaa853..e2679c2223 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -132,6 +132,7 @@ + @@ -1665,4 +1666,4 @@ - + \ No newline at end of file From a6b04a941ced30381230c3d37e0c667960a2bfda Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Wed, 13 Oct 2021 13:42:18 +0200 Subject: [PATCH 2/5] Replace arbitrary group size with constant --- .../Persistence/NPocoDatabaseExtensions-Bulk.cs | 2 +- .../Repositories/Implement/AuditEntryRepository.cs | 2 +- .../Repositories/Implement/ContentRepositoryBase.cs | 4 ++-- .../Repositories/Implement/ContentTypeRepositoryBase.cs | 8 ++++---- .../Repositories/Implement/DictionaryRepository.cs | 3 +-- .../Repositories/Implement/DocumentRepository.cs | 6 +++--- .../Repositories/Implement/EntityContainerRepository.cs | 2 +- .../Repositories/Implement/EntityRepository.cs | 2 +- .../Repositories/Implement/PermissionRepository.cs | 9 ++++----- .../Repositories/Implement/RedirectUrlRepository.cs | 5 +++-- .../Repositories/Implement/RepositoryBaseOfTIdTEntity.cs | 5 ++--- .../Persistence/Repositories/Implement/TagRepository.cs | 2 +- src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs | 2 +- 13 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs index 77cc0d6601..a50e3c2aaa 100644 --- a/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs +++ b/src/Umbraco.Core/Persistence/NPocoDatabaseExtensions-Bulk.cs @@ -137,7 +137,7 @@ namespace Umbraco.Core.Persistence // Math.Floor(2100 / 8) = 262 record per command // 4168 / 262 = 15.908... = there will be 16 command in total // (if we have disabled db parameters, then all records will be included, in only one command) - var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor(2000.00 / paramsPerRecord)); + var recordsPerCommand = paramsPerRecord == 0 ? int.MaxValue : Convert.ToInt32(Math.Floor((double)Constants.Sql.MaxParameterCount / paramsPerRecord)); var commandsCount = Convert.ToInt32(Math.Ceiling((double)records.Length / recordsPerCommand)); var commands = new IDbCommand[commandsCount]; diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs index c3d34cc3e9..d921bb6d51 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/AuditEntryRepository.cs @@ -54,7 +54,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement var entries = new List(); - foreach (var group in ids.InGroupsOf(2000)) + foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) { var sql = Sql() .Select() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs index 85814ef681..a6b08c2bfd 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentRepositoryBase.cs @@ -638,7 +638,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // in the table? // get all PropertyDataDto for all definitions / versions - var allPropertyDataDtos = Database.FetchByGroups(versions, 2000, batch => + var allPropertyDataDtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch => SqlContext.Sql() .Select() .From() @@ -647,7 +647,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // get PropertyDataDto distinct PropertyTypeDto var allPropertyTypeIds = allPropertyDataDtos.Select(x => x.PropertyTypeId).Distinct().ToList(); - var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, 2000, batch => + var allPropertyTypeDtos = Database.FetchByGroups(allPropertyTypeIds, Constants.Sql.MaxParameterCount, batch => SqlContext.Sql() .Select(r => r.Select(x => x.DataTypeDto)) .From() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs index aff53e44b0..8835f8c647 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs @@ -768,7 +768,7 @@ AND umbracoNode.id <> @id", // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); // delete existing relations (for target language) @@ -906,7 +906,7 @@ AND umbracoNode.id <> @id", // note: important to use SqlNullableEquals for nullable types, cannot directly compare language identifiers // var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); //first clear out any existing property data that might already exists under the target language @@ -1005,7 +1005,7 @@ AND umbracoNode.id <> @id", //based on the current variance of each item to see if it's 'edited' value should be true/false. var whereInArgsCount = propertyTypeIds.Count + (contentTypeIds?.Count ?? 0); - if (whereInArgsCount > 2000) + if (whereInArgsCount > Constants.Sql.MaxParameterCount) throw new NotSupportedException("Too many property/content types."); var propertySql = Sql() @@ -1095,7 +1095,7 @@ AND umbracoNode.id <> @id", } //lookup all matching rows in umbracoDocumentCultureVariation - var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(2000) + var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(Constants.Sql.MaxParameterCount) .SelectMany(_ => Database.Fetch( Sql().Select().From() .WhereIn(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList()) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs index ac1f7c3f2a..7895b7aa8f 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs @@ -260,8 +260,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement Func>> getItemsFromParents = guids => { - //needs to be in groups of 2000 because we are doing an IN clause and there's a max parameter count that can be used. - return guids.InGroupsOf(2000) + return guids.InGroupsOf(Constants.Sql.MaxParameterCount) .Select(@group => { var sqlClause = GetBaseQuery(false) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs index 5716fbe129..753630d186 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DocumentRepository.cs @@ -1342,7 +1342,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var result = new Dictionary(); - var scheduleDtos = Database.FetchByGroups(contentIds, 2000, batch => Sql() + var scheduleDtos = Database.FetchByGroups(contentIds, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() .WhereIn(x => x.NodeId, batch)); @@ -1391,7 +1391,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement } if (versions.Count == 0) return new Dictionary>(); - var dtos = Database.FetchByGroups(versions, 2000, batch + var dtos = Database.FetchByGroups(versions, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() @@ -1420,7 +1420,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var ids = temps.Select(x => x.Id); - var dtos = Database.FetchByGroups(ids, 2000, batch => + var dtos = Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => Sql() .Select() .From() diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs index 505cbfc816..07b82190f5 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityContainerRepository.cs @@ -60,7 +60,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { if (ids.Any()) { - return Database.FetchByGroups(ids, 2000, batch => + return Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => GetBaseQuery(false) .Where(x => x.NodeObjectType == NodeObjectTypeId) .WhereIn(x => x.NodeId, batch)) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs index a7502a338a..7bcc5f6b2d 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/EntityRepository.cs @@ -281,7 +281,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement if (v == null) return entitiesList; // fetch all variant info dtos - var dtos = Database.FetchByGroups(v.Select(x => x.Id), 2000, GetVariantInfos); + var dtos = Database.FetchByGroups(v.Select(x => x.Id), Constants.Sql.MaxParameterCount, GetVariantInfos); // group by node id (each group contains all languages) var xdtos = dtos.GroupBy(x => x.NodeId).ToDictionary(x => x.Key, x => x); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs index 259f0b89c0..ce324df685 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs @@ -44,7 +44,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var result = new EntityPermissionCollection(); - foreach (var groupOfGroupIds in groupIds.InGroupsOf(2000)) + foreach (var groupOfGroupIds in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { //copy local var localIds = groupOfGroupIds.ToArray(); @@ -64,7 +64,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement else { //iterate in groups of 2000 since we don't want to exceed the max SQL param count - foreach (var groupOfEntityIds in entityIds.InGroupsOf(2000)) + foreach (var groupOfEntityIds in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { var ids = groupOfEntityIds; var sql = Sql() @@ -133,11 +133,10 @@ namespace Umbraco.Core.Persistence.Repositories.Implement var db = AmbientScope.Database; - //we need to batch these in groups of 2000 so we don't exceed the max 2100 limit var sql = "DELETE FROM umbracoUserGroup2NodePermission WHERE userGroupId = @groupId AND nodeId in (@nodeIds)"; - foreach (var idGroup in entityIds.InGroupsOf(2000)) + foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { - db.Execute(sql, new { groupId, nodeIds = idGroup }); + db.Execute(sql, new { groupId, nodeIds = group }); } var toInsert = new List(); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs index 24c1e31c20..099d49fbf8 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RedirectUrlRepository.cs @@ -37,8 +37,9 @@ namespace Umbraco.Core.Persistence.Repositories.Implement protected override IEnumerable PerformGetAll(params Guid[] ids) { - if (ids.Length > 2000) - throw new NotSupportedException("This repository does not support more than 2000 ids."); + if (ids.Length > Constants.Sql.MaxParameterCount) + throw new NotSupportedException($"This repository does not support more than {Constants.Sql.MaxParameterCount} ids."); + var sql = GetBaseQuery(false).WhereIn(x => x.Id, ids); var dtos = Database.Fetch(sql); return dtos.WhereNotNull().Select(Map); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs index 69e4db5940..c2aec05fa7 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs @@ -186,14 +186,13 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // can't query more than 2000 ids at a time... but if someone is really querying 2000+ entities, // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group - const int maxParams = 2000; - if (ids.Length <= maxParams) + if (ids.Length <= Constants.Sql.MaxParameterCount) { return CachePolicy.GetAll(ids, PerformGetAll); } var entities = new List(); - foreach (var groupOfIds in ids.InGroupsOf(maxParams)) + foreach (var groupOfIds in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) { entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll)); } diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs index 279e03b19e..83170d9dbc 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/TagRepository.cs @@ -39,7 +39,7 @@ namespace Umbraco.Core.Persistence.Repositories.Implement { var dtos = ids.Length == 0 ? Database.Fetch(Sql().Select().From()) - : Database.FetchByGroups(ids, 2000, batch => Sql().Select().From().WhereIn(x => x.Id, batch)); + : Database.FetchByGroups(ids, Constants.Sql.MaxParameterCount, batch => Sql().Select().From().WhereIn(x => x.Id, batch)); return dtos.Select(TagFactory.BuildEntity).ToList(); } diff --git a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs index 6198630898..f66a1ee934 100644 --- a/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs +++ b/src/Umbraco.Web/Models/Mapping/UserMapDefinition.cs @@ -240,7 +240,7 @@ namespace Umbraco.Web.Models.Mapping // the entity service due to too many Sql parameters. var list = new List(); - foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(2000)) + foreach (var idGroup in allContentPermissions.Keys.InGroupsOf(Constants.Sql.MaxParameterCount)) list.AddRange(_entityService.GetAll(UmbracoObjectTypes.Document, idGroup.ToArray())); contentEntities = list.ToArray(); } From 88a20cb13cbde845829cbe00efa581f9737713e4 Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Wed, 13 Oct 2021 13:46:51 +0200 Subject: [PATCH 3/5] Minor query cleanup --- .../Repositories/Implement/DictionaryRepository.cs | 4 ++-- .../Repositories/Implement/MemberRepository.cs | 11 ++++------- .../Implement/RepositoryBaseOfTIdTEntity.cs | 6 ++++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs index 7895b7aa8f..da957a7288 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/DictionaryRepository.cs @@ -261,11 +261,11 @@ namespace Umbraco.Core.Persistence.Repositories.Implement Func>> getItemsFromParents = guids => { return guids.InGroupsOf(Constants.Sql.MaxParameterCount) - .Select(@group => + .Select(group => { var sqlClause = GetBaseQuery(false) .Where(x => x.Parent != null) - .Where($"{SqlSyntax.GetQuotedColumnName("parent")} IN (@parentIds)", new { parentIds = @group }); + .WhereIn(x => x.Parent, group); var translator = new SqlTranslator(sqlClause, Query()); var sql = translator.Translate(); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs index 62f41aa727..1982d6ebf9 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/MemberRepository.cs @@ -431,16 +431,13 @@ namespace Umbraco.Core.Persistence.Repositories.Implement var matchedMembers = Get(query).ToArray(); var membersInGroup = new List(); - //then we need to filter the matched members that are in the role - //since the max sql params are 2100 on sql server, we'll reduce that to be safe for potentially other servers and run the queries in batches - var inGroups = matchedMembers.InGroupsOf(1000); - foreach (var batch in inGroups) - { - var memberIdBatch = batch.Select(x => x.Id); + //then we need to filter the matched members that are in the role + foreach (var group in matchedMembers.Select(x => x.Id).InGroupsOf(Constants.Sql.MaxParameterCount)) + { var sql = Sql().SelectAll().From() .Where(dto => dto.MemberGroup == memberGroup.Id) - .WhereIn(dto => dto.Member, memberIdBatch); + .WhereIn(dto => dto.Member, group); var memberIdsInGroup = Database.Fetch(sql) .Select(x => x.Member).ToArray(); diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs index c2aec05fa7..5892f2bf49 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs @@ -188,14 +188,16 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group if (ids.Length <= Constants.Sql.MaxParameterCount) { + // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group return CachePolicy.GetAll(ids, PerformGetAll); } var entities = new List(); - foreach (var groupOfIds in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) + foreach (var group in ids.InGroupsOf(Constants.Sql.MaxParameterCount)) { - entities.AddRange(CachePolicy.GetAll(groupOfIds.ToArray(), PerformGetAll)); + entities.AddRange(CachePolicy.GetAll(group.ToArray(), PerformGetAll)); } + return entities; } From 1bc3290497626ebe720806c4af1b62820d12a665 Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Wed, 13 Oct 2021 14:10:24 +0200 Subject: [PATCH 4/5] Fix bugs in fetching more than 2000 items (permissions and culture variations) --- .../Implement/ContentTypeRepositoryBase.cs | 22 +++++++---- .../Implement/PermissionRepository.cs | 37 +++++++++---------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs index 8835f8c647..3a74b9209b 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/ContentTypeRepositoryBase.cs @@ -1094,14 +1094,20 @@ AND umbracoNode.id <> @id", } } - //lookup all matching rows in umbracoDocumentCultureVariation - var docCultureVariationsToUpdate = editedLanguageVersions.InGroupsOf(Constants.Sql.MaxParameterCount) - .SelectMany(_ => Database.Fetch( - Sql().Select().From() - .WhereIn(x => x.LanguageId, editedLanguageVersions.Keys.Select(x => x.langId).ToList()) - .WhereIn(x => x.NodeId, editedLanguageVersions.Keys.Select(x => x.nodeId)))) - //convert to dictionary with the same key type - .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); + // lookup all matching rows in umbracoDocumentCultureVariation + // fetch in batches to account for maximum parameter count (distinct languages can't exceed 2000) + var languageIds = editedLanguageVersions.Keys.Select(x => x.langId).Distinct().ToArray(); + var nodeIds = editedLanguageVersions.Keys.Select(x => x.nodeId).Distinct(); + var docCultureVariationsToUpdate = nodeIds.InGroupsOf(Constants.Sql.MaxParameterCount - languageIds.Length) + .SelectMany(group => + { + var sql = Sql().Select().From() + .WhereIn(x => x.LanguageId, languageIds) + .WhereIn(x => x.NodeId, group); + + return Database.Fetch(sql); + }) + .ToDictionary(x => (x.NodeId, (int?)x.LanguageId), x => x); //convert to dictionary with the same key type var toUpdate = new List(); foreach (var ev in editedLanguageVersions) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs index ce324df685..735496a0d1 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/PermissionRepository.cs @@ -38,44 +38,41 @@ namespace Umbraco.Core.Persistence.Repositories.Implement /// /// /// - /// This method will not support passing in more than 2000 group Ids + /// This method will not support passing in more than 2000 group IDs when also passing in entity IDs. /// public EntityPermissionCollection GetPermissionsForEntities(int[] groupIds, params int[] entityIds) { var result = new EntityPermissionCollection(); - foreach (var groupOfGroupIds in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount)) + if (entityIds.Length == 0) { - //copy local - var localIds = groupOfGroupIds.ToArray(); - - if (entityIds.Length == 0) + foreach (var group in groupIds.InGroupsOf(Constants.Sql.MaxParameterCount)) { var sql = Sql() .SelectAll() .From() - .Where(dto => localIds.Contains(dto.UserGroupId)); + .Where(dto => group.Contains(dto.UserGroupId)); + var permissions = AmbientScope.Database.Fetch(sql); foreach (var permission in ConvertToPermissionList(permissions)) { result.Add(permission); } } - else + } + else + { + foreach (var group in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount - groupIds.Length)) { - //iterate in groups of 2000 since we don't want to exceed the max SQL param count - foreach (var groupOfEntityIds in entityIds.InGroupsOf(Constants.Sql.MaxParameterCount)) + var sql = Sql() + .SelectAll() + .From() + .Where(dto => groupIds.Contains(dto.UserGroupId) && group.Contains(dto.NodeId)); + + var permissions = AmbientScope.Database.Fetch(sql); + foreach (var permission in ConvertToPermissionList(permissions)) { - var ids = groupOfEntityIds; - var sql = Sql() - .SelectAll() - .From() - .Where(dto => localIds.Contains(dto.UserGroupId) && ids.Contains(dto.NodeId)); - var permissions = AmbientScope.Database.Fetch(sql); - foreach (var permission in ConvertToPermissionList(permissions)) - { - result.Add(permission); - } + result.Add(permission); } } } From 60be71d1a0037ffb5556511342065f85afce2825 Mon Sep 17 00:00:00 2001 From: Ronald Barendse Date: Wed, 13 Oct 2021 23:34:50 +0200 Subject: [PATCH 5/5] Remove duplicated comment --- .../Repositories/Implement/RepositoryBaseOfTIdTEntity.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs index 5892f2bf49..a7704272a8 100644 --- a/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs +++ b/src/Umbraco.Core/Persistence/Repositories/Implement/RepositoryBaseOfTIdTEntity.cs @@ -188,7 +188,6 @@ namespace Umbraco.Core.Persistence.Repositories.Implement // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group if (ids.Length <= Constants.Sql.MaxParameterCount) { - // the additional overhead of fetching them in groups is minimal compared to the lookup time of each group return CachePolicy.GetAll(ids, PerformGetAll); }