diff --git a/tests/Umbraco.Tests.Integration/Testing/ContentServiceBenchmarkBase.cs b/tests/Umbraco.Tests.Integration/Testing/ContentServiceBenchmarkBase.cs index 6b76d9a3a1..92399d253b 100644 --- a/tests/Umbraco.Tests.Integration/Testing/ContentServiceBenchmarkBase.cs +++ b/tests/Umbraco.Tests.Integration/Testing/ContentServiceBenchmarkBase.cs @@ -31,6 +31,30 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo { private readonly List _results = new(); + // Regression enforcement configuration + private const double DefaultRegressionThreshold = 20.0; + + // Allow CI override via environment variable + private static readonly double RegressionThreshold = + double.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REGRESSION_THRESHOLD"), out var t) + ? t + : DefaultRegressionThreshold; + + // Optional strict mode: fail if baseline is missing (useful for CI) + private static readonly bool RequireBaseline = + bool.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REQUIRE_BASELINE"), out var b) && b; + + // Thread-safe lazy initialization of repository root + private static readonly Lazy _repositoryRoot = new(FindRepositoryRoot); + + // Thread-safe lazy initialization of baseline data + private static readonly Lazy> _baselineLoader = + new(() => LoadBaselineInternal(), LazyThreadSafetyMode.ExecutionAndPublication); + + private static Dictionary Baseline => _baselineLoader.Value; + + private static string BaselinePath => Path.Combine(_repositoryRoot.Value, "docs", "plans", "baseline-phase0.json"); + /// /// Records a benchmark result for later output. /// @@ -118,6 +142,108 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo _results.Clear(); } + /// + /// Finds the repository root by searching for umbraco.sln. + /// + private static string FindRepositoryRoot() + { + var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + while (dir != null) + { + if (File.Exists(Path.Combine(dir.FullName, "umbraco.sln"))) + { + return dir.FullName; + } + dir = dir.Parent; + } + throw new InvalidOperationException( + $"Cannot find repository root (umbraco.sln) starting from {TestContext.CurrentContext.TestDirectory}"); + } + + /// + /// Records a benchmark and asserts no regression beyond the threshold. + /// + /// Benchmark name (must match baseline JSON key). + /// Measured elapsed time in milliseconds. + /// Number of items processed. + /// Maximum allowed regression percentage (default: 20%, configurable via BENCHMARK_REGRESSION_THRESHOLD env var). + protected void AssertNoRegression(string name, long elapsedMs, int itemCount, double thresholdPercent = -1) + { + RecordBenchmark(name, elapsedMs, itemCount); + + // Use environment-configurable threshold if not explicitly specified + var effectiveThreshold = thresholdPercent < 0 ? RegressionThreshold : thresholdPercent; + + if (Baseline.TryGetValue(name, out var baselineResult)) + { + var maxAllowed = baselineResult.ElapsedMs * (1 + effectiveThreshold / 100); + + if (elapsedMs > maxAllowed) + { + var regressionPct = ((double)(elapsedMs - baselineResult.ElapsedMs) / baselineResult.ElapsedMs) * 100; + Assert.Fail( + $"Performance regression detected for '{name}': " + + $"{elapsedMs}ms exceeds threshold of {maxAllowed:F0}ms " + + $"(baseline: {baselineResult.ElapsedMs}ms, regression: +{regressionPct:F1}%, threshold: {effectiveThreshold}%)"); + } + + TestContext.WriteLine($"[REGRESSION_CHECK] {name}: PASS ({elapsedMs}ms <= {maxAllowed:F0}ms, baseline: {baselineResult.ElapsedMs}ms, threshold: {effectiveThreshold}%)"); + } + else if (RequireBaseline) + { + Assert.Fail($"No baseline entry found for '{name}' and BENCHMARK_REQUIRE_BASELINE=true"); + } + else + { + TestContext.WriteLine($"[REGRESSION_CHECK] {name}: SKIPPED (no baseline entry)"); + } + } + + /// + /// Measures, records, and asserts no regression for the given action. + /// + protected long MeasureAndAssertNoRegression(string name, int itemCount, Action action, bool skipWarmup = false, double thresholdPercent = -1) + { + // Warmup iteration (skip for destructive operations) + if (!skipWarmup) + { + try { action(); } + catch (Exception ex) + { + TestContext.WriteLine($"[WARMUP] {name} warmup failed: {ex.Message}"); + } + } + + var sw = Stopwatch.StartNew(); + action(); + sw.Stop(); + + AssertNoRegression(name, sw.ElapsedMilliseconds, itemCount, thresholdPercent); + return sw.ElapsedMilliseconds; + } + + private static Dictionary LoadBaselineInternal() + { + if (!File.Exists(BaselinePath)) + { + TestContext.WriteLine($"[BASELINE] File not found: {BaselinePath}"); + return new Dictionary(); + } + + try + { + var json = File.ReadAllText(BaselinePath); + var results = JsonSerializer.Deserialize>(json) ?? new List(); + TestContext.WriteLine($"[BASELINE] Loaded {results.Count} baseline entries from {BaselinePath}"); + return results.ToDictionary(r => r.Name, r => r); + } + catch (Exception ex) + { + TestContext.WriteLine($"[BASELINE] Failed to load baseline: {ex.Message}"); + return new Dictionary(); + } + } + /// /// Represents a single benchmark measurement. /// diff --git a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Services/ContentServiceRefactoringBenchmarks.cs b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Services/ContentServiceRefactoringBenchmarks.cs index ff06bb45f0..14b8c1cb20 100644 --- a/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Services/ContentServiceRefactoringBenchmarks.cs +++ b/tests/Umbraco.Tests.Integration/Umbraco.Infrastructure/Services/ContentServiceRefactoringBenchmarks.cs @@ -65,7 +65,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm var sw = Stopwatch.StartNew(); ContentService.Save(content); sw.Stop(); - RecordBenchmark("Save_SingleItem", sw.ElapsedMilliseconds, 1); + + // Gate: Fail if >20% regression from baseline + AssertNoRegression("Save_SingleItem", sw.ElapsedMilliseconds, 1); Assert.That(content.Id, Is.GreaterThan(0)); } @@ -97,7 +99,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm var sw = Stopwatch.StartNew(); ContentService.Save(items); sw.Stop(); - RecordBenchmark("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount); + + // Gate: Fail if >20% regression from baseline + AssertNoRegression("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount); Assert.That(items.All(c => c.Id > 0), Is.True); } @@ -129,7 +133,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm var sw = Stopwatch.StartNew(); ContentService.Save(items); sw.Stop(); - RecordBenchmark("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount); + + // Gate: Fail if >20% regression from baseline + AssertNoRegression("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount); Assert.That(items.All(c => c.Id > 0), Is.True); } @@ -147,7 +153,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm var id = content.Id; IContent? result = null; - MeasureAndRecord("GetById_Single", 1, () => + MeasureAndAssertNoRegression("GetById_Single", 1, () => { result = ContentService.GetById(id); }); @@ -176,7 +182,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm var ids = items.Select(c => c.Id).ToList(); IEnumerable? results = null; - MeasureAndRecord("GetByIds_BatchOf100", itemCount, () => + MeasureAndAssertNoRegression("GetByIds_BatchOf100", itemCount, () => { results = ContentService.GetByIds(ids); }); @@ -194,7 +200,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm var content = ContentBuilder.CreateSimpleContent(ContentType, "DeleteTest", -1); ContentService.Save(content); - MeasureAndRecord("Delete_SingleItem", 1, () => + MeasureAndAssertNoRegression("Delete_SingleItem", 1, () => { ContentService.Delete(content); }, skipWarmup: true); // Destructive operation - cannot repeat @@ -220,7 +226,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm ContentService.Save(child); } - MeasureAndRecord("Delete_WithDescendants", childCount + 1, () => + MeasureAndAssertNoRegression("Delete_WithDescendants", childCount + 1, () => { ContentService.Delete(parent); }, skipWarmup: true); // Destructive operation - cannot repeat @@ -253,7 +259,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm IEnumerable? results = null; long totalRecords = 0; - MeasureAndRecord("GetPagedChildren_100Items", childCount, () => + MeasureAndAssertNoRegression("GetPagedChildren_100Items", childCount, () => { results = ContentService.GetPagedChildren(parent.Id, 0, 100, out totalRecords); }); @@ -293,7 +299,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm IEnumerable? results = null; long totalRecords = 0; - MeasureAndRecord("GetPagedDescendants_DeepTree", 300, () => + MeasureAndAssertNoRegression("GetPagedDescendants_DeepTree", 300, () => { results = ContentService.GetPagedDescendants(root.Id, 0, 1000, out totalRecords); }); @@ -324,7 +330,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm var deepestId = current.Id; IEnumerable? results = null; - MeasureAndRecord("GetAncestors_DeepHierarchy", depth, () => + MeasureAndAssertNoRegression("GetAncestors_DeepHierarchy", depth, () => { results = ContentService.GetAncestors(deepestId); });