test: add benchmark regression enforcement for Phase 1

Adds AssertNoRegression method to ContentServiceBenchmarkBase:
- Loads baseline from baseline-phase0.json
- Fails test if benchmark exceeds 20% of baseline
- Logs regression check status for visibility
- Supports environment variable overrides (BENCHMARK_REGRESSION_THRESHOLD, BENCHMARK_REQUIRE_BASELINE)

Updates 10 Phase 1 CRUD benchmarks to use regression assertions:
- Save_SingleItem, Save_BatchOf100, Save_BatchOf1000
- GetById_Single, GetByIds_BatchOf100
- Delete_SingleItem, Delete_WithDescendants
- GetAncestors_DeepHierarchy, GetPagedChildren_100Items, GetPagedDescendants_DeepTree

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-12-21 03:43:28 +00:00
parent 9962df50ee
commit 1d082e1ed6
2 changed files with 142 additions and 10 deletions

View File

@@ -31,6 +31,30 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
{
private readonly List<BenchmarkResult> _results = new();
// Regression enforcement configuration
private const double DefaultRegressionThreshold = 20.0;
// Allow CI override via environment variable
private static readonly double RegressionThreshold =
double.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REGRESSION_THRESHOLD"), out var t)
? t
: DefaultRegressionThreshold;
// Optional strict mode: fail if baseline is missing (useful for CI)
private static readonly bool RequireBaseline =
bool.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REQUIRE_BASELINE"), out var b) && b;
// Thread-safe lazy initialization of repository root
private static readonly Lazy<string> _repositoryRoot = new(FindRepositoryRoot);
// Thread-safe lazy initialization of baseline data
private static readonly Lazy<Dictionary<string, BenchmarkResult>> _baselineLoader =
new(() => LoadBaselineInternal(), LazyThreadSafetyMode.ExecutionAndPublication);
private static Dictionary<string, BenchmarkResult> Baseline => _baselineLoader.Value;
private static string BaselinePath => Path.Combine(_repositoryRoot.Value, "docs", "plans", "baseline-phase0.json");
/// <summary>
/// Records a benchmark result for later output.
/// </summary>
@@ -118,6 +142,108 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
_results.Clear();
}
/// <summary>
/// Finds the repository root by searching for umbraco.sln.
/// </summary>
private static string FindRepositoryRoot()
{
var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
while (dir != null)
{
if (File.Exists(Path.Combine(dir.FullName, "umbraco.sln")))
{
return dir.FullName;
}
dir = dir.Parent;
}
throw new InvalidOperationException(
$"Cannot find repository root (umbraco.sln) starting from {TestContext.CurrentContext.TestDirectory}");
}
/// <summary>
/// Records a benchmark and asserts no regression beyond the threshold.
/// </summary>
/// <param name="name">Benchmark name (must match baseline JSON key).</param>
/// <param name="elapsedMs">Measured elapsed time in milliseconds.</param>
/// <param name="itemCount">Number of items processed.</param>
/// <param name="thresholdPercent">Maximum allowed regression percentage (default: 20%, configurable via BENCHMARK_REGRESSION_THRESHOLD env var).</param>
protected void AssertNoRegression(string name, long elapsedMs, int itemCount, double thresholdPercent = -1)
{
RecordBenchmark(name, elapsedMs, itemCount);
// Use environment-configurable threshold if not explicitly specified
var effectiveThreshold = thresholdPercent < 0 ? RegressionThreshold : thresholdPercent;
if (Baseline.TryGetValue(name, out var baselineResult))
{
var maxAllowed = baselineResult.ElapsedMs * (1 + effectiveThreshold / 100);
if (elapsedMs > maxAllowed)
{
var regressionPct = ((double)(elapsedMs - baselineResult.ElapsedMs) / baselineResult.ElapsedMs) * 100;
Assert.Fail(
$"Performance regression detected for '{name}': " +
$"{elapsedMs}ms exceeds threshold of {maxAllowed:F0}ms " +
$"(baseline: {baselineResult.ElapsedMs}ms, regression: +{regressionPct:F1}%, threshold: {effectiveThreshold}%)");
}
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: PASS ({elapsedMs}ms <= {maxAllowed:F0}ms, baseline: {baselineResult.ElapsedMs}ms, threshold: {effectiveThreshold}%)");
}
else if (RequireBaseline)
{
Assert.Fail($"No baseline entry found for '{name}' and BENCHMARK_REQUIRE_BASELINE=true");
}
else
{
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: SKIPPED (no baseline entry)");
}
}
/// <summary>
/// Measures, records, and asserts no regression for the given action.
/// </summary>
protected long MeasureAndAssertNoRegression(string name, int itemCount, Action action, bool skipWarmup = false, double thresholdPercent = -1)
{
// Warmup iteration (skip for destructive operations)
if (!skipWarmup)
{
try { action(); }
catch (Exception ex)
{
TestContext.WriteLine($"[WARMUP] {name} warmup failed: {ex.Message}");
}
}
var sw = Stopwatch.StartNew();
action();
sw.Stop();
AssertNoRegression(name, sw.ElapsedMilliseconds, itemCount, thresholdPercent);
return sw.ElapsedMilliseconds;
}
private static Dictionary<string, BenchmarkResult> LoadBaselineInternal()
{
if (!File.Exists(BaselinePath))
{
TestContext.WriteLine($"[BASELINE] File not found: {BaselinePath}");
return new Dictionary<string, BenchmarkResult>();
}
try
{
var json = File.ReadAllText(BaselinePath);
var results = JsonSerializer.Deserialize<List<BenchmarkResult>>(json) ?? new List<BenchmarkResult>();
TestContext.WriteLine($"[BASELINE] Loaded {results.Count} baseline entries from {BaselinePath}");
return results.ToDictionary(r => r.Name, r => r);
}
catch (Exception ex)
{
TestContext.WriteLine($"[BASELINE] Failed to load baseline: {ex.Message}");
return new Dictionary<string, BenchmarkResult>();
}
}
/// <summary>
/// Represents a single benchmark measurement.
/// </summary>

View File

@@ -65,7 +65,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
var sw = Stopwatch.StartNew();
ContentService.Save(content);
sw.Stop();
RecordBenchmark("Save_SingleItem", sw.ElapsedMilliseconds, 1);
// Gate: Fail if >20% regression from baseline
AssertNoRegression("Save_SingleItem", sw.ElapsedMilliseconds, 1);
Assert.That(content.Id, Is.GreaterThan(0));
}
@@ -97,7 +99,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
var sw = Stopwatch.StartNew();
ContentService.Save(items);
sw.Stop();
RecordBenchmark("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount);
// Gate: Fail if >20% regression from baseline
AssertNoRegression("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount);
Assert.That(items.All(c => c.Id > 0), Is.True);
}
@@ -129,7 +133,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
var sw = Stopwatch.StartNew();
ContentService.Save(items);
sw.Stop();
RecordBenchmark("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount);
// Gate: Fail if >20% regression from baseline
AssertNoRegression("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount);
Assert.That(items.All(c => c.Id > 0), Is.True);
}
@@ -147,7 +153,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
var id = content.Id;
IContent? result = null;
MeasureAndRecord("GetById_Single", 1, () =>
MeasureAndAssertNoRegression("GetById_Single", 1, () =>
{
result = ContentService.GetById(id);
});
@@ -176,7 +182,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
var ids = items.Select(c => c.Id).ToList();
IEnumerable<IContent>? results = null;
MeasureAndRecord("GetByIds_BatchOf100", itemCount, () =>
MeasureAndAssertNoRegression("GetByIds_BatchOf100", itemCount, () =>
{
results = ContentService.GetByIds(ids);
});
@@ -194,7 +200,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
var content = ContentBuilder.CreateSimpleContent(ContentType, "DeleteTest", -1);
ContentService.Save(content);
MeasureAndRecord("Delete_SingleItem", 1, () =>
MeasureAndAssertNoRegression("Delete_SingleItem", 1, () =>
{
ContentService.Delete(content);
}, skipWarmup: true); // Destructive operation - cannot repeat
@@ -220,7 +226,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
ContentService.Save(child);
}
MeasureAndRecord("Delete_WithDescendants", childCount + 1, () =>
MeasureAndAssertNoRegression("Delete_WithDescendants", childCount + 1, () =>
{
ContentService.Delete(parent);
}, skipWarmup: true); // Destructive operation - cannot repeat
@@ -253,7 +259,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
IEnumerable<IContent>? results = null;
long totalRecords = 0;
MeasureAndRecord("GetPagedChildren_100Items", childCount, () =>
MeasureAndAssertNoRegression("GetPagedChildren_100Items", childCount, () =>
{
results = ContentService.GetPagedChildren(parent.Id, 0, 100, out totalRecords);
});
@@ -293,7 +299,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
IEnumerable<IContent>? results = null;
long totalRecords = 0;
MeasureAndRecord("GetPagedDescendants_DeepTree", 300, () =>
MeasureAndAssertNoRegression("GetPagedDescendants_DeepTree", 300, () =>
{
results = ContentService.GetPagedDescendants(root.Id, 0, 1000, out totalRecords);
});
@@ -324,7 +330,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
var deepestId = current.Id;
IEnumerable<IContent>? results = null;
MeasureAndRecord("GetAncestors_DeepHierarchy", depth, () =>
MeasureAndAssertNoRegression("GetAncestors_DeepHierarchy", depth, () =>
{
results = ContentService.GetAncestors(deepestId);
});