test: add benchmark regression enforcement for Phase 1

Adds AssertNoRegression method to ContentServiceBenchmarkBase:
- Loads baseline from baseline-phase0.json
- Fails test if benchmark exceeds 20% of baseline
- Logs regression check status for visibility
- Supports environment variable overrides (BENCHMARK_REGRESSION_THRESHOLD, BENCHMARK_REQUIRE_BASELINE)

Updates 10 Phase 1 CRUD benchmarks to use regression assertions:
- Save_SingleItem, Save_BatchOf100, Save_BatchOf1000
- GetById_Single, GetByIds_BatchOf100
- Delete_SingleItem, Delete_WithDescendants
- GetAncestors_DeepHierarchy, GetPagedChildren_100Items, GetPagedDescendants_DeepTree

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-12-21 03:43:28 +00:00
parent 9962df50ee
commit 1d082e1ed6
2 changed files with 142 additions and 10 deletions

View File

@@ -31,6 +31,30 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
{
private readonly List<BenchmarkResult> _results = new();
// Regression enforcement configuration
private const double DefaultRegressionThreshold = 20.0;
// Allow CI override via environment variable
private static readonly double RegressionThreshold =
double.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REGRESSION_THRESHOLD"), out var t)
? t
: DefaultRegressionThreshold;
// Optional strict mode: fail if baseline is missing (useful for CI)
private static readonly bool RequireBaseline =
bool.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REQUIRE_BASELINE"), out var b) && b;
// Thread-safe lazy initialization of repository root
private static readonly Lazy<string> _repositoryRoot = new(FindRepositoryRoot);
// Thread-safe lazy initialization of baseline data
private static readonly Lazy<Dictionary<string, BenchmarkResult>> _baselineLoader =
new(() => LoadBaselineInternal(), LazyThreadSafetyMode.ExecutionAndPublication);
private static Dictionary<string, BenchmarkResult> Baseline => _baselineLoader.Value;
private static string BaselinePath => Path.Combine(_repositoryRoot.Value, "docs", "plans", "baseline-phase0.json");
/// <summary>
/// Records a benchmark result for later output.
/// </summary>
@@ -118,6 +142,108 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
_results.Clear();
}
/// <summary>
/// Finds the repository root by searching for umbraco.sln.
/// </summary>
private static string FindRepositoryRoot()
{
var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
while (dir != null)
{
if (File.Exists(Path.Combine(dir.FullName, "umbraco.sln")))
{
return dir.FullName;
}
dir = dir.Parent;
}
throw new InvalidOperationException(
$"Cannot find repository root (umbraco.sln) starting from {TestContext.CurrentContext.TestDirectory}");
}
/// <summary>
/// Records a benchmark and asserts no regression beyond the threshold.
/// </summary>
/// <param name="name">Benchmark name (must match baseline JSON key).</param>
/// <param name="elapsedMs">Measured elapsed time in milliseconds.</param>
/// <param name="itemCount">Number of items processed.</param>
/// <param name="thresholdPercent">Maximum allowed regression percentage (default: 20%, configurable via BENCHMARK_REGRESSION_THRESHOLD env var).</param>
protected void AssertNoRegression(string name, long elapsedMs, int itemCount, double thresholdPercent = -1)
{
RecordBenchmark(name, elapsedMs, itemCount);
// Use environment-configurable threshold if not explicitly specified
var effectiveThreshold = thresholdPercent < 0 ? RegressionThreshold : thresholdPercent;
if (Baseline.TryGetValue(name, out var baselineResult))
{
var maxAllowed = baselineResult.ElapsedMs * (1 + effectiveThreshold / 100);
if (elapsedMs > maxAllowed)
{
var regressionPct = ((double)(elapsedMs - baselineResult.ElapsedMs) / baselineResult.ElapsedMs) * 100;
Assert.Fail(
$"Performance regression detected for '{name}': " +
$"{elapsedMs}ms exceeds threshold of {maxAllowed:F0}ms " +
$"(baseline: {baselineResult.ElapsedMs}ms, regression: +{regressionPct:F1}%, threshold: {effectiveThreshold}%)");
}
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: PASS ({elapsedMs}ms <= {maxAllowed:F0}ms, baseline: {baselineResult.ElapsedMs}ms, threshold: {effectiveThreshold}%)");
}
else if (RequireBaseline)
{
Assert.Fail($"No baseline entry found for '{name}' and BENCHMARK_REQUIRE_BASELINE=true");
}
else
{
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: SKIPPED (no baseline entry)");
}
}
/// <summary>
/// Measures, records, and asserts no regression for the given action.
/// </summary>
protected long MeasureAndAssertNoRegression(string name, int itemCount, Action action, bool skipWarmup = false, double thresholdPercent = -1)
{
// Warmup iteration (skip for destructive operations)
if (!skipWarmup)
{
try { action(); }
catch (Exception ex)
{
TestContext.WriteLine($"[WARMUP] {name} warmup failed: {ex.Message}");
}
}
var sw = Stopwatch.StartNew();
action();
sw.Stop();
AssertNoRegression(name, sw.ElapsedMilliseconds, itemCount, thresholdPercent);
return sw.ElapsedMilliseconds;
}
private static Dictionary<string, BenchmarkResult> LoadBaselineInternal()
{
if (!File.Exists(BaselinePath))
{
TestContext.WriteLine($"[BASELINE] File not found: {BaselinePath}");
return new Dictionary<string, BenchmarkResult>();
}
try
{
var json = File.ReadAllText(BaselinePath);
var results = JsonSerializer.Deserialize<List<BenchmarkResult>>(json) ?? new List<BenchmarkResult>();
TestContext.WriteLine($"[BASELINE] Loaded {results.Count} baseline entries from {BaselinePath}");
return results.ToDictionary(r => r.Name, r => r);
}
catch (Exception ex)
{
TestContext.WriteLine($"[BASELINE] Failed to load baseline: {ex.Message}");
return new Dictionary<string, BenchmarkResult>();
}
}
/// <summary>
/// Represents a single benchmark measurement.
/// </summary>