test: add benchmark regression enforcement for Phase 1
Adds AssertNoRegression method to ContentServiceBenchmarkBase: - Loads baseline from baseline-phase0.json - Fails test if benchmark exceeds 20% of baseline - Logs regression check status for visibility - Supports environment variable overrides (BENCHMARK_REGRESSION_THRESHOLD, BENCHMARK_REQUIRE_BASELINE) Updates 10 Phase 1 CRUD benchmarks to use regression assertions: - Save_SingleItem, Save_BatchOf100, Save_BatchOf1000 - GetById_Single, GetByIds_BatchOf100 - Delete_SingleItem, Delete_WithDescendants - GetAncestors_DeepHierarchy, GetPagedChildren_100Items, GetPagedDescendants_DeepTree 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -31,6 +31,30 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
|
||||
{
|
||||
private readonly List<BenchmarkResult> _results = new();
|
||||
|
||||
// Regression enforcement configuration
|
||||
private const double DefaultRegressionThreshold = 20.0;
|
||||
|
||||
// Allow CI override via environment variable
|
||||
private static readonly double RegressionThreshold =
|
||||
double.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REGRESSION_THRESHOLD"), out var t)
|
||||
? t
|
||||
: DefaultRegressionThreshold;
|
||||
|
||||
// Optional strict mode: fail if baseline is missing (useful for CI)
|
||||
private static readonly bool RequireBaseline =
|
||||
bool.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REQUIRE_BASELINE"), out var b) && b;
|
||||
|
||||
// Thread-safe lazy initialization of repository root
|
||||
private static readonly Lazy<string> _repositoryRoot = new(FindRepositoryRoot);
|
||||
|
||||
// Thread-safe lazy initialization of baseline data
|
||||
private static readonly Lazy<Dictionary<string, BenchmarkResult>> _baselineLoader =
|
||||
new(() => LoadBaselineInternal(), LazyThreadSafetyMode.ExecutionAndPublication);
|
||||
|
||||
private static Dictionary<string, BenchmarkResult> Baseline => _baselineLoader.Value;
|
||||
|
||||
private static string BaselinePath => Path.Combine(_repositoryRoot.Value, "docs", "plans", "baseline-phase0.json");
|
||||
|
||||
/// <summary>
|
||||
/// Records a benchmark result for later output.
|
||||
/// </summary>
|
||||
@@ -118,6 +142,108 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
|
||||
_results.Clear();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the repository root by searching for umbraco.sln.
|
||||
/// </summary>
|
||||
private static string FindRepositoryRoot()
|
||||
{
|
||||
var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
|
||||
while (dir != null)
|
||||
{
|
||||
if (File.Exists(Path.Combine(dir.FullName, "umbraco.sln")))
|
||||
{
|
||||
return dir.FullName;
|
||||
}
|
||||
dir = dir.Parent;
|
||||
}
|
||||
throw new InvalidOperationException(
|
||||
$"Cannot find repository root (umbraco.sln) starting from {TestContext.CurrentContext.TestDirectory}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records a benchmark and asserts no regression beyond the threshold.
|
||||
/// </summary>
|
||||
/// <param name="name">Benchmark name (must match baseline JSON key).</param>
|
||||
/// <param name="elapsedMs">Measured elapsed time in milliseconds.</param>
|
||||
/// <param name="itemCount">Number of items processed.</param>
|
||||
/// <param name="thresholdPercent">Maximum allowed regression percentage (default: 20%, configurable via BENCHMARK_REGRESSION_THRESHOLD env var).</param>
|
||||
protected void AssertNoRegression(string name, long elapsedMs, int itemCount, double thresholdPercent = -1)
|
||||
{
|
||||
RecordBenchmark(name, elapsedMs, itemCount);
|
||||
|
||||
// Use environment-configurable threshold if not explicitly specified
|
||||
var effectiveThreshold = thresholdPercent < 0 ? RegressionThreshold : thresholdPercent;
|
||||
|
||||
if (Baseline.TryGetValue(name, out var baselineResult))
|
||||
{
|
||||
var maxAllowed = baselineResult.ElapsedMs * (1 + effectiveThreshold / 100);
|
||||
|
||||
if (elapsedMs > maxAllowed)
|
||||
{
|
||||
var regressionPct = ((double)(elapsedMs - baselineResult.ElapsedMs) / baselineResult.ElapsedMs) * 100;
|
||||
Assert.Fail(
|
||||
$"Performance regression detected for '{name}': " +
|
||||
$"{elapsedMs}ms exceeds threshold of {maxAllowed:F0}ms " +
|
||||
$"(baseline: {baselineResult.ElapsedMs}ms, regression: +{regressionPct:F1}%, threshold: {effectiveThreshold}%)");
|
||||
}
|
||||
|
||||
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: PASS ({elapsedMs}ms <= {maxAllowed:F0}ms, baseline: {baselineResult.ElapsedMs}ms, threshold: {effectiveThreshold}%)");
|
||||
}
|
||||
else if (RequireBaseline)
|
||||
{
|
||||
Assert.Fail($"No baseline entry found for '{name}' and BENCHMARK_REQUIRE_BASELINE=true");
|
||||
}
|
||||
else
|
||||
{
|
||||
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: SKIPPED (no baseline entry)");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures, records, and asserts no regression for the given action.
|
||||
/// </summary>
|
||||
protected long MeasureAndAssertNoRegression(string name, int itemCount, Action action, bool skipWarmup = false, double thresholdPercent = -1)
|
||||
{
|
||||
// Warmup iteration (skip for destructive operations)
|
||||
if (!skipWarmup)
|
||||
{
|
||||
try { action(); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestContext.WriteLine($"[WARMUP] {name} warmup failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
action();
|
||||
sw.Stop();
|
||||
|
||||
AssertNoRegression(name, sw.ElapsedMilliseconds, itemCount, thresholdPercent);
|
||||
return sw.ElapsedMilliseconds;
|
||||
}
|
||||
|
||||
private static Dictionary<string, BenchmarkResult> LoadBaselineInternal()
|
||||
{
|
||||
if (!File.Exists(BaselinePath))
|
||||
{
|
||||
TestContext.WriteLine($"[BASELINE] File not found: {BaselinePath}");
|
||||
return new Dictionary<string, BenchmarkResult>();
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var json = File.ReadAllText(BaselinePath);
|
||||
var results = JsonSerializer.Deserialize<List<BenchmarkResult>>(json) ?? new List<BenchmarkResult>();
|
||||
TestContext.WriteLine($"[BASELINE] Loaded {results.Count} baseline entries from {BaselinePath}");
|
||||
return results.ToDictionary(r => r.Name, r => r);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
TestContext.WriteLine($"[BASELINE] Failed to load baseline: {ex.Message}");
|
||||
return new Dictionary<string, BenchmarkResult>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a single benchmark measurement.
|
||||
/// </summary>
|
||||
|
||||
@@ -65,7 +65,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
var sw = Stopwatch.StartNew();
|
||||
ContentService.Save(content);
|
||||
sw.Stop();
|
||||
RecordBenchmark("Save_SingleItem", sw.ElapsedMilliseconds, 1);
|
||||
|
||||
// Gate: Fail if >20% regression from baseline
|
||||
AssertNoRegression("Save_SingleItem", sw.ElapsedMilliseconds, 1);
|
||||
|
||||
Assert.That(content.Id, Is.GreaterThan(0));
|
||||
}
|
||||
@@ -97,7 +99,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
var sw = Stopwatch.StartNew();
|
||||
ContentService.Save(items);
|
||||
sw.Stop();
|
||||
RecordBenchmark("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount);
|
||||
|
||||
// Gate: Fail if >20% regression from baseline
|
||||
AssertNoRegression("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount);
|
||||
|
||||
Assert.That(items.All(c => c.Id > 0), Is.True);
|
||||
}
|
||||
@@ -129,7 +133,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
var sw = Stopwatch.StartNew();
|
||||
ContentService.Save(items);
|
||||
sw.Stop();
|
||||
RecordBenchmark("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount);
|
||||
|
||||
// Gate: Fail if >20% regression from baseline
|
||||
AssertNoRegression("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount);
|
||||
|
||||
Assert.That(items.All(c => c.Id > 0), Is.True);
|
||||
}
|
||||
@@ -147,7 +153,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
var id = content.Id;
|
||||
|
||||
IContent? result = null;
|
||||
MeasureAndRecord("GetById_Single", 1, () =>
|
||||
MeasureAndAssertNoRegression("GetById_Single", 1, () =>
|
||||
{
|
||||
result = ContentService.GetById(id);
|
||||
});
|
||||
@@ -176,7 +182,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
var ids = items.Select(c => c.Id).ToList();
|
||||
|
||||
IEnumerable<IContent>? results = null;
|
||||
MeasureAndRecord("GetByIds_BatchOf100", itemCount, () =>
|
||||
MeasureAndAssertNoRegression("GetByIds_BatchOf100", itemCount, () =>
|
||||
{
|
||||
results = ContentService.GetByIds(ids);
|
||||
});
|
||||
@@ -194,7 +200,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
var content = ContentBuilder.CreateSimpleContent(ContentType, "DeleteTest", -1);
|
||||
ContentService.Save(content);
|
||||
|
||||
MeasureAndRecord("Delete_SingleItem", 1, () =>
|
||||
MeasureAndAssertNoRegression("Delete_SingleItem", 1, () =>
|
||||
{
|
||||
ContentService.Delete(content);
|
||||
}, skipWarmup: true); // Destructive operation - cannot repeat
|
||||
@@ -220,7 +226,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
ContentService.Save(child);
|
||||
}
|
||||
|
||||
MeasureAndRecord("Delete_WithDescendants", childCount + 1, () =>
|
||||
MeasureAndAssertNoRegression("Delete_WithDescendants", childCount + 1, () =>
|
||||
{
|
||||
ContentService.Delete(parent);
|
||||
}, skipWarmup: true); // Destructive operation - cannot repeat
|
||||
@@ -253,7 +259,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
IEnumerable<IContent>? results = null;
|
||||
long totalRecords = 0;
|
||||
|
||||
MeasureAndRecord("GetPagedChildren_100Items", childCount, () =>
|
||||
MeasureAndAssertNoRegression("GetPagedChildren_100Items", childCount, () =>
|
||||
{
|
||||
results = ContentService.GetPagedChildren(parent.Id, 0, 100, out totalRecords);
|
||||
});
|
||||
@@ -293,7 +299,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
IEnumerable<IContent>? results = null;
|
||||
long totalRecords = 0;
|
||||
|
||||
MeasureAndRecord("GetPagedDescendants_DeepTree", 300, () =>
|
||||
MeasureAndAssertNoRegression("GetPagedDescendants_DeepTree", 300, () =>
|
||||
{
|
||||
results = ContentService.GetPagedDescendants(root.Id, 0, 1000, out totalRecords);
|
||||
});
|
||||
@@ -324,7 +330,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
||||
var deepestId = current.Id;
|
||||
|
||||
IEnumerable<IContent>? results = null;
|
||||
MeasureAndRecord("GetAncestors_DeepHierarchy", depth, () =>
|
||||
MeasureAndAssertNoRegression("GetAncestors_DeepHierarchy", depth, () =>
|
||||
{
|
||||
results = ContentService.GetAncestors(deepestId);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user