test: add benchmark regression enforcement for Phase 1
Adds AssertNoRegression method to ContentServiceBenchmarkBase: - Loads baseline from baseline-phase0.json - Fails test if benchmark exceeds 20% of baseline - Logs regression check status for visibility - Supports environment variable overrides (BENCHMARK_REGRESSION_THRESHOLD, BENCHMARK_REQUIRE_BASELINE) Updates 10 Phase 1 CRUD benchmarks to use regression assertions: - Save_SingleItem, Save_BatchOf100, Save_BatchOf1000 - GetById_Single, GetByIds_BatchOf100 - Delete_SingleItem, Delete_WithDescendants - GetAncestors_DeepHierarchy, GetPagedChildren_100Items, GetPagedDescendants_DeepTree 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -31,6 +31,30 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
|
|||||||
{
|
{
|
||||||
private readonly List<BenchmarkResult> _results = new();
|
private readonly List<BenchmarkResult> _results = new();
|
||||||
|
|
||||||
|
// Regression enforcement configuration
|
||||||
|
private const double DefaultRegressionThreshold = 20.0;
|
||||||
|
|
||||||
|
// Allow CI override via environment variable
|
||||||
|
private static readonly double RegressionThreshold =
|
||||||
|
double.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REGRESSION_THRESHOLD"), out var t)
|
||||||
|
? t
|
||||||
|
: DefaultRegressionThreshold;
|
||||||
|
|
||||||
|
// Optional strict mode: fail if baseline is missing (useful for CI)
|
||||||
|
private static readonly bool RequireBaseline =
|
||||||
|
bool.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REQUIRE_BASELINE"), out var b) && b;
|
||||||
|
|
||||||
|
// Thread-safe lazy initialization of repository root
|
||||||
|
private static readonly Lazy<string> _repositoryRoot = new(FindRepositoryRoot);
|
||||||
|
|
||||||
|
// Thread-safe lazy initialization of baseline data
|
||||||
|
private static readonly Lazy<Dictionary<string, BenchmarkResult>> _baselineLoader =
|
||||||
|
new(() => LoadBaselineInternal(), LazyThreadSafetyMode.ExecutionAndPublication);
|
||||||
|
|
||||||
|
private static Dictionary<string, BenchmarkResult> Baseline => _baselineLoader.Value;
|
||||||
|
|
||||||
|
private static string BaselinePath => Path.Combine(_repositoryRoot.Value, "docs", "plans", "baseline-phase0.json");
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a benchmark result for later output.
|
/// Records a benchmark result for later output.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -118,6 +142,108 @@ public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithCo
|
|||||||
_results.Clear();
|
_results.Clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Finds the repository root by searching for umbraco.sln.
|
||||||
|
/// </summary>
|
||||||
|
private static string FindRepositoryRoot()
|
||||||
|
{
|
||||||
|
var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
|
||||||
|
while (dir != null)
|
||||||
|
{
|
||||||
|
if (File.Exists(Path.Combine(dir.FullName, "umbraco.sln")))
|
||||||
|
{
|
||||||
|
return dir.FullName;
|
||||||
|
}
|
||||||
|
dir = dir.Parent;
|
||||||
|
}
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"Cannot find repository root (umbraco.sln) starting from {TestContext.CurrentContext.TestDirectory}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Records a benchmark and asserts no regression beyond the threshold.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="name">Benchmark name (must match baseline JSON key).</param>
|
||||||
|
/// <param name="elapsedMs">Measured elapsed time in milliseconds.</param>
|
||||||
|
/// <param name="itemCount">Number of items processed.</param>
|
||||||
|
/// <param name="thresholdPercent">Maximum allowed regression percentage (default: 20%, configurable via BENCHMARK_REGRESSION_THRESHOLD env var).</param>
|
||||||
|
protected void AssertNoRegression(string name, long elapsedMs, int itemCount, double thresholdPercent = -1)
|
||||||
|
{
|
||||||
|
RecordBenchmark(name, elapsedMs, itemCount);
|
||||||
|
|
||||||
|
// Use environment-configurable threshold if not explicitly specified
|
||||||
|
var effectiveThreshold = thresholdPercent < 0 ? RegressionThreshold : thresholdPercent;
|
||||||
|
|
||||||
|
if (Baseline.TryGetValue(name, out var baselineResult))
|
||||||
|
{
|
||||||
|
var maxAllowed = baselineResult.ElapsedMs * (1 + effectiveThreshold / 100);
|
||||||
|
|
||||||
|
if (elapsedMs > maxAllowed)
|
||||||
|
{
|
||||||
|
var regressionPct = ((double)(elapsedMs - baselineResult.ElapsedMs) / baselineResult.ElapsedMs) * 100;
|
||||||
|
Assert.Fail(
|
||||||
|
$"Performance regression detected for '{name}': " +
|
||||||
|
$"{elapsedMs}ms exceeds threshold of {maxAllowed:F0}ms " +
|
||||||
|
$"(baseline: {baselineResult.ElapsedMs}ms, regression: +{regressionPct:F1}%, threshold: {effectiveThreshold}%)");
|
||||||
|
}
|
||||||
|
|
||||||
|
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: PASS ({elapsedMs}ms <= {maxAllowed:F0}ms, baseline: {baselineResult.ElapsedMs}ms, threshold: {effectiveThreshold}%)");
|
||||||
|
}
|
||||||
|
else if (RequireBaseline)
|
||||||
|
{
|
||||||
|
Assert.Fail($"No baseline entry found for '{name}' and BENCHMARK_REQUIRE_BASELINE=true");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: SKIPPED (no baseline entry)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Measures, records, and asserts no regression for the given action.
|
||||||
|
/// </summary>
|
||||||
|
protected long MeasureAndAssertNoRegression(string name, int itemCount, Action action, bool skipWarmup = false, double thresholdPercent = -1)
|
||||||
|
{
|
||||||
|
// Warmup iteration (skip for destructive operations)
|
||||||
|
if (!skipWarmup)
|
||||||
|
{
|
||||||
|
try { action(); }
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
TestContext.WriteLine($"[WARMUP] {name} warmup failed: {ex.Message}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var sw = Stopwatch.StartNew();
|
||||||
|
action();
|
||||||
|
sw.Stop();
|
||||||
|
|
||||||
|
AssertNoRegression(name, sw.ElapsedMilliseconds, itemCount, thresholdPercent);
|
||||||
|
return sw.ElapsedMilliseconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Dictionary<string, BenchmarkResult> LoadBaselineInternal()
|
||||||
|
{
|
||||||
|
if (!File.Exists(BaselinePath))
|
||||||
|
{
|
||||||
|
TestContext.WriteLine($"[BASELINE] File not found: {BaselinePath}");
|
||||||
|
return new Dictionary<string, BenchmarkResult>();
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var json = File.ReadAllText(BaselinePath);
|
||||||
|
var results = JsonSerializer.Deserialize<List<BenchmarkResult>>(json) ?? new List<BenchmarkResult>();
|
||||||
|
TestContext.WriteLine($"[BASELINE] Loaded {results.Count} baseline entries from {BaselinePath}");
|
||||||
|
return results.ToDictionary(r => r.Name, r => r);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
TestContext.WriteLine($"[BASELINE] Failed to load baseline: {ex.Message}");
|
||||||
|
return new Dictionary<string, BenchmarkResult>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Represents a single benchmark measurement.
|
/// Represents a single benchmark measurement.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -65,7 +65,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
var sw = Stopwatch.StartNew();
|
var sw = Stopwatch.StartNew();
|
||||||
ContentService.Save(content);
|
ContentService.Save(content);
|
||||||
sw.Stop();
|
sw.Stop();
|
||||||
RecordBenchmark("Save_SingleItem", sw.ElapsedMilliseconds, 1);
|
|
||||||
|
// Gate: Fail if >20% regression from baseline
|
||||||
|
AssertNoRegression("Save_SingleItem", sw.ElapsedMilliseconds, 1);
|
||||||
|
|
||||||
Assert.That(content.Id, Is.GreaterThan(0));
|
Assert.That(content.Id, Is.GreaterThan(0));
|
||||||
}
|
}
|
||||||
@@ -97,7 +99,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
var sw = Stopwatch.StartNew();
|
var sw = Stopwatch.StartNew();
|
||||||
ContentService.Save(items);
|
ContentService.Save(items);
|
||||||
sw.Stop();
|
sw.Stop();
|
||||||
RecordBenchmark("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount);
|
|
||||||
|
// Gate: Fail if >20% regression from baseline
|
||||||
|
AssertNoRegression("Save_BatchOf100", sw.ElapsedMilliseconds, itemCount);
|
||||||
|
|
||||||
Assert.That(items.All(c => c.Id > 0), Is.True);
|
Assert.That(items.All(c => c.Id > 0), Is.True);
|
||||||
}
|
}
|
||||||
@@ -129,7 +133,9 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
var sw = Stopwatch.StartNew();
|
var sw = Stopwatch.StartNew();
|
||||||
ContentService.Save(items);
|
ContentService.Save(items);
|
||||||
sw.Stop();
|
sw.Stop();
|
||||||
RecordBenchmark("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount);
|
|
||||||
|
// Gate: Fail if >20% regression from baseline
|
||||||
|
AssertNoRegression("Save_BatchOf1000", sw.ElapsedMilliseconds, itemCount);
|
||||||
|
|
||||||
Assert.That(items.All(c => c.Id > 0), Is.True);
|
Assert.That(items.All(c => c.Id > 0), Is.True);
|
||||||
}
|
}
|
||||||
@@ -147,7 +153,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
var id = content.Id;
|
var id = content.Id;
|
||||||
|
|
||||||
IContent? result = null;
|
IContent? result = null;
|
||||||
MeasureAndRecord("GetById_Single", 1, () =>
|
MeasureAndAssertNoRegression("GetById_Single", 1, () =>
|
||||||
{
|
{
|
||||||
result = ContentService.GetById(id);
|
result = ContentService.GetById(id);
|
||||||
});
|
});
|
||||||
@@ -176,7 +182,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
var ids = items.Select(c => c.Id).ToList();
|
var ids = items.Select(c => c.Id).ToList();
|
||||||
|
|
||||||
IEnumerable<IContent>? results = null;
|
IEnumerable<IContent>? results = null;
|
||||||
MeasureAndRecord("GetByIds_BatchOf100", itemCount, () =>
|
MeasureAndAssertNoRegression("GetByIds_BatchOf100", itemCount, () =>
|
||||||
{
|
{
|
||||||
results = ContentService.GetByIds(ids);
|
results = ContentService.GetByIds(ids);
|
||||||
});
|
});
|
||||||
@@ -194,7 +200,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
var content = ContentBuilder.CreateSimpleContent(ContentType, "DeleteTest", -1);
|
var content = ContentBuilder.CreateSimpleContent(ContentType, "DeleteTest", -1);
|
||||||
ContentService.Save(content);
|
ContentService.Save(content);
|
||||||
|
|
||||||
MeasureAndRecord("Delete_SingleItem", 1, () =>
|
MeasureAndAssertNoRegression("Delete_SingleItem", 1, () =>
|
||||||
{
|
{
|
||||||
ContentService.Delete(content);
|
ContentService.Delete(content);
|
||||||
}, skipWarmup: true); // Destructive operation - cannot repeat
|
}, skipWarmup: true); // Destructive operation - cannot repeat
|
||||||
@@ -220,7 +226,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
ContentService.Save(child);
|
ContentService.Save(child);
|
||||||
}
|
}
|
||||||
|
|
||||||
MeasureAndRecord("Delete_WithDescendants", childCount + 1, () =>
|
MeasureAndAssertNoRegression("Delete_WithDescendants", childCount + 1, () =>
|
||||||
{
|
{
|
||||||
ContentService.Delete(parent);
|
ContentService.Delete(parent);
|
||||||
}, skipWarmup: true); // Destructive operation - cannot repeat
|
}, skipWarmup: true); // Destructive operation - cannot repeat
|
||||||
@@ -253,7 +259,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
IEnumerable<IContent>? results = null;
|
IEnumerable<IContent>? results = null;
|
||||||
long totalRecords = 0;
|
long totalRecords = 0;
|
||||||
|
|
||||||
MeasureAndRecord("GetPagedChildren_100Items", childCount, () =>
|
MeasureAndAssertNoRegression("GetPagedChildren_100Items", childCount, () =>
|
||||||
{
|
{
|
||||||
results = ContentService.GetPagedChildren(parent.Id, 0, 100, out totalRecords);
|
results = ContentService.GetPagedChildren(parent.Id, 0, 100, out totalRecords);
|
||||||
});
|
});
|
||||||
@@ -293,7 +299,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
IEnumerable<IContent>? results = null;
|
IEnumerable<IContent>? results = null;
|
||||||
long totalRecords = 0;
|
long totalRecords = 0;
|
||||||
|
|
||||||
MeasureAndRecord("GetPagedDescendants_DeepTree", 300, () =>
|
MeasureAndAssertNoRegression("GetPagedDescendants_DeepTree", 300, () =>
|
||||||
{
|
{
|
||||||
results = ContentService.GetPagedDescendants(root.Id, 0, 1000, out totalRecords);
|
results = ContentService.GetPagedDescendants(root.Id, 0, 1000, out totalRecords);
|
||||||
});
|
});
|
||||||
@@ -324,7 +330,7 @@ internal sealed class ContentServiceRefactoringBenchmarks : ContentServiceBenchm
|
|||||||
var deepestId = current.Id;
|
var deepestId = current.Id;
|
||||||
|
|
||||||
IEnumerable<IContent>? results = null;
|
IEnumerable<IContent>? results = null;
|
||||||
MeasureAndRecord("GetAncestors_DeepHierarchy", depth, () =>
|
MeasureAndAssertNoRegression("GetAncestors_DeepHierarchy", depth, () =>
|
||||||
{
|
{
|
||||||
results = ContentService.GetAncestors(deepestId);
|
results = ContentService.GetAncestors(deepestId);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user