// Copyright (c) Umbraco.
// See LICENSE for more details.
using System.Diagnostics;
using System.Text.Json;
using NUnit.Framework;
namespace Umbraco.Cms.Tests.Integration.Testing;
///
/// Base class for ContentService performance benchmarks.
/// Extends UmbracoIntegrationTestWithContent with structured benchmark recording.
///
///
/// Usage:
///
/// [Test]
/// [LongRunning]
/// public void MyBenchmark()
/// {
/// var sw = Stopwatch.StartNew();
/// // ... operation under test ...
/// sw.Stop();
/// RecordBenchmark("MyBenchmark", sw.ElapsedMilliseconds, itemCount);
/// }
///
///
/// Results are output in both human-readable and JSON formats for baseline comparison.
///
public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithContent
{
private readonly List _results = new();
// Regression enforcement configuration
private const double DefaultRegressionThreshold = 20.0;
// Allow CI override via environment variable
private static readonly double RegressionThreshold =
double.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REGRESSION_THRESHOLD"), out var t)
? t
: DefaultRegressionThreshold;
// Optional strict mode: fail if baseline is missing (useful for CI)
private static readonly bool RequireBaseline =
bool.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REQUIRE_BASELINE"), out var b) && b;
// Thread-safe lazy initialization of repository root
private static readonly Lazy _repositoryRoot = new(FindRepositoryRoot);
// Thread-safe lazy initialization of baseline data
private static readonly Lazy> _baselineLoader =
new(() => LoadBaselineInternal(), LazyThreadSafetyMode.ExecutionAndPublication);
private static Dictionary Baseline => _baselineLoader.Value;
private static string BaselinePath => Path.Combine(_repositoryRoot.Value, "docs", "plans", "baseline-phase0.json");
///
/// Records a benchmark result for later output.
///
/// Name of the benchmark (should match method name).
/// Elapsed time in milliseconds.
/// Number of items processed (for per-item metrics).
protected void RecordBenchmark(string name, long elapsedMs, int itemCount)
{
var result = new BenchmarkResult(name, elapsedMs, itemCount);
_results.Add(result);
// Human-readable output
TestContext.WriteLine($"[BENCHMARK] {name}: {elapsedMs}ms ({result.MsPerItem:F2}ms/item, {itemCount} items)");
}
///
/// Records a benchmark result without item count (for single-item operations).
///
protected void RecordBenchmark(string name, long elapsedMs)
=> RecordBenchmark(name, elapsedMs, 1);
///
/// Measures and records a benchmark for the given action.
///
/// Name of the benchmark.
/// Number of items processed.
/// The action to benchmark.
/// Skip warmup for destructive operations (delete, empty recycle bin).
/// Elapsed time in milliseconds.
protected long MeasureAndRecord(string name, int itemCount, Action action, bool skipWarmup = false)
{
// Warmup iteration: triggers JIT compilation, warms connection pool and caches.
// Skip for destructive operations that would fail on second execution.
if (!skipWarmup)
{
try
{
action();
}
catch
{
// Warmup failure is acceptable for some operations; continue to measured run
}
}
var sw = Stopwatch.StartNew();
action();
sw.Stop();
RecordBenchmark(name, sw.ElapsedMilliseconds, itemCount);
return sw.ElapsedMilliseconds;
}
///
/// Measures and records a benchmark, returning the result of the function.
///
///
/// Performs a warmup call before measurement to trigger JIT compilation.
/// Safe for read-only operations that can be repeated without side effects.
///
protected T MeasureAndRecord(string name, int itemCount, Func func)
{
// Warmup: triggers JIT compilation, warms caches
try { func(); } catch { /* ignore warmup errors */ }
var sw = Stopwatch.StartNew();
var result = func();
sw.Stop();
RecordBenchmark(name, sw.ElapsedMilliseconds, itemCount);
return result;
}
[TearDown]
public void OutputBenchmarkResults()
{
if (_results.Count == 0)
{
return;
}
// JSON output for automated comparison
// Wrapped in markers for easy extraction from test output
var json = JsonSerializer.Serialize(_results, new JsonSerializerOptions { WriteIndented = true });
TestContext.WriteLine($"[BENCHMARK_JSON]{json}[/BENCHMARK_JSON]");
_results.Clear();
}
///
/// Finds the repository root by searching for umbraco.sln.
///
private static string FindRepositoryRoot()
{
var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
while (dir != null)
{
if (File.Exists(Path.Combine(dir.FullName, "umbraco.sln")))
{
return dir.FullName;
}
dir = dir.Parent;
}
throw new InvalidOperationException(
$"Cannot find repository root (umbraco.sln) starting from {TestContext.CurrentContext.TestDirectory}");
}
///
/// Records a benchmark and asserts no regression beyond the threshold.
///
/// Benchmark name (must match baseline JSON key).
/// Measured elapsed time in milliseconds.
/// Number of items processed.
/// Maximum allowed regression percentage (default: 20%, configurable via BENCHMARK_REGRESSION_THRESHOLD env var).
protected void AssertNoRegression(string name, long elapsedMs, int itemCount, double thresholdPercent = -1)
{
RecordBenchmark(name, elapsedMs, itemCount);
// Use environment-configurable threshold if not explicitly specified
var effectiveThreshold = thresholdPercent < 0 ? RegressionThreshold : thresholdPercent;
if (Baseline.TryGetValue(name, out var baselineResult))
{
var maxAllowed = baselineResult.ElapsedMs * (1 + effectiveThreshold / 100);
if (elapsedMs > maxAllowed)
{
var regressionPct = ((double)(elapsedMs - baselineResult.ElapsedMs) / baselineResult.ElapsedMs) * 100;
Assert.Fail(
$"Performance regression detected for '{name}': " +
$"{elapsedMs}ms exceeds threshold of {maxAllowed:F0}ms " +
$"(baseline: {baselineResult.ElapsedMs}ms, regression: +{regressionPct:F1}%, threshold: {effectiveThreshold}%)");
}
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: PASS ({elapsedMs}ms <= {maxAllowed:F0}ms, baseline: {baselineResult.ElapsedMs}ms, threshold: {effectiveThreshold}%)");
}
else if (RequireBaseline)
{
Assert.Fail($"No baseline entry found for '{name}' and BENCHMARK_REQUIRE_BASELINE=true");
}
else
{
TestContext.WriteLine($"[REGRESSION_CHECK] {name}: SKIPPED (no baseline entry)");
}
}
///
/// Measures, records, and asserts no regression for the given action.
///
protected long MeasureAndAssertNoRegression(string name, int itemCount, Action action, bool skipWarmup = false, double thresholdPercent = -1)
{
// Warmup iteration (skip for destructive operations)
if (!skipWarmup)
{
try { action(); }
catch (Exception ex)
{
TestContext.WriteLine($"[WARMUP] {name} warmup failed: {ex.Message}");
}
}
var sw = Stopwatch.StartNew();
action();
sw.Stop();
AssertNoRegression(name, sw.ElapsedMilliseconds, itemCount, thresholdPercent);
return sw.ElapsedMilliseconds;
}
private static Dictionary LoadBaselineInternal()
{
if (!File.Exists(BaselinePath))
{
TestContext.WriteLine($"[BASELINE] File not found: {BaselinePath}");
return new Dictionary();
}
try
{
var json = File.ReadAllText(BaselinePath);
var results = JsonSerializer.Deserialize>(json) ?? new List();
TestContext.WriteLine($"[BASELINE] Loaded {results.Count} baseline entries from {BaselinePath}");
return results.ToDictionary(r => r.Name, r => r);
}
catch (Exception ex)
{
TestContext.WriteLine($"[BASELINE] Failed to load baseline: {ex.Message}");
return new Dictionary();
}
}
///
/// Represents a single benchmark measurement.
///
internal sealed record BenchmarkResult(string Name, long ElapsedMs, int ItemCount)
{
public double MsPerItem => ItemCount > 0 ? (double)ElapsedMs / ItemCount : 0;
}
}