// Copyright (c) Umbraco. // See LICENSE for more details. using System.Diagnostics; using System.Text.Json; using NUnit.Framework; namespace Umbraco.Cms.Tests.Integration.Testing; /// /// Base class for ContentService performance benchmarks. /// Extends UmbracoIntegrationTestWithContent with structured benchmark recording. /// /// /// Usage: /// /// [Test] /// [LongRunning] /// public void MyBenchmark() /// { /// var sw = Stopwatch.StartNew(); /// // ... operation under test ... /// sw.Stop(); /// RecordBenchmark("MyBenchmark", sw.ElapsedMilliseconds, itemCount); /// } /// /// /// Results are output in both human-readable and JSON formats for baseline comparison. /// public abstract class ContentServiceBenchmarkBase : UmbracoIntegrationTestWithContent { private readonly List _results = new(); // Regression enforcement configuration private const double DefaultRegressionThreshold = 20.0; // Allow CI override via environment variable private static readonly double RegressionThreshold = double.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REGRESSION_THRESHOLD"), out var t) ? t : DefaultRegressionThreshold; // Optional strict mode: fail if baseline is missing (useful for CI) private static readonly bool RequireBaseline = bool.TryParse(Environment.GetEnvironmentVariable("BENCHMARK_REQUIRE_BASELINE"), out var b) && b; // Thread-safe lazy initialization of repository root private static readonly Lazy _repositoryRoot = new(FindRepositoryRoot); // Thread-safe lazy initialization of baseline data private static readonly Lazy> _baselineLoader = new(() => LoadBaselineInternal(), LazyThreadSafetyMode.ExecutionAndPublication); private static Dictionary Baseline => _baselineLoader.Value; private static string BaselinePath => Path.Combine(_repositoryRoot.Value, "docs", "plans", "baseline-phase0.json"); /// /// Records a benchmark result for later output. /// /// Name of the benchmark (should match method name). /// Elapsed time in milliseconds. /// Number of items processed (for per-item metrics). protected void RecordBenchmark(string name, long elapsedMs, int itemCount) { var result = new BenchmarkResult(name, elapsedMs, itemCount); _results.Add(result); // Human-readable output TestContext.WriteLine($"[BENCHMARK] {name}: {elapsedMs}ms ({result.MsPerItem:F2}ms/item, {itemCount} items)"); } /// /// Records a benchmark result without item count (for single-item operations). /// protected void RecordBenchmark(string name, long elapsedMs) => RecordBenchmark(name, elapsedMs, 1); /// /// Measures and records a benchmark for the given action. /// /// Name of the benchmark. /// Number of items processed. /// The action to benchmark. /// Skip warmup for destructive operations (delete, empty recycle bin). /// Elapsed time in milliseconds. protected long MeasureAndRecord(string name, int itemCount, Action action, bool skipWarmup = false) { // Warmup iteration: triggers JIT compilation, warms connection pool and caches. // Skip for destructive operations that would fail on second execution. if (!skipWarmup) { try { action(); } catch { // Warmup failure is acceptable for some operations; continue to measured run } } var sw = Stopwatch.StartNew(); action(); sw.Stop(); RecordBenchmark(name, sw.ElapsedMilliseconds, itemCount); return sw.ElapsedMilliseconds; } /// /// Measures and records a benchmark, returning the result of the function. /// /// /// Performs a warmup call before measurement to trigger JIT compilation. /// Safe for read-only operations that can be repeated without side effects. /// protected T MeasureAndRecord(string name, int itemCount, Func func) { // Warmup: triggers JIT compilation, warms caches try { func(); } catch { /* ignore warmup errors */ } var sw = Stopwatch.StartNew(); var result = func(); sw.Stop(); RecordBenchmark(name, sw.ElapsedMilliseconds, itemCount); return result; } [TearDown] public void OutputBenchmarkResults() { if (_results.Count == 0) { return; } // JSON output for automated comparison // Wrapped in markers for easy extraction from test output var json = JsonSerializer.Serialize(_results, new JsonSerializerOptions { WriteIndented = true }); TestContext.WriteLine($"[BENCHMARK_JSON]{json}[/BENCHMARK_JSON]"); _results.Clear(); } /// /// Finds the repository root by searching for umbraco.sln. /// private static string FindRepositoryRoot() { var dir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); while (dir != null) { if (File.Exists(Path.Combine(dir.FullName, "umbraco.sln"))) { return dir.FullName; } dir = dir.Parent; } throw new InvalidOperationException( $"Cannot find repository root (umbraco.sln) starting from {TestContext.CurrentContext.TestDirectory}"); } /// /// Records a benchmark and asserts no regression beyond the threshold. /// /// Benchmark name (must match baseline JSON key). /// Measured elapsed time in milliseconds. /// Number of items processed. /// Maximum allowed regression percentage (default: 20%, configurable via BENCHMARK_REGRESSION_THRESHOLD env var). protected void AssertNoRegression(string name, long elapsedMs, int itemCount, double thresholdPercent = -1) { RecordBenchmark(name, elapsedMs, itemCount); // Use environment-configurable threshold if not explicitly specified var effectiveThreshold = thresholdPercent < 0 ? RegressionThreshold : thresholdPercent; if (Baseline.TryGetValue(name, out var baselineResult)) { var maxAllowed = baselineResult.ElapsedMs * (1 + effectiveThreshold / 100); if (elapsedMs > maxAllowed) { var regressionPct = ((double)(elapsedMs - baselineResult.ElapsedMs) / baselineResult.ElapsedMs) * 100; Assert.Fail( $"Performance regression detected for '{name}': " + $"{elapsedMs}ms exceeds threshold of {maxAllowed:F0}ms " + $"(baseline: {baselineResult.ElapsedMs}ms, regression: +{regressionPct:F1}%, threshold: {effectiveThreshold}%)"); } TestContext.WriteLine($"[REGRESSION_CHECK] {name}: PASS ({elapsedMs}ms <= {maxAllowed:F0}ms, baseline: {baselineResult.ElapsedMs}ms, threshold: {effectiveThreshold}%)"); } else if (RequireBaseline) { Assert.Fail($"No baseline entry found for '{name}' and BENCHMARK_REQUIRE_BASELINE=true"); } else { TestContext.WriteLine($"[REGRESSION_CHECK] {name}: SKIPPED (no baseline entry)"); } } /// /// Measures, records, and asserts no regression for the given action. /// protected long MeasureAndAssertNoRegression(string name, int itemCount, Action action, bool skipWarmup = false, double thresholdPercent = -1) { // Warmup iteration (skip for destructive operations) if (!skipWarmup) { try { action(); } catch (Exception ex) { TestContext.WriteLine($"[WARMUP] {name} warmup failed: {ex.Message}"); } } var sw = Stopwatch.StartNew(); action(); sw.Stop(); AssertNoRegression(name, sw.ElapsedMilliseconds, itemCount, thresholdPercent); return sw.ElapsedMilliseconds; } private static Dictionary LoadBaselineInternal() { if (!File.Exists(BaselinePath)) { TestContext.WriteLine($"[BASELINE] File not found: {BaselinePath}"); return new Dictionary(); } try { var json = File.ReadAllText(BaselinePath); var results = JsonSerializer.Deserialize>(json) ?? new List(); TestContext.WriteLine($"[BASELINE] Loaded {results.Count} baseline entries from {BaselinePath}"); return results.ToDictionary(r => r.Name, r => r); } catch (Exception ex) { TestContext.WriteLine($"[BASELINE] Failed to load baseline: {ex.Message}"); return new Dictionary(); } } /// /// Represents a single benchmark measurement. /// internal sealed record BenchmarkResult(string Name, long ElapsedMs, int ItemCount) { public double MsPerItem => ItemCount > 0 ? (double)ElapsedMs / ItemCount : 0; } }