360 lines
10 KiB
Nim
360 lines
10 KiB
Nim
## Dependency Resolver Performance Benchmarks
|
|
##
|
|
## This benchmark suite measures resolver performance across different
|
|
## package complexity levels and validates cache effectiveness.
|
|
##
|
|
## **Benchmark Categories:**
|
|
## - Simple packages (10-20 dependencies)
|
|
## - Complex packages (50-100 dependencies)
|
|
## - Massive packages (200+ dependencies)
|
|
## - Cache effectiveness (hit rates, speedup)
|
|
##
|
|
## **Metrics Tracked:**
|
|
## - Resolution time (cold cache)
|
|
## - Resolution time (warm cache)
|
|
## - Cache hit rate
|
|
## - Memory usage
|
|
## - Speedup factor
|
|
|
|
import times
|
|
import strformat
|
|
import tables
|
|
import ../src/nip/resolver/types
|
|
import ../src/nip/resolver/graph_builder
|
|
import ../src/nip/resolver/resolution_cache
|
|
import ../src/nip/resolver/serialization
|
|
import ../src/nip/cas/storage
|
|
|
|
type
|
|
BenchmarkResult* = object
|
|
name*: string
|
|
packageCount*: int
|
|
dependencyCount*: int
|
|
coldTime*: float # seconds
|
|
warmTime*: float # seconds
|
|
cacheHitRate*: float
|
|
speedup*: float
|
|
memoryUsed*: int # bytes
|
|
|
|
BenchmarkSuite* = object
|
|
results*: seq[BenchmarkResult]
|
|
totalTime*: float
|
|
|
|
# ============================================================================
|
|
# Test Data Generation
|
|
# ============================================================================
|
|
|
|
proc generateSimplePackage*(name: string, depCount: int): PackageSpec =
|
|
## Generate simple package with linear dependencies
|
|
result = PackageSpec(
|
|
name: name,
|
|
version: "1.0.0",
|
|
dependencies: @[]
|
|
)
|
|
|
|
for i in 0..<depCount:
|
|
result.dependencies.add(PackageSpec(
|
|
name: fmt"dep-{i}",
|
|
version: "1.0.0",
|
|
dependencies: @[]
|
|
))
|
|
|
|
proc generateComplexPackage*(name: string, depCount: int): PackageSpec =
|
|
## Generate complex package with diamond dependencies
|
|
result = PackageSpec(
|
|
name: name,
|
|
version: "1.0.0",
|
|
dependencies: @[]
|
|
)
|
|
|
|
# Create diamond pattern: A → B,C → D
|
|
let layers = depCount div 3
|
|
|
|
for layer in 0..<layers:
|
|
for branch in 0..1:
|
|
let depName = fmt"dep-L{layer}-B{branch}"
|
|
var dep = PackageSpec(
|
|
name: depName,
|
|
version: "1.0.0",
|
|
dependencies: @[]
|
|
)
|
|
|
|
# Add shared dependency
|
|
if layer < layers - 1:
|
|
dep.dependencies.add(PackageSpec(
|
|
name: fmt"dep-L{layer+1}-shared",
|
|
version: "1.0.0",
|
|
dependencies: @[]
|
|
))
|
|
|
|
result.dependencies.add(dep)
|
|
|
|
proc generateMassivePackage*(name: string, depCount: int): PackageSpec =
|
|
## Generate massive package with deep dependency tree
|
|
result = PackageSpec(
|
|
name: name,
|
|
version: "1.0.0",
|
|
dependencies: @[]
|
|
)
|
|
|
|
# Create deep tree with branching
|
|
let depth = 10
|
|
let branchFactor = depCount div depth
|
|
|
|
proc addDeps(pkg: var PackageSpec, currentDepth: int, maxDepth: int, branchFactor: int) =
|
|
if currentDepth >= maxDepth:
|
|
return
|
|
|
|
for i in 0..<branchFactor:
|
|
var dep = PackageSpec(
|
|
name: fmt"dep-D{currentDepth}-B{i}",
|
|
version: "1.0.0",
|
|
dependencies: @[]
|
|
)
|
|
|
|
addDeps(dep, currentDepth + 1, maxDepth, max(1, branchFactor div 2))
|
|
pkg.dependencies.add(dep)
|
|
|
|
addDeps(result, 0, depth, branchFactor)
|
|
|
|
# ============================================================================
|
|
# Benchmark Execution
|
|
# ============================================================================
|
|
|
|
proc benchmarkResolution*(
|
|
name: string,
|
|
pkg: PackageSpec,
|
|
cache: ResolutionCache,
|
|
iterations: int = 10
|
|
): BenchmarkResult =
|
|
## Benchmark dependency resolution with and without cache
|
|
|
|
echo fmt"Benchmarking: {name}"
|
|
|
|
# Count total packages and dependencies
|
|
var packageCount = 1
|
|
var dependencyCount = 0
|
|
|
|
proc countDeps(p: PackageSpec) =
|
|
dependencyCount += p.dependencies.len
|
|
for dep in p.dependencies:
|
|
packageCount += 1
|
|
countDeps(dep)
|
|
|
|
countDeps(pkg)
|
|
|
|
echo fmt" Packages: {packageCount}, Dependencies: {dependencyCount}"
|
|
|
|
# Benchmark cold cache (first resolution)
|
|
cache.clear()
|
|
|
|
let coldStart = cpuTime()
|
|
for i in 0..<iterations:
|
|
cache.clear()
|
|
# TODO: Call actual resolver
|
|
# let graph = resolve(pkg)
|
|
discard
|
|
let coldEnd = cpuTime()
|
|
let coldTime = (coldEnd - coldStart) / iterations.float
|
|
|
|
echo fmt" Cold cache: {coldTime * 1000:.2f}ms"
|
|
|
|
# Benchmark warm cache (cached resolution)
|
|
let warmStart = cpuTime()
|
|
for i in 0..<iterations:
|
|
# TODO: Call actual resolver with cache
|
|
# let graph = resolve(pkg)
|
|
discard
|
|
let warmEnd = cpuTime()
|
|
let warmTime = (warmEnd - warmStart) / iterations.float
|
|
|
|
echo fmt" Warm cache: {warmTime * 1000:.2f}ms"
|
|
|
|
# Calculate metrics
|
|
let speedup = if warmTime > 0: coldTime / warmTime else: 0.0
|
|
let metrics = cache.getMetrics()
|
|
|
|
echo fmt" Speedup: {speedup:.2f}x"
|
|
echo fmt" Cache hit rate: {metrics.l1HitRate * 100:.2f}%"
|
|
echo ""
|
|
|
|
result = BenchmarkResult(
|
|
name: name,
|
|
packageCount: packageCount,
|
|
dependencyCount: dependencyCount,
|
|
coldTime: coldTime,
|
|
warmTime: warmTime,
|
|
cacheHitRate: metrics.l1HitRate,
|
|
speedup: speedup,
|
|
memoryUsed: 0 # TODO: Measure actual memory
|
|
)
|
|
|
|
# ============================================================================
|
|
# Benchmark Suites
|
|
# ============================================================================
|
|
|
|
proc runSimpleBenchmarks*(cache: ResolutionCache): seq[BenchmarkResult] =
|
|
## Run benchmarks for simple packages (10-20 dependencies)
|
|
|
|
echo "=" .repeat(60)
|
|
echo "SIMPLE PACKAGES (10-20 dependencies)"
|
|
echo "=" .repeat(60)
|
|
echo ""
|
|
|
|
result = @[]
|
|
|
|
# 10 dependencies
|
|
let pkg10 = generateSimplePackage("simple-10", 10)
|
|
result.add(benchmarkResolution("Simple 10 deps", pkg10, cache))
|
|
|
|
# 15 dependencies
|
|
let pkg15 = generateSimplePackage("simple-15", 15)
|
|
result.add(benchmarkResolution("Simple 15 deps", pkg15, cache))
|
|
|
|
# 20 dependencies
|
|
let pkg20 = generateSimplePackage("simple-20", 20)
|
|
result.add(benchmarkResolution("Simple 20 deps", pkg20, cache))
|
|
|
|
proc runComplexBenchmarks*(cache: ResolutionCache): seq[BenchmarkResult] =
|
|
## Run benchmarks for complex packages (50-100 dependencies)
|
|
|
|
echo "=" .repeat(60)
|
|
echo "COMPLEX PACKAGES (50-100 dependencies)"
|
|
echo "=" .repeat(60)
|
|
echo ""
|
|
|
|
result = @[]
|
|
|
|
# 50 dependencies
|
|
let pkg50 = generateComplexPackage("complex-50", 50)
|
|
result.add(benchmarkResolution("Complex 50 deps", pkg50, cache))
|
|
|
|
# 75 dependencies
|
|
let pkg75 = generateComplexPackage("complex-75", 75)
|
|
result.add(benchmarkResolution("Complex 75 deps", pkg75, cache))
|
|
|
|
# 100 dependencies
|
|
let pkg100 = generateComplexPackage("complex-100", 100)
|
|
result.add(benchmarkResolution("Complex 100 deps", pkg100, cache))
|
|
|
|
proc runMassiveBenchmarks*(cache: ResolutionCache): seq[BenchmarkResult] =
|
|
## Run benchmarks for massive packages (200+ dependencies)
|
|
|
|
echo "=" .repeat(60)
|
|
echo "MASSIVE PACKAGES (200+ dependencies)"
|
|
echo "=" .repeat(60)
|
|
echo ""
|
|
|
|
result = @[]
|
|
|
|
# 200 dependencies
|
|
let pkg200 = generateMassivePackage("massive-200", 200)
|
|
result.add(benchmarkResolution("Massive 200 deps", pkg200, cache))
|
|
|
|
# 300 dependencies
|
|
let pkg300 = generateMassivePackage("massive-300", 300)
|
|
result.add(benchmarkResolution("Massive 300 deps", pkg300, cache))
|
|
|
|
# 500 dependencies
|
|
let pkg500 = generateMassivePackage("massive-500", 500)
|
|
result.add(benchmarkResolution("Massive 500 deps", pkg500, cache))
|
|
|
|
# ============================================================================
|
|
# Results Reporting
|
|
# ============================================================================
|
|
|
|
proc printSummary*(results: seq[BenchmarkResult]) =
|
|
## Print benchmark summary table
|
|
|
|
echo ""
|
|
echo "=" .repeat(80)
|
|
echo "BENCHMARK SUMMARY"
|
|
echo "=" .repeat(80)
|
|
echo ""
|
|
|
|
echo fmt"{'Benchmark':<25} {'Pkgs':>6} {'Deps':>6} {'Cold':>10} {'Warm':>10} {'Speedup':>8} {'Hit%':>6}"
|
|
echo "-" .repeat(80)
|
|
|
|
for result in results:
|
|
echo fmt"{result.name:<25} {result.packageCount:>6} {result.dependencyCount:>6} " &
|
|
fmt"{result.coldTime * 1000:>9.2f}ms {result.warmTime * 1000:>9.2f}ms " &
|
|
fmt"{result.speedup:>7.2f}x {result.cacheHitRate * 100:>5.1f}%"
|
|
|
|
echo ""
|
|
|
|
# Calculate averages
|
|
var avgColdTime = 0.0
|
|
var avgWarmTime = 0.0
|
|
var avgSpeedup = 0.0
|
|
var avgHitRate = 0.0
|
|
|
|
for result in results:
|
|
avgColdTime += result.coldTime
|
|
avgWarmTime += result.warmTime
|
|
avgSpeedup += result.speedup
|
|
avgHitRate += result.cacheHitRate
|
|
|
|
let count = results.len.float
|
|
avgColdTime /= count
|
|
avgWarmTime /= count
|
|
avgSpeedup /= count
|
|
avgHitRate /= count
|
|
|
|
echo fmt"{'AVERAGE':<25} {'':>6} {'':>6} " &
|
|
fmt"{avgColdTime * 1000:>9.2f}ms {avgWarmTime * 1000:>9.2f}ms " &
|
|
fmt"{avgSpeedup:>7.2f}x {avgHitRate * 100:>5.1f}%"
|
|
echo ""
|
|
|
|
proc exportResults*(results: seq[BenchmarkResult], filename: string) =
|
|
## Export benchmark results to CSV
|
|
|
|
var csv = "Name,Packages,Dependencies,ColdTime(ms),WarmTime(ms),Speedup,HitRate(%)\n"
|
|
|
|
for result in results:
|
|
csv &= fmt"{result.name},{result.packageCount},{result.dependencyCount}," &
|
|
fmt"{result.coldTime * 1000:.2f},{result.warmTime * 1000:.2f}," &
|
|
fmt"{result.speedup:.2f},{result.cacheHitRate * 100:.2f}\n"
|
|
|
|
writeFile(filename, csv)
|
|
echo fmt"Results exported to: {filename}"
|
|
|
|
# ============================================================================
|
|
# Main Benchmark Runner
|
|
# ============================================================================
|
|
|
|
proc runAllBenchmarks*() =
|
|
## Run complete benchmark suite
|
|
|
|
echo ""
|
|
echo "╔" & "═".repeat(78) & "╗"
|
|
echo "║" & " ".repeat(20) & "NIP DEPENDENCY RESOLVER BENCHMARKS" & " ".repeat(24) & "║"
|
|
echo "╚" & "═".repeat(78) & "╝"
|
|
echo ""
|
|
|
|
let cas = newCASStorage("/tmp/benchmark-cas")
|
|
let cache = newResolutionCache(cas, l1Capacity = 1000)
|
|
|
|
var allResults: seq[BenchmarkResult] = @[]
|
|
|
|
# Run benchmark suites
|
|
allResults.add(runSimpleBenchmarks(cache))
|
|
allResults.add(runComplexBenchmarks(cache))
|
|
allResults.add(runMassiveBenchmarks(cache))
|
|
|
|
# Print summary
|
|
printSummary(allResults)
|
|
|
|
# Export results
|
|
exportResults(allResults, "/tmp/benchmark-results.csv")
|
|
|
|
echo ""
|
|
echo "Benchmark complete!"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Entry Point
|
|
# ============================================================================
|
|
|
|
when isMainModule:
|
|
runAllBenchmarks()
|