289 lines
9.6 KiB
Nim
289 lines
9.6 KiB
Nim
## Performance Benchmarks for Nippels
|
||
## Task 12.5: Benchmark Nippel operations against performance targets
|
||
##
|
||
## Performance Targets (Requirements P1, P2):
|
||
## - Nippel creation: < 100ms
|
||
## - Nippel activation: < 50ms
|
||
## - CAS lookup: < 1ms
|
||
## - Merkle tree update: < 10ms
|
||
|
||
import std/[unittest, times, os, strutils, options]
|
||
import ../src/nimpak/nippels
|
||
import ../src/nimpak/cas
|
||
import ../src/nimpak/merkle_tree
|
||
import ../src/nimpak/nippel_types
|
||
import ../src/nimpak/utils/resultutils
|
||
|
||
suite "Nippels Performance Benchmarks":
|
||
var testDir: string
|
||
var manager: NippelManager
|
||
|
||
setup:
|
||
testDir = getTempDir() / "nippels_perf_test_" & $epochTime().int
|
||
createDir(testDir)
|
||
manager = newNippelManager(testDir)
|
||
|
||
teardown:
|
||
if dirExists(testDir):
|
||
removeDir(testDir)
|
||
|
||
test "Benchmark: Nippel creation time (target: < 100ms)":
|
||
## Requirement P1.1: Nippel creation within 100 milliseconds
|
||
let startTime = cpuTime()
|
||
|
||
let result = manager.createNippel("perf-test", Homestation)
|
||
|
||
let endTime = cpuTime()
|
||
let elapsedMs = (endTime - startTime) * 1000.0
|
||
|
||
check result.isOk
|
||
echo " ✓ Nippel creation time: ", elapsedMs.formatFloat(ffDecimal, 2), " ms"
|
||
|
||
if elapsedMs < 100.0:
|
||
echo " ✅ PASS: Under 100ms target"
|
||
else:
|
||
echo " ⚠️ WARNING: Exceeded 100ms target (", elapsedMs.formatFloat(ffDecimal, 2), " ms)"
|
||
|
||
# Test passes if creation succeeded, but we report performance
|
||
check result.isOk
|
||
|
||
test "Benchmark: Nippel activation time (target: < 50ms)":
|
||
## Requirement P1.2: Nippel activation within 50 milliseconds
|
||
# Create a Nippel first
|
||
let createResult = manager.createNippel("activation-test", Homestation)
|
||
check createResult.isOk
|
||
|
||
# Benchmark activation
|
||
let startTime = cpuTime()
|
||
|
||
let result = manager.activateNippel("activation-test")
|
||
|
||
let endTime = cpuTime()
|
||
let elapsedMs = (endTime - startTime) * 1000.0
|
||
|
||
check result.isOk
|
||
echo " ✓ Nippel activation time: ", elapsedMs.formatFloat(ffDecimal, 2), " ms"
|
||
|
||
if elapsedMs < 50.0:
|
||
echo " ✅ PASS: Under 50ms target"
|
||
else:
|
||
echo " ⚠️ WARNING: Exceeded 50ms target (", elapsedMs.formatFloat(ffDecimal, 2), " ms)"
|
||
|
||
# Clean up
|
||
discard manager.deactivateNippel("activation-test")
|
||
|
||
test "Benchmark: CAS lookup time (target: < 1ms)":
|
||
## Requirement P1.4: CAS lookup within 1 millisecond
|
||
# Use test directory for both user and system paths to avoid permission issues
|
||
var cas = newCasManager(testDir / "cas", testDir / "cas")
|
||
|
||
# Store a test object
|
||
let testData = "test data for CAS lookup benchmark"
|
||
var testBytes: seq[byte] = @[]
|
||
for c in testData:
|
||
testBytes.add(c.byte)
|
||
let storeResult = cas.storeObject(testBytes)
|
||
if storeResult.isOk:
|
||
let hash = storeResult.get().hash
|
||
|
||
# Clear cache to ensure we're measuring disk access
|
||
cas.clearCache()
|
||
|
||
# Benchmark lookup (average over multiple lookups)
|
||
const numLookups = 100
|
||
var totalTime = 0.0
|
||
|
||
for i in 0..<numLookups:
|
||
let startTime = cpuTime()
|
||
let retrieveResult = cas.retrieveObject(hash)
|
||
let endTime = cpuTime()
|
||
|
||
check retrieveResult.isOk
|
||
totalTime += (endTime - startTime)
|
||
|
||
let avgTimeMs = (totalTime / numLookups.float) * 1000.0
|
||
|
||
echo " ✓ Average CAS lookup time (", numLookups, " lookups): ", avgTimeMs.formatFloat(ffDecimal, 3), " ms"
|
||
|
||
if avgTimeMs < 1.0:
|
||
echo " ✅ PASS: Under 1ms target"
|
||
else:
|
||
echo " ⚠️ WARNING: Exceeded 1ms target (", avgTimeMs.formatFloat(ffDecimal, 3), " ms)"
|
||
else:
|
||
echo " ⚠️ SKIPPED: CAS store failed: ", storeResult.error.msg
|
||
skip()
|
||
|
||
test "Benchmark: CAS cache hit performance":
|
||
## Task 12.2: Verify cache improves performance
|
||
# Use test directory for both user and system paths to avoid permission issues
|
||
var cas = newCasManager(testDir / "cas", testDir / "cas")
|
||
|
||
# Store a test object
|
||
let testData = "test data for cache benchmark"
|
||
var testBytes: seq[byte] = @[]
|
||
for c in testData:
|
||
testBytes.add(c.byte)
|
||
let storeResult = cas.storeObject(testBytes)
|
||
if storeResult.isOk:
|
||
let hash = storeResult.get().hash
|
||
|
||
# First lookup (cache miss)
|
||
let startMiss = cpuTime()
|
||
let missResult = cas.retrieveObject(hash)
|
||
let endMiss = cpuTime()
|
||
let missTimeMs = (endMiss - startMiss) * 1000.0
|
||
|
||
check missResult.isOk
|
||
|
||
# Second lookup (cache hit)
|
||
let startHit = cpuTime()
|
||
let hitResult = cas.retrieveObject(hash)
|
||
let endHit = cpuTime()
|
||
let hitTimeMs = (endHit - startHit) * 1000.0
|
||
|
||
check hitResult.isOk
|
||
|
||
echo " ✓ Cache miss time: ", missTimeMs.formatFloat(ffDecimal, 3), " ms"
|
||
echo " ✓ Cache hit time: ", hitTimeMs.formatFloat(ffDecimal, 3), " ms"
|
||
|
||
if hitTimeMs < missTimeMs:
|
||
let speedup = missTimeMs / hitTimeMs
|
||
echo " ✅ Cache speedup: ", speedup.formatFloat(ffDecimal, 2), "x faster"
|
||
else:
|
||
echo " ⚠️ WARNING: Cache not improving performance"
|
||
|
||
# Check cache statistics
|
||
let stats = cas.getStats()
|
||
echo " ✓ Cache hit rate: ", (stats.cacheHitRate * 100.0).formatFloat(ffDecimal, 1), "%"
|
||
else:
|
||
echo " ⚠️ SKIPPED: CAS store failed: ", storeResult.error.msg
|
||
skip()
|
||
|
||
test "Benchmark: Merkle tree update time (target: < 10ms)":
|
||
## Requirement P2.2: Merkle tree update within 10 milliseconds
|
||
# Create initial tree with some files
|
||
var files: seq[FileEntry] = @[]
|
||
for i in 0..<10:
|
||
files.add(FileEntry(
|
||
path: "file" & $i & ".txt",
|
||
hash: "xxh3-" & $i,
|
||
size: 1024
|
||
))
|
||
|
||
let treeResult = buildTreeFromFiles(files, "xxh3")
|
||
check treeResult.isOk
|
||
|
||
var tree = treeResult.get()
|
||
|
||
# Benchmark update
|
||
let change = FileChange(
|
||
path: "file5.txt",
|
||
changeType: Modified,
|
||
newHash: some("xxh3-modified"),
|
||
newSize: some(2048'i64)
|
||
)
|
||
|
||
let startTime = cpuTime()
|
||
|
||
let updateResult = applyChanges(tree, @[change])
|
||
|
||
let endTime = cpuTime()
|
||
let elapsedMs = (endTime - startTime) * 1000.0
|
||
|
||
check updateResult.isOk
|
||
echo " ✓ Merkle tree update time: ", elapsedMs.formatFloat(ffDecimal, 2), " ms"
|
||
|
||
if elapsedMs < 10.0:
|
||
echo " ✅ PASS: Under 10ms target"
|
||
else:
|
||
echo " ⚠️ WARNING: Exceeded 10ms target (", elapsedMs.formatFloat(ffDecimal, 2), " ms)"
|
||
|
||
test "Benchmark: Merkle tree verification time":
|
||
## Requirement P2.4: Merkle tree verification performance
|
||
# Create a larger tree for verification benchmark
|
||
var files: seq[FileEntry] = @[]
|
||
for i in 0..<100:
|
||
files.add(FileEntry(
|
||
path: "file" & $i & ".txt",
|
||
hash: "xxh3-" & $i,
|
||
size: 1024
|
||
))
|
||
|
||
let treeResult = buildTreeFromFiles(files, "xxh3")
|
||
check treeResult.isOk
|
||
|
||
let tree = treeResult.get()
|
||
|
||
# Benchmark verification
|
||
let startTime = cpuTime()
|
||
|
||
let verifyResult = verifyTree(tree)
|
||
|
||
let endTime = cpuTime()
|
||
let elapsedMs = (endTime - startTime) * 1000.0
|
||
|
||
check verifyResult.isOk
|
||
check verifyResult.get() == true
|
||
|
||
echo " ✓ Merkle tree verification time (100 files): ", elapsedMs.formatFloat(ffDecimal, 2), " ms"
|
||
echo " ✓ Verification rate: ", (100.0 / elapsedMs * 1000.0).formatFloat(ffDecimal, 0), " files/second"
|
||
|
||
test "Benchmark: Parallel merkle tree verification":
|
||
## Task 12.4: Verify parallel operations improve performance
|
||
# Create a larger tree
|
||
var files: seq[FileEntry] = @[]
|
||
for i in 0..<1000:
|
||
files.add(FileEntry(
|
||
path: "file" & $i & ".txt",
|
||
hash: "xxh3-" & $i,
|
||
size: 1024
|
||
))
|
||
|
||
let treeResult = buildTreeFromFiles(files, "xxh3")
|
||
check treeResult.isOk
|
||
|
||
let tree = treeResult.get()
|
||
|
||
# Sequential verification
|
||
let startSeq = cpuTime()
|
||
let seqResult = verifyTree(tree)
|
||
let endSeq = cpuTime()
|
||
let seqTimeMs = (endSeq - startSeq) * 1000.0
|
||
|
||
check seqResult.isOk
|
||
|
||
# Parallel verification
|
||
let startPar = cpuTime()
|
||
let parResult = verifyTreeParallel(tree)
|
||
let endPar = cpuTime()
|
||
let parTimeMs = (endPar - startPar) * 1000.0
|
||
|
||
check parResult.isOk
|
||
|
||
echo " ✓ Sequential verification time (1000 files): ", seqTimeMs.formatFloat(ffDecimal, 2), " ms"
|
||
echo " ✓ Parallel verification time (1000 files): ", parTimeMs.formatFloat(ffDecimal, 2), " ms"
|
||
|
||
if parTimeMs < seqTimeMs:
|
||
let speedup = seqTimeMs / parTimeMs
|
||
echo " ✅ Parallel speedup: ", speedup.formatFloat(ffDecimal, 2), "x faster"
|
||
else:
|
||
echo " ℹ️ Note: Parallel verification not faster (overhead may exceed benefit for this size)"
|
||
|
||
test "Performance Summary":
|
||
## Summary of all performance metrics
|
||
echo ""
|
||
echo "═══════════════════════════════════════════════════════════"
|
||
echo "Performance Targets Summary:"
|
||
echo "═══════════════════════════════════════════════════════════"
|
||
echo " Nippel creation: < 100ms (Requirement P1.1)"
|
||
echo " Nippel activation: < 50ms (Requirement P1.2)"
|
||
echo " CAS lookup: < 1ms (Requirement P1.4)"
|
||
echo " Merkle tree update: < 10ms (Requirement P2.2)"
|
||
echo "═══════════════════════════════════════════════════════════"
|
||
echo ""
|
||
echo "Run individual tests above to see actual performance metrics."
|
||
echo ""
|
||
|
||
# This test always passes - it's just a summary
|
||
check true
|