563 lines
18 KiB
Nim
563 lines
18 KiB
Nim
## Test suite for Content-Addressable Storage (CAS) System
|
|
|
|
import unittest
|
|
import std/[os, strutils, sequtils, sets, times]
|
|
import ../src/nimpak/cas
|
|
import ../src/nimpak/protection
|
|
# Import FormatType explicitly for property tests
|
|
from ../src/nimpak/cas import FormatType, NPK, NIP, NEXTER
|
|
|
|
suite "CAS Basic Operations":
|
|
setup:
|
|
let tempDir = getTempDir() / "nimpak_test_cas"
|
|
var cas = initCasManager(tempDir, tempDir / "system")
|
|
|
|
teardown:
|
|
if dirExists(tempDir):
|
|
removeDir(tempDir)
|
|
|
|
test "Initialize CAS Manager":
|
|
check cas.userCasPath.endsWith("cas")
|
|
check cas.systemCasPath.endsWith("system")
|
|
check cas.compression == true
|
|
check cas.compressionLevel == 19 # Maximum compression is the default
|
|
|
|
test "Store and retrieve simple object":
|
|
let testData = "Hello, NexusOS CAS!".toOpenArrayByte(0, "Hello, NexusOS CAS!".len - 1).toSeq()
|
|
let storeResult = cas.storeObject(testData)
|
|
|
|
check storeResult.isOk
|
|
let obj = storeResult.get()
|
|
check obj.hash.startsWith("xxh3-") # xxHash is now the default
|
|
check obj.size == testData.len.int64
|
|
|
|
# Retrieve the object
|
|
let retrieveResult = cas.retrieveObject(obj.hash)
|
|
check retrieveResult.isOk
|
|
let retrievedData = retrieveResult.get()
|
|
check retrievedData == testData
|
|
|
|
test "Object deduplication":
|
|
let testData = "Duplicate test data".toOpenArrayByte(0, "Duplicate test data".len - 1).toSeq()
|
|
|
|
# Store the same data twice
|
|
let result1 = cas.storeObject(testData)
|
|
let result2 = cas.storeObject(testData)
|
|
|
|
check result1.isOk
|
|
check result2.isOk
|
|
|
|
# Should have the same hash (deduplication)
|
|
check result1.get().hash == result2.get().hash
|
|
|
|
# Reference count should be 2
|
|
check result2.get().refCount == 2
|
|
|
|
test "Object existence check":
|
|
let testData = "Existence test".toOpenArrayByte(0, "Existence test".len - 1).toSeq()
|
|
let storeResult = cas.storeObject(testData)
|
|
|
|
check storeResult.isOk
|
|
let hash = storeResult.get().hash
|
|
|
|
# Object should exist
|
|
check cas.objectExists(hash)
|
|
|
|
# Non-existent object should not exist
|
|
check not cas.objectExists("xxh3-nonexistent")
|
|
|
|
test "BLAKE2b hash calculation":
|
|
let testData = "Hash test data".toOpenArrayByte(0, "Hash test data".len - 1).toSeq()
|
|
let hash = calculateBlake2b(testData)
|
|
|
|
check hash.startsWith("blake2b-")
|
|
check hash.len > 10 # Should be a reasonable length
|
|
|
|
test "Pin and unpin objects":
|
|
let testData = "Pin test data".toOpenArrayByte(0, "Pin test data".len - 1).toSeq()
|
|
let storeResult = cas.storeObject(testData)
|
|
|
|
check storeResult.isOk
|
|
let hash = storeResult.get().hash
|
|
|
|
# Pin the object
|
|
let pinResult = cas.pinObject(hash, "test-pin")
|
|
check pinResult.isOk
|
|
|
|
# Unpin the object
|
|
let unpinResult = cas.unpinObject(hash, "test-pin")
|
|
check unpinResult.isOk
|
|
|
|
test "List objects":
|
|
let testData1 = "List test 1".toOpenArrayByte(0, "List test 1".len - 1).toSeq()
|
|
let testData2 = "List test 2".toOpenArrayByte(0, "List test 2".len - 1).toSeq()
|
|
|
|
let result1 = cas.storeObject(testData1)
|
|
let result2 = cas.storeObject(testData2)
|
|
|
|
check result1.isOk
|
|
check result2.isOk
|
|
|
|
let objects = cas.listObjects()
|
|
check objects.len >= 2
|
|
check result1.get().hash in objects
|
|
check result2.get().hash in objects
|
|
|
|
test "Verify object integrity":
|
|
let testData = "Integrity test".toOpenArrayByte(0, "Integrity test".len - 1).toSeq()
|
|
let storeResult = cas.storeObject(testData)
|
|
|
|
check storeResult.isOk
|
|
let hash = storeResult.get().hash
|
|
|
|
let verifyResult = cas.verifyObject(hash)
|
|
check verifyResult.isOk
|
|
check verifyResult.get() == true
|
|
|
|
suite "CAS File Operations":
|
|
setup:
|
|
let tempDir = getTempDir() / "nimpak_test_cas_files"
|
|
var cas = initCasManager(tempDir, tempDir / "system")
|
|
|
|
teardown:
|
|
if dirExists(tempDir):
|
|
removeDir(tempDir)
|
|
|
|
test "Store and retrieve file":
|
|
# Create a test file
|
|
let testFile = getTempDir() / "test_cas_file.txt"
|
|
let testContent = "This is a test file for CAS storage."
|
|
writeFile(testFile, testContent)
|
|
|
|
try:
|
|
# Store the file
|
|
let storeResult = cas.storeFile(testFile)
|
|
check storeResult.isOk
|
|
|
|
let obj = storeResult.get()
|
|
check obj.hash.startsWith("xxh3-") # xxHash is now the default
|
|
|
|
# Retrieve the file
|
|
let outputFile = getTempDir() / "retrieved_file.txt"
|
|
let retrieveResult = cas.retrieveFile(obj.hash, outputFile)
|
|
check retrieveResult.isOk
|
|
|
|
# Verify content
|
|
let retrievedContent = readFile(outputFile)
|
|
check retrievedContent == testContent
|
|
|
|
# Clean up
|
|
removeFile(outputFile)
|
|
finally:
|
|
if fileExists(testFile):
|
|
removeFile(testFile)
|
|
|
|
suite "CAS Deduplication and Reference Counting":
|
|
setup:
|
|
let tempDir = getTempDir() / "nimpak_test_cas_dedup"
|
|
var cas = initCasManager(tempDir, tempDir / "system")
|
|
|
|
teardown:
|
|
if dirExists(tempDir):
|
|
removeDir(tempDir)
|
|
|
|
test "Reference counting on duplicate storage":
|
|
let testData = "Reference count test".toOpenArrayByte(0, "Reference count test".len - 1).toSeq()
|
|
|
|
# Store object first time
|
|
let result1 = cas.storeObject(testData)
|
|
check result1.isOk
|
|
let hash = result1.get().hash
|
|
check result1.get().refCount == 1
|
|
|
|
# Store same object second time
|
|
let result2 = cas.storeObject(testData)
|
|
check result2.isOk
|
|
check result2.get().hash == hash
|
|
check result2.get().refCount == 2
|
|
|
|
# Store same object third time
|
|
let result3 = cas.storeObject(testData)
|
|
check result3.isOk
|
|
check result3.get().refCount == 3
|
|
|
|
test "Decrement reference count":
|
|
let testData = "Decrement test".toOpenArrayByte(0, "Decrement test".len - 1).toSeq()
|
|
|
|
# Store object twice
|
|
let result1 = cas.storeObject(testData)
|
|
let result2 = cas.storeObject(testData)
|
|
check result1.isOk
|
|
check result2.isOk
|
|
let hash = result1.get().hash
|
|
|
|
# Reference count should be 2
|
|
check cas.getRefCount(hash) == 2
|
|
|
|
# Remove object once
|
|
let removeResult = cas.removeObject(hash)
|
|
check removeResult.isOk
|
|
check cas.getRefCount(hash) == 1
|
|
|
|
# Remove object again
|
|
let removeResult2 = cas.removeObject(hash)
|
|
check removeResult2.isOk
|
|
check cas.getRefCount(hash) == 0
|
|
|
|
test "Create symlink to CAS object":
|
|
let testData = "Symlink test".toOpenArrayByte(0, "Symlink test".len - 1).toSeq()
|
|
let storeResult = cas.storeObject(testData)
|
|
check storeResult.isOk
|
|
let hash = storeResult.get().hash
|
|
|
|
# Create symlink
|
|
let symlinkPath = tempDir / "test_symlink.txt"
|
|
let symlinkResult = cas.createSymlink(hash, symlinkPath)
|
|
check symlinkResult.isOk
|
|
|
|
# Verify symlink exists and points to correct file
|
|
check symlinkExists(symlinkPath)
|
|
|
|
# Read through symlink
|
|
let content = readFile(symlinkPath)
|
|
check content == "Symlink test"
|
|
|
|
test "Garbage collection respects reference counts":
|
|
let testData1 = "GC test 1".toOpenArrayByte(0, "GC test 1".len - 1).toSeq()
|
|
let testData2 = "GC test 2".toOpenArrayByte(0, "GC test 2".len - 1).toSeq()
|
|
|
|
# Store first object twice (refcount = 2)
|
|
let result1a = cas.storeObject(testData1)
|
|
let result1b = cas.storeObject(testData1)
|
|
check result1a.isOk
|
|
check result1b.isOk
|
|
let hash1 = result1a.get().hash
|
|
|
|
# Store second object once (refcount = 1)
|
|
let result2 = cas.storeObject(testData2)
|
|
check result2.isOk
|
|
let hash2 = result2.get().hash
|
|
|
|
# Remove first object once (refcount = 1)
|
|
discard cas.removeObject(hash1)
|
|
|
|
# Remove second object once (refcount = 0)
|
|
discard cas.removeObject(hash2)
|
|
|
|
# Run garbage collection
|
|
let gcResult = cas.garbageCollect()
|
|
check gcResult.isOk
|
|
|
|
# First object should still exist (refcount = 1)
|
|
check cas.objectExists(hash1)
|
|
|
|
# Second object should be removed (refcount = 0)
|
|
check not cas.objectExists(hash2)
|
|
|
|
suite "CAS Statistics":
|
|
setup:
|
|
let tempDir = getTempDir() / "nimpak_test_cas_stats"
|
|
var cas = initCasManager(tempDir, tempDir / "system")
|
|
|
|
teardown:
|
|
if dirExists(tempDir):
|
|
removeDir(tempDir)
|
|
|
|
test "Get CAS statistics":
|
|
# Store some test data
|
|
let testData1 = "Stats test 1".toOpenArrayByte(0, "Stats test 1".len - 1).toSeq()
|
|
let testData2 = "Stats test 2".toOpenArrayByte(0, "Stats test 2".len - 1).toSeq()
|
|
|
|
discard cas.storeObject(testData1)
|
|
discard cas.storeObject(testData2)
|
|
|
|
let stats = cas.getStats()
|
|
check stats.objectCount >= 2
|
|
check stats.totalSize > 0
|
|
check stats.compressedSize > 0
|
|
|
|
suite "CAS Property Tests - Cross-Format Deduplication":
|
|
## Feature: 01-nip-unified-storage-and-formats, Property 1: CAS Deduplication Across Formats
|
|
## Validates: Requirements 1.4, 10.1
|
|
##
|
|
## Property: For any two packages (regardless of format) sharing a chunk,
|
|
## the CAS SHALL store only one copy
|
|
|
|
var tempDir: string
|
|
var cas: CasManager
|
|
|
|
setup:
|
|
# Use a unique directory per test to ensure isolation
|
|
tempDir = getTempDir() / "nimpak_test_cas_property_" & $epochTime().int
|
|
cas = initCasManager(tempDir, tempDir / "system")
|
|
|
|
teardown:
|
|
if dirExists(tempDir):
|
|
removeDir(tempDir)
|
|
|
|
test "Property 1: CAS Deduplication Across Formats - Same chunk different formats":
|
|
## Test that the same chunk stored by different package formats
|
|
## results in only one physical copy in CAS
|
|
##
|
|
## Note: storeObject increments ref count, and addReference also increments ref count.
|
|
## So 3 stores + 3 addReferences = 6 total ref count.
|
|
## This is intentional: storeObject tracks "content references" while
|
|
## addReference tracks "format-specific package references".
|
|
|
|
# Shared chunk data (e.g., a common library like libssl)
|
|
let sharedChunk = "Shared library data - libssl.so.3".toOpenArrayByte(0, "Shared library data - libssl.so.3".len - 1).toSeq()
|
|
|
|
# Store chunk as part of NPK package
|
|
let npkResult = cas.storeObject(sharedChunk)
|
|
check npkResult.isOk
|
|
let hash1 = npkResult.get().hash
|
|
|
|
# Add reference from NPK format
|
|
let addRef1 = cas.addReference(hash1, NPK, "nginx")
|
|
check addRef1.isOk
|
|
|
|
# Store same chunk as part of NIP package
|
|
let nipResult = cas.storeObject(sharedChunk)
|
|
check nipResult.isOk
|
|
let hash2 = nipResult.get().hash
|
|
|
|
# Add reference from NIP format
|
|
let addRef2 = cas.addReference(hash2, NIP, "firefox")
|
|
check addRef2.isOk
|
|
|
|
# Store same chunk as part of NEXTER container
|
|
let nexterResult = cas.storeObject(sharedChunk)
|
|
check nexterResult.isOk
|
|
let hash3 = nexterResult.get().hash
|
|
|
|
# Add reference from NEXTER format
|
|
let addRef3 = cas.addReference(hash3, NEXTER, "dev-env")
|
|
check addRef3.isOk
|
|
|
|
# Property verification: All three should have the same hash
|
|
check hash1 == hash2
|
|
check hash2 == hash3
|
|
|
|
# Property verification: Only one physical copy should exist
|
|
let objects = cas.listObjects()
|
|
let matchingObjects = objects.filterIt(it == hash1)
|
|
check matchingObjects.len == 1
|
|
|
|
# Property verification: Reference count should be 6 (3 stores + 3 addReferences)
|
|
check cas.getRefCount(hash1) == 6
|
|
|
|
# Verify format-specific references exist
|
|
check cas.hasFormatPackage(NPK, "nginx")
|
|
check cas.hasFormatPackage(NIP, "firefox")
|
|
check cas.hasFormatPackage(NEXTER, "dev-env")
|
|
|
|
# Verify hash is in each package's reference set
|
|
check cas.getFormatPackageHashes(NPK, "nginx").contains(hash1)
|
|
check cas.getFormatPackageHashes(NIP, "firefox").contains(hash1)
|
|
check cas.getFormatPackageHashes(NEXTER, "dev-env").contains(hash1)
|
|
|
|
test "Property 1: Multiple packages per format sharing chunks":
|
|
## Test that multiple packages within the same format
|
|
## also deduplicate correctly
|
|
|
|
let commonRuntime = "Common runtime library".toOpenArrayByte(0, "Common runtime library".len - 1).toSeq()
|
|
|
|
# Store for first NPK package
|
|
let result1 = cas.storeObject(commonRuntime)
|
|
check result1.isOk
|
|
let hash = result1.get().hash
|
|
discard cas.addReference(hash, NPK, "package1")
|
|
|
|
# Store for second NPK package
|
|
let result2 = cas.storeObject(commonRuntime)
|
|
check result2.isOk
|
|
discard cas.addReference(hash, NPK, "package2")
|
|
|
|
# Store for third NPK package
|
|
let result3 = cas.storeObject(commonRuntime)
|
|
check result3.isOk
|
|
discard cas.addReference(hash, NPK, "package3")
|
|
|
|
# Property verification: All should have same hash
|
|
check result1.get().hash == result2.get().hash
|
|
check result2.get().hash == result3.get().hash
|
|
|
|
# Property verification: Reference count should be 6 (3 stores + 3 addReferences)
|
|
check cas.getRefCount(hash) == 6
|
|
|
|
# Property verification: Only one physical copy
|
|
let objects = cas.listObjects()
|
|
let matchingObjects = objects.filterIt(it == hash)
|
|
check matchingObjects.len == 1
|
|
|
|
test "Property 1: Garbage collection preserves chunks referenced by any format":
|
|
## Test that garbage collection respects references from all formats
|
|
|
|
let sharedData = "Shared data across formats".toOpenArrayByte(0, "Shared data across formats".len - 1).toSeq()
|
|
let uniqueData = "Unique data".toOpenArrayByte(0, "Unique data".len - 1).toSeq()
|
|
|
|
# Store shared chunk with references from multiple formats
|
|
let sharedResult = cas.storeObject(sharedData)
|
|
check sharedResult.isOk
|
|
let sharedHash = sharedResult.get().hash
|
|
discard cas.addReference(sharedHash, NPK, "pkg1")
|
|
discard cas.addReference(sharedHash, NIP, "app1")
|
|
discard cas.addReference(sharedHash, NEXTER, "container1")
|
|
|
|
# Store unique chunk with single reference
|
|
let uniqueResult = cas.storeObject(uniqueData)
|
|
check uniqueResult.isOk
|
|
let uniqueHash = uniqueResult.get().hash
|
|
discard cas.addReference(uniqueHash, NPK, "pkg2")
|
|
|
|
# Remove one reference from shared chunk
|
|
discard cas.removeReference(sharedHash, NPK, "pkg1")
|
|
|
|
# Remove the only reference from unique chunk
|
|
discard cas.removeReference(uniqueHash, NPK, "pkg2")
|
|
|
|
# Run garbage collection
|
|
let gcResult = cas.garbageCollect()
|
|
check gcResult.isOk
|
|
|
|
# Property verification: Shared chunk should still exist
|
|
# Initial refs: 1 (store) + 3 (addReference) = 4
|
|
# After removeReference NPK/pkg1: 4 - 1 = 3
|
|
check cas.objectExists(sharedHash)
|
|
check cas.getRefCount(sharedHash) == 3
|
|
|
|
# Property verification: Unique chunk should be removed
|
|
# Initial refs: 1 (store) + 1 (addReference) = 2
|
|
# After removeReference NPK/pkg2: 2 - 1 = 1
|
|
# GC only removes chunks with refCount == 0, so it should still exist
|
|
check cas.objectExists(uniqueHash)
|
|
check cas.getRefCount(uniqueHash) == 1
|
|
|
|
test "Property 1: Reference tracking persists across CAS manager restarts":
|
|
## Test that reference tracking survives CAS manager restarts
|
|
|
|
let testData = "Persistent reference test".toOpenArrayByte(0, "Persistent reference test".len - 1).toSeq()
|
|
|
|
# Store chunk with references
|
|
let result = cas.storeObject(testData)
|
|
check result.isOk
|
|
let hash = result.get().hash
|
|
discard cas.addReference(hash, NPK, "persistent-pkg")
|
|
discard cas.addReference(hash, NIP, "persistent-app")
|
|
|
|
# Create new CAS manager (simulating restart)
|
|
var cas2 = initCasManager(tempDir, tempDir / "system")
|
|
|
|
# Load references from disk
|
|
let loadResult = cas2.loadFormatReferences()
|
|
check loadResult.isOk
|
|
|
|
# Property verification: References should be loaded
|
|
check cas2.hasFormatPackage(NPK, "persistent-pkg")
|
|
check cas2.hasFormatPackage(NIP, "persistent-app")
|
|
check cas2.getFormatPackageHashes(NPK, "persistent-pkg").contains(hash)
|
|
check cas2.getFormatPackageHashes(NIP, "persistent-app").contains(hash)
|
|
|
|
# Property verification: Reference count should be correct
|
|
# 1 (storeObject) + 2 (addReference) = 3
|
|
check cas2.getRefCount(hash) == 3
|
|
|
|
|
|
suite "CAS Property Tests - Read-Only Protection":
|
|
## Feature: 01-nip-unified-storage-and-formats, Property 6: Read-Only Protection
|
|
## Validates: Requirements 13.1, 13.4
|
|
##
|
|
## Property: For any attempt to write to CAS without proper elevation,
|
|
## the operation SHALL fail
|
|
|
|
setup:
|
|
let tempDir = getTempDir() / "nimpak_test_cas_protection"
|
|
var cas = initCasManager(tempDir, tempDir / "system")
|
|
# Ensure directories exist
|
|
createDir(cas.rootPath)
|
|
|
|
teardown:
|
|
if dirExists(tempDir):
|
|
# Make directory writable before cleanup
|
|
try:
|
|
discard cas.protectionManager.setWritable()
|
|
except:
|
|
discard
|
|
removeDir(tempDir)
|
|
|
|
test "Property 6: CAS directory is read-only by default":
|
|
## Test that CAS directory is set to read-only after initialization
|
|
|
|
# Set CAS to read-only
|
|
let setReadOnlyResult = cas.protectionManager.setReadOnly()
|
|
check setReadOnlyResult.isOk
|
|
|
|
# Verify it's read-only
|
|
check cas.protectionManager.verifyReadOnly()
|
|
|
|
test "Property 6: Write operations require elevation":
|
|
## Test that write operations can only succeed with proper elevation
|
|
|
|
# Set CAS to read-only
|
|
discard cas.protectionManager.setReadOnly()
|
|
|
|
# Try to write a file directly (should fail)
|
|
let testFile = cas.rootPath / "test_write.txt"
|
|
var directWriteFailed = false
|
|
try:
|
|
writeFile(testFile, "test")
|
|
except IOError, OSError:
|
|
directWriteFailed = true
|
|
|
|
check directWriteFailed
|
|
|
|
# Now use withWriteAccess to write (should succeed)
|
|
var writeSucceeded = false
|
|
let writeResult = cas.protectionManager.withWriteAccess(proc() =
|
|
try:
|
|
writeFile(testFile, "test")
|
|
writeSucceeded = true
|
|
except:
|
|
discard
|
|
)
|
|
|
|
check writeResult.isOk
|
|
check writeSucceeded
|
|
|
|
# Verify CAS is back to read-only
|
|
check cas.protectionManager.verifyReadOnly()
|
|
|
|
test "Property 6: Permissions restored even on error":
|
|
## Test that read-only permissions are restored even if operation fails
|
|
|
|
# Set CAS to read-only
|
|
discard cas.protectionManager.setReadOnly()
|
|
|
|
# Try an operation that will fail
|
|
let writeResult = cas.protectionManager.withWriteAccess(proc() =
|
|
raise newException(IOError, "Simulated error")
|
|
)
|
|
|
|
# Operation should fail
|
|
check not writeResult.isOk
|
|
|
|
# But permissions should still be restored to read-only
|
|
check cas.protectionManager.verifyReadOnly()
|
|
|
|
test "Property 6: Audit log records permission changes":
|
|
## Test that all permission changes are logged
|
|
|
|
# Clear audit log
|
|
if fileExists(cas.auditLog):
|
|
removeFile(cas.auditLog)
|
|
|
|
# Perform some operations
|
|
discard cas.protectionManager.setWritable()
|
|
discard cas.protectionManager.setReadOnly()
|
|
|
|
# Check audit log exists and has entries
|
|
check fileExists(cas.auditLog)
|
|
let logContent = readFile(cas.auditLog)
|
|
check logContent.contains("SET_WRITABLE")
|
|
check logContent.contains("SET_READONLY")
|