# SPDX-License-Identifier: LSL-1.0 # Copyright (c) 2026 Markus Maiwald # Stewardship: Self Sovereign Society Foundation # # This file is part of the Nexus Sovereign Core. # See legal/LICENSE_SOVEREIGN.md for license terms. ## NPK Package Format Handler ## ## This module implements the native .npk.zst package format with KDL metadata ## and provides conversion capabilities from grafted packages. It handles ## package creation, validation, and integrity checking with digital signature ## support for package verification. ## ## Package Format: .npk.zst (Nexus Package, Zstandard compressed) ## - Tar archives compressed with zstd --fast ## - KDL metadata for human-readable configuration ## - BLAKE3 integrity verification (future-ready) ## - Ed25519 digital signatures ## - Content-addressable storage integration import std/[os, json, times, strutils, sequtils, tables, options, osproc, strformat, algorithm] import ./types_fixed import ./formats import ./cas except Result, VoidResult, ok, err, ChunkRef import ./grafting # KDL parsing will be added when kdl library is available # For now, we'll use JSON as intermediate format and generate KDL strings type NpkError* = object of NimPakError packageName*: string ValidationResult* = object valid*: bool errors*: seq[ValidationError] warnings*: seq[string] NpkArchiveFormat* = enum ## Archive format for NPK packages NpkZst, ## .npk.zst - Zstandard compressed (default) NpkTar ## .npk.tar - Uncompressed (for debugging) # ============================================================================= # NPK Package Creation # ============================================================================= proc createNpkPackage*(fragment: Fragment, sourceDir: string, cas: var CasManager): Result[NpkPackage, NpkError] = ## Create NPK package from Fragment definition and source directory with CAS integration ## Files are stored in content-addressable storage for deduplication and integrity try: var files: seq[PackageFile] = @[] var totalSize: int64 = 0 # Scan source directory and create file entries with CAS storage for filePath in walkDirRec(sourceDir): let relativePath = filePath.relativePath(sourceDir) let info = getFileInfo(filePath) # Store file in CAS and get object metadata let storeResult = cas.storeFile(filePath) if not storeResult.isOk: return err[NpkPackage, NpkError](NpkError( code: CasGeneralError, msg: "Failed to store file in CAS: " & storeResult.errValue.msg, packageName: fragment.id.name )) let casObject = storeResult.okValue let packageFile = PackageFile( path: relativePath, hash: casObject.hash, hashAlgorithm: "blake3", # Use BLAKE3 for quantum-resistant hashing permissions: FilePermissions( mode: cast[int](info.permissions), # Convert permission set to int bitmask owner: "root", # Default ownership - TODO: preserve actual ownership group: "root" ), chunks: if casObject.chunks.len > 0: # Convert cas.ChunkRef to types_fixed.ChunkRef some(casObject.chunks.mapIt(ChunkRef(hash: it.hash, offset: it.offset, size: it.size))) else: none(seq[ChunkRef]) ) files.add(packageFile) totalSize += info.size # Create package manifest with proper Merkle root calculation let manifest = PackageManifest( files: files, totalSize: totalSize, created: now(), merkleRoot: "" # Will be calculated from all file hashes ) # Calculate Merkle root from all file hashes (sorted for deterministic results) # Use BLAKE3 for quantum-resistant hashing as specified in requirements let sortedHashes = files.mapIt(it.hash).sorted().join("") let merkleRoot = calculateBlake3(sortedHashes.toOpenArrayByte(0, sortedHashes.len - 1).toSeq()) let finalManifest = PackageManifest( files: manifest.files, totalSize: manifest.totalSize, created: manifest.created, merkleRoot: merkleRoot ) # Create NPK package with proper defaults and cryptographic algorithms let npkPackage = NpkPackage( metadata: fragment, files: files, manifest: finalManifest, signature: none(Signature), format: NpkBinary, cryptoAlgorithms: CryptoAlgorithms( hashAlgorithm: "BLAKE3", signatureAlgorithm: "Ed25519", version: "1.0" ) ) return ok[NpkPackage, NpkError](npkPackage) except IOError as e: return err[NpkPackage, NpkError](NpkError( code: FileReadError, msg: "Failed to create NPK package: " & e.msg, packageName: fragment.id.name )) except Exception as e: return err[NpkPackage, NpkError](NpkError( code: UnknownError, msg: "Unexpected error creating NPK package: " & e.msg, packageName: fragment.id.name )) # ============================================================================= # KDL Metadata Serialization (Placeholder) # ============================================================================= proc escapeKdlString(s: string): string = ## Escape special characters in KDL strings result = "\"" for c in s: case c: of '"': result.add("\\\"") of '\\': result.add("\\\\") of '\n': result.add("\\n") of '\r': result.add("\\r") of '\t': result.add("\\t") else: result.add(c) result.add("\"") proc formatKdlBoolean(b: bool): string = ## Format boolean for KDL if b: "true" else: "false" proc formatKdlArray(items: seq[string]): string = ## Format string array for KDL if items.len == 0: return "" result = "" for i, item in items: if i > 0: result.add(" ") result.add(escapeKdlString(item)) proc toHex(b: byte): string = ## Convert byte to hex string const hexChars = "0123456789abcdef" result = $hexChars[b shr 4] & $hexChars[b and 0x0F] proc serializeToKdl*(npk: NpkPackage): string = ## Serialize NPK package metadata to KDL format with robust string handling ## Follows the latest .npk.zst format specification with quantum-resistant algorithm support ## Enhanced KDL serialization with proper escaping and formatting result = "package " & escapeKdlString(npk.metadata.id.name) & " {\n" result.add(" version " & escapeKdlString(npk.metadata.id.version) & "\n") result.add(" stream " & escapeKdlString($npk.metadata.id.stream) & "\n") result.add(" format " & escapeKdlString($npk.format) & "\n") result.add("\n") # Source information with comprehensive metadata result.add(" source {\n") result.add(" method " & escapeKdlString($npk.metadata.source.sourceMethod) & "\n") result.add(" url " & escapeKdlString(npk.metadata.source.url) & "\n") result.add(" hash " & escapeKdlString(npk.metadata.source.hash) & "\n") result.add(" hash-algorithm " & escapeKdlString(npk.metadata.source.hashAlgorithm) & "\n") result.add(" timestamp " & escapeKdlString($npk.metadata.source.timestamp) & "\n") result.add(" }\n\n") # Cryptographic integrity section with quantum-ready algorithms result.add(" integrity {\n") result.add(" hash " & escapeKdlString(npk.manifest.merkleRoot) & "\n") result.add(" algorithm " & escapeKdlString(npk.cryptoAlgorithms.hashAlgorithm) & "\n") result.add(" signature-algorithm " & escapeKdlString(npk.cryptoAlgorithms.signatureAlgorithm) & "\n") result.add(" version " & escapeKdlString(npk.cryptoAlgorithms.version) & "\n") if npk.signature.isSome: let sig = npk.signature.get() result.add(" signature " & escapeKdlString(sig.signature.mapIt(it.toHex()).join("")) & "\n") result.add(" key-id " & escapeKdlString(sig.keyId) & "\n") result.add(" }\n\n") # Package metadata result.add(" metadata {\n") result.add(" description " & escapeKdlString(npk.metadata.metadata.description) & "\n") result.add(" license " & escapeKdlString(npk.metadata.metadata.license) & "\n") result.add(" maintainer " & escapeKdlString(npk.metadata.metadata.maintainer) & "\n") if npk.metadata.metadata.tags.len > 0: result.add(" tags " & formatKdlArray(npk.metadata.metadata.tags) & "\n") result.add(" }\n\n") # Runtime profile with comprehensive settings result.add(" runtime {\n") result.add(" libc " & escapeKdlString($npk.metadata.metadata.runtime.libc) & "\n") result.add(" allocator " & escapeKdlString($npk.metadata.metadata.runtime.allocator) & "\n") result.add(" systemd-aware " & formatKdlBoolean(npk.metadata.metadata.runtime.systemdAware) & "\n") result.add(" reproducible " & formatKdlBoolean(npk.metadata.metadata.runtime.reproducible) & "\n") if npk.metadata.metadata.runtime.tags.len > 0: result.add(" tags " & formatKdlArray(npk.metadata.metadata.runtime.tags) & "\n") result.add(" }\n\n") # Build system information result.add(" build {\n") result.add(" system " & escapeKdlString($npk.metadata.buildSystem) & "\n") result.add(" }\n\n") # Dependencies with version constraints if npk.metadata.dependencies.len > 0: result.add(" dependencies {\n") for dep in npk.metadata.dependencies: result.add(" " & escapeKdlString(dep.name) & " " & escapeKdlString(dep.version) & " stream=" & escapeKdlString($dep.stream) & "\n") result.add(" }\n\n") # ACUL compliance with comprehensive metadata result.add(" acul {\n") result.add(" required " & formatKdlBoolean(npk.metadata.acul.required) & "\n") if npk.metadata.acul.membership.len > 0: result.add(" membership " & escapeKdlString(npk.metadata.acul.membership) & "\n") if npk.metadata.acul.attribution.len > 0: result.add(" attribution " & escapeKdlString(npk.metadata.acul.attribution) & "\n") if npk.metadata.acul.buildLog.len > 0: result.add(" build-log " & escapeKdlString(npk.metadata.acul.buildLog) & "\n") result.add(" }\n\n") # Package manifest with comprehensive file information result.add(" manifest {\n") result.add(" total-size " & $npk.manifest.totalSize & "\n") result.add(" created " & escapeKdlString($npk.manifest.created) & "\n") result.add(" merkle-root " & escapeKdlString(npk.manifest.merkleRoot) & "\n") result.add(" file-count " & $npk.manifest.files.len & "\n") result.add(" }\n\n") # File entries with chunk information for deduplication result.add(" files {\n") let maxFiles = min(npk.files.len, 20) # Show first 20 files for better visibility for i in 0.. maxFiles: result.add(" // ... " & $(npk.files.len - maxFiles) & " more files (truncated for readability)\n") result.add(" }\n") result.add("}\n") proc deserializeFromKdl*(kdlContent: string): Result[NpkPackage, NpkError] = ## Deserialize NPK package from KDL format ## TODO: Implement proper KDL parsing when kdl library is available ## For now, return an error indicating this is not yet implemented return err[NpkPackage, NpkError](NpkError( code: InvalidMetadata, msg: "KDL deserialization not yet implemented - waiting for kdl library", packageName: "unknown" )) # ============================================================================= # Package Validation # ============================================================================= proc validateNpkPackage*(npk: NpkPackage): ValidationResult = ## Validate NPK package integrity and metadata var result = ValidationResult(valid: true, errors: @[], warnings: @[]) # Validate basic metadata if npk.metadata.id.name.len == 0: result.errors.add(ValidationError( field: "metadata.id.name", message: "Package name cannot be empty", suggestions: @["Provide a valid package name"] )) result.valid = false if npk.metadata.id.version.len == 0: result.errors.add(ValidationError( field: "metadata.id.version", message: "Package version cannot be empty", suggestions: @["Provide a valid version string"] )) result.valid = false # Validate source information if npk.metadata.source.url.len == 0: result.errors.add(ValidationError( field: "metadata.source.url", message: "Source URL cannot be empty", suggestions: @["Provide a valid source URL"] )) result.valid = false if npk.metadata.source.hash.len == 0: result.errors.add(ValidationError( field: "metadata.source.hash", message: "Source hash cannot be empty", suggestions: @["Calculate and provide source hash"] )) result.valid = false # Validate file entries if npk.files.len == 0: result.warnings.add("Package contains no files") for i, file in npk.files: if file.path.len == 0: result.errors.add(ValidationError( field: "files[" & $i & "].path", message: "File path cannot be empty", suggestions: @["Provide valid file path"] )) result.valid = false if file.hash.len == 0: result.errors.add(ValidationError( field: "files[" & $i & "].hash", message: "File hash cannot be empty", suggestions: @["Calculate file hash"] )) result.valid = false if not file.hash.startsWith("blake3-"): result.warnings.add("File " & file.path & " uses non-standard hash algorithm: " & file.hashAlgorithm) # Validate manifest consistency let calculatedSize = npk.files.mapIt(0'i64).foldl(a + b, 0'i64) # Simplified - would need actual file sizes if npk.manifest.totalSize <= 0: result.warnings.add("Manifest total size is zero or negative") # Validate Merkle root if npk.manifest.merkleRoot.len == 0: result.errors.add(ValidationError( field: "manifest.merkleRoot", message: "Merkle root cannot be empty", suggestions: @["Calculate Merkle root from file hashes"] )) result.valid = false return result # ============================================================================= # Digital Signature Support # ============================================================================= proc signNpkPackage*(npk: var NpkPackage, keyId: string, privateKey: seq[byte]): VoidResult[NpkError] = ## Sign NPK package with Ed25519 private key ## Creates a comprehensive signature payload including all critical package metadata try: # Create comprehensive signature payload from package metadata and manifest # Include all critical fields to ensure integrity let payload = npk.metadata.id.name & npk.metadata.id.version & $npk.metadata.id.stream & npk.manifest.merkleRoot & npk.metadata.source.hash & $npk.manifest.totalSize & $npk.manifest.created # TODO: Implement actual Ed25519 signing when crypto library is available # The implementation would be: # import ed25519 # let signatureBytes = ed25519.sign(privateKey, payload.toOpenArrayByte(0, payload.len - 1)) # For now, create a deterministic placeholder signature based on payload # This allows testing the signature infrastructure without actual crypto let payloadHash = calculateBlake3(payload.toOpenArrayByte(0, payload.len - 1).toSeq()) let placeholderSig = payloadHash[0..63].toOpenArrayByte(0, 63).toSeq() # 64 bytes like Ed25519 let signature = Signature( keyId: keyId, algorithm: npk.cryptoAlgorithms.signatureAlgorithm, signature: placeholderSig ) npk.signature = some(signature) return ok(NpkError) except Exception as e: return err[NpkError](NpkError( code: UnknownError, msg: "Failed to sign package: " & e.msg, packageName: npk.metadata.id.name )) proc verifyNpkSignature*(npk: NpkPackage, publicKey: seq[byte]): Result[bool, NpkError] = ## Verify NPK package signature ## TODO: Implement proper Ed25519 verification when crypto library is available if npk.signature.isNone: return ok[bool, NpkError](false) # No signature to verify try: let sig = npk.signature.get() # TODO: Implement actual Ed25519 verification # For now, just check if signature exists let isValid = sig.signature.len > 0 and sig.keyId.len > 0 return ok[bool, NpkError](isValid) except Exception as e: return err[bool, NpkError](NpkError( code: UnknownError, msg: "Failed to verify signature: " & e.msg, packageName: npk.metadata.id.name )) # ============================================================================= # Package Extraction # ============================================================================= proc extractNpkPackage*(npk: NpkPackage, targetDir: string, cas: var CasManager): VoidResult[NpkError] = ## Extract NPK package to target directory using CAS for file retrieval try: createDir(targetDir) for file in npk.files: let targetPath = targetDir / file.path let targetParent = targetPath.parentDir() # Ensure parent directory exists if not dirExists(targetParent): createDir(targetParent) # Retrieve file from CAS let retrieveResult = cas.retrieveFile(file.hash, targetPath) if not retrieveResult.isOk: return err[NpkError](NpkError( code: CasGeneralError, msg: "Failed to retrieve file from CAS: " & retrieveResult.errValue.msg, packageName: npk.metadata.id.name )) # Set file permissions try: setFilePermissions(targetPath, {fpUserRead, fpUserWrite}) # Simplified permissions except OSError: # Permission setting failed, but file was extracted discard return ok(NpkError) except IOError as e: return err[NpkError](NpkError( code: FileWriteError, msg: "Failed to extract package: " & e.msg, packageName: npk.metadata.id.name )) # ============================================================================= # Package Archive Creation (.npk.zst format) # ============================================================================= proc createNpkArchive*(npk: NpkPackage, archivePath: string, format: NpkArchiveFormat = NpkZst): VoidResult[NpkError] = ## Create .npk.zst archive file containing package data and metadata ## Uses tar archives compressed with zstd --fast for optimal speed and compression ## ## Format specification: ## - .npk.zst: Zstandard compressed (default, production use) ## - .npk.tar: Uncompressed tar (debugging only) try: # Create temporary directory for packaging let tempDir = getTempDir() / "npk_" & npk.metadata.id.name & "_" & npk.metadata.id.version if dirExists(tempDir): removeDir(tempDir) createDir(tempDir) # Write KDL metadata let kdlContent = serializeToKdl(npk) writeFile(tempDir / "package.kdl", kdlContent) # Write manifest as JSON let manifestJson = %*{ "files": npk.manifest.files.mapIt(%*{ "path": it.path, "hash": it.hash, "hash_algorithm": it.hashAlgorithm, "permissions": %*{ "mode": it.permissions.mode, "owner": it.permissions.owner, "group": it.permissions.group } }), "total_size": npk.manifest.totalSize, "created": $npk.manifest.created, "merkle_root": npk.manifest.merkleRoot } writeFile(tempDir / "manifest.json", $manifestJson) # Determine final archive path based on format let finalArchivePath = case format: of NpkZst: if not archivePath.endsWith(".npk.zst"): archivePath & ".npk.zst" else: archivePath of NpkTar: if not archivePath.endsWith(".npk.tar"): archivePath & ".npk.tar" else: archivePath case format: of NpkZst: # Create tar archive first let tarPath = tempDir / "package.tar" let tarCmd = "tar -cf " & tarPath & " -C " & tempDir & " ." let tarResult = execCmdEx(tarCmd, options = {poUsePath}) if tarResult.exitCode != 0: return err[NpkError](NpkError( code: FileWriteError, msg: "Failed to create tar archive: " & tarResult.output, packageName: npk.metadata.id.name )) # Compress with zstd --fast for optimal speed and compression let zstdCmd = "zstd -q --fast -o " & finalArchivePath & " " & tarPath let zstdResult = execCmdEx(zstdCmd, options = {poUsePath}) if zstdResult.exitCode != 0: return err[NpkError](NpkError( code: FileWriteError, msg: "Failed to compress archive with zstd: " & zstdResult.output, packageName: npk.metadata.id.name )) of NpkTar: # Create uncompressed tar archive for debugging let tarCmd = "tar -cf " & finalArchivePath & " -C " & tempDir & " ." let tarResult = execCmdEx(tarCmd, options = {poUsePath}) if tarResult.exitCode != 0: return err[NpkError](NpkError( code: FileWriteError, msg: "Failed to create tar archive: " & tarResult.output, packageName: npk.metadata.id.name )) # Clean up temp directory if dirExists(tempDir): removeDir(tempDir) return ok(NpkError) except IOError as e: return err[NpkError](NpkError( code: FileWriteError, msg: "Failed to create NPK archive: " & e.msg, packageName: npk.metadata.id.name )) proc loadNpkArchive*(archivePath: string): Result[NpkPackage, NpkError] = ## Load NPK package from archive file ## Supports tar.zst compressed archives try: if not fileExists(archivePath): return err[NpkPackage, NpkError](NpkError( code: PackageNotFound, msg: "NPK archive not found: " & archivePath, packageName: "unknown" )) # Create temporary directory for extraction let tempDir = getTempDir() / "npk_extract_" & $epochTime() if dirExists(tempDir): removeDir(tempDir) createDir(tempDir) # Decompress with zstd let decompressCmd = "zstd -d -q -o " & tempDir & "/archive.tar " & archivePath let decompressResult = execCmdEx(decompressCmd, options = {poUsePath}) if decompressResult.exitCode != 0: return err[NpkPackage, NpkError](NpkError( code: FileReadError, msg: "Failed to decompress archive with zstd: " & decompressResult.output, packageName: "unknown" )) # Extract tar archive let tarCmd = "tar -xf " & tempDir & "/archive.tar -C " & tempDir let tarResult = execCmdEx(tarCmd, options = {poUsePath}) if tarResult.exitCode != 0: return err[NpkPackage, NpkError](NpkError( code: FileReadError, msg: "Failed to extract tar archive: " & tarResult.output, packageName: "unknown" )) # Read KDL metadata let kdlPath = tempDir / "package.kdl" if not fileExists(kdlPath): return err[NpkPackage, NpkError](NpkError( code: InvalidMetadata, msg: "Package metadata not found in archive", packageName: "unknown" )) let kdlContent = readFile(kdlPath) # TODO: Implement proper KDL parsing when kdl library is available # For now, return error indicating not implemented return err[NpkPackage, NpkError](NpkError( code: InvalidMetadata, msg: "NPK archive loading not fully implemented - waiting for KDL and archive libraries", packageName: "unknown" )) except IOError as e: return err[NpkPackage, NpkError](NpkError( code: FileReadError, msg: "Failed to load NPK archive: " & e.msg, packageName: "unknown" )) # ============================================================================= # Utility Functions # ============================================================================= proc calculateBlake2b*(data: seq[byte]): string = ## Calculate BLAKE2b hash - imported from CAS module cas.calculateBlake2b(data) proc calculateBlake3*(data: seq[byte]): string = ## Calculate BLAKE3 hash - imported from CAS module cas.calculateBlake3(data) proc getNpkInfo*(npk: NpkPackage): string = ## Get human-readable package information result = "NPK Package: " & npk.metadata.id.name & " v" & npk.metadata.id.version & "\n" result.add("Stream: " & $npk.metadata.id.stream & "\n") result.add("Files: " & $npk.files.len & "\n") result.add("Total Size: " & $npk.manifest.totalSize & " bytes\n") result.add("Created: " & $npk.manifest.created & "\n") result.add("Merkle Root: " & npk.manifest.merkleRoot & "\n") if npk.signature.isSome: result.add("Signed: Yes (Key: " & npk.signature.get().keyId & ")\n") else: result.add("Signed: No\n") # ============================================================================= # Conversion from Grafted Packages # ============================================================================= proc convertGraftToNpk*(graftResult: GraftResult, cas: var CasManager): Result[NpkPackage, NpkError] = ## Convert a grafted package (GraftResult) into an NPK package ## This includes preserving provenance and audit log information ## Files are stored in CAS for deduplication and integrity verification # Construct Fragment from GraftResult metadata let pkgId = PackageId( name: graftResult.metadata.packageName, version: graftResult.metadata.version, stream: Custom # Default to Custom for grafts ) let source = Source( url: graftResult.metadata.provenance.downloadUrl, hash: graftResult.metadata.originalHash, hashAlgorithm: "blake2b", # Default assumption sourceMethod: Grafted, timestamp: graftResult.metadata.graftedAt ) let fragment = Fragment( id: pkgId, source: source, dependencies: @[], # Dependencies not captured in simple GraftResult buildSystem: Custom, metadata: PackageMetadata( description: "Grafted from " & graftResult.metadata.source, license: "Unknown", maintainer: "Auto-Graft", tags: @["grafted"], runtime: RuntimeProfile( libc: Glibc, # Assumption allocator: System, systemdAware: false, reproducible: false, tags: @[] ) ), acul: AculCompliance( required: false, membership: "", attribution: "Grafted package", buildLog: graftResult.metadata.buildLog ) ) let extractedPath = graftResult.metadata.provenance.extractedPath if extractedPath.len == 0 or not dirExists(extractedPath): return err[NpkPackage, NpkError](NpkError( code: PackageNotFound, msg: "Extracted path not found or empty in graft result", packageName: pkgId.name )) # Use the constructed fragment and extractedPath to create NPK package let createResult = createNpkPackage(fragment, extractedPath, cas) if not createResult.isOk: return err[NpkPackage, NpkError](createResult.errValue) var npk = createResult.okValue # Map provenance information # Add provenance information to runtime tags for tracking let provenanceTag = "grafted:" & graftResult.metadata.source & ":" & $graftResult.metadata.graftedAt npk.metadata.metadata.runtime.tags.add(provenanceTag) # Add deduplication status to tags for audit purposes (simplified) let deduplicationTag = "dedup:unknown" npk.metadata.metadata.runtime.tags.add(deduplicationTag) # Preserve original archive hash in attribution if npk.metadata.acul.attribution.len > 0: npk.metadata.acul.attribution.add(" | ") npk.metadata.acul.attribution.add("Original: " & graftResult.metadata.originalHash) # Return the constructed NPK package with full provenance return ok[NpkPackage, NpkError](npk)