diff --git a/nip.nim b/nip.nim index 362d7e3..6d61542 100644 --- a/nip.nim +++ b/nip.nim @@ -1,10 +1,15 @@ #!/usr/bin/env nim -## NIP MVP - Minimal Viable Product CLI -## Simple, focused package grafting from Nix, PKGSRC, and Pacman +# Copyright (c) 2026 Nexus Foundation +# Licensed under the Libertaria Sovereign License (LSL-1.0) +# See legal/LICENSE_SOVEREIGN.md for details. + +# NIP MVP - Minimal Viable Product CLI +# Simple, focused package grafting from Nix, PKGSRC, and Pacman import std/[os, strutils, strformat] import src/nimpak/cli/graft_commands import src/nimpak/cli/bootstrap_commands +import src/nimpak/cli/store_commands const Version = "0.1.0-mvp" @@ -30,6 +35,7 @@ COMMANDS: doctor Check system health setup Setup system integration (PATH, libraries) bootstrap Build tool management (nix, pkgsrc, gentoo) + store Interact with Content-Addressable Storage (CAS) config [show|init] Show or initialize configuration logs [lines] Show recent log entries (default: 50) search Search for packages (coming soon) @@ -227,6 +233,9 @@ proc main() = bootstrapHelpCommand() exitCode = 1 + of "store": + exitCode = dispatchStoreCommand(commandArgs, verbose) + else: echo fmt"Error: Unknown command '{command}'" echo "Run 'nip --help' for usage information" diff --git a/src/nimpak/adapters/aur.nim b/src/nimpak/adapters/aur.nim index 990317f..936efc7 100644 --- a/src/nimpak/adapters/aur.nim +++ b/src/nimpak/adapters/aur.nim @@ -4,7 +4,7 @@ import std/[strutils, json, os, times, osproc, tables, strformat, httpclient] import ../grafting -from ../cas import Result, ok, err, isErr, get +import ../types type AURAdapter* = ref object of PackageAdapter @@ -240,10 +240,10 @@ proc downloadPKGBUILD(adapter: AURAdapter, packageName: string): Result[string, writeFile(pkgbuildPath, content) - return Result[string, string](isOk: true, value: pkgbuildPath) + return Result[string, string](isOk: true, okValue: pkgbuildPath) except Exception as e: - return Result[string, string](isOk: false, error: fmt"Failed to download PKGBUILD: {e.msg}") + return Result[string, string](isOk: false, errValue: fmt"Failed to download PKGBUILD: {e.msg}") proc showPKGBUILDReview(pkgbuildPath: string): bool = ## Show PKGBUILD for user review @@ -316,26 +316,26 @@ proc calculateAURHash(pkgbuildPath: string): string = "aur-hash-error" -method validatePackage*(adapter: AURAdapter, packageName: string): Result[bool, string] {.base.} = +method validatePackage*(adapter: AURAdapter, packageName: string): Result[bool, string] = ## Validate that a package exists in AUR try: let info = searchAURPackage(adapter, packageName) if info.name == "": - return Result[bool, string](isOk: false, error: fmt"Package '{packageName}' not found in AUR") + return Result[bool, string](isOk: false, errValue: fmt"Package '{packageName}' not found in AUR") - return Result[bool, string](isOk: true, value: true) + return Result[bool, string](isOk: true, okValue: true) except Exception as e: - return Result[bool, string](isOk: false, error: fmt"Validation error: {e.msg}") + return Result[bool, string](isOk: false, errValue: fmt"Validation error: {e.msg}") -method getPackageInfo*(adapter: AURAdapter, packageName: string): Result[JsonNode, string] {.base.} = +method getPackageInfo*(adapter: AURAdapter, packageName: string): Result[JsonNode, string] = ## Get detailed package information from AUR try: let info = searchAURPackage(adapter, packageName) if info.name == "": - return Result[JsonNode, string](isOk: false, error: fmt"Package '{packageName}' not found in AUR") + return Result[JsonNode, string](isOk: false, errValue: fmt"Package '{packageName}' not found in AUR") let jsonResult = %*{ "name": info.name, @@ -354,7 +354,7 @@ method getPackageInfo*(adapter: AURAdapter, packageName: string): Result[JsonNod "build_method": "nippel" } - return Result[JsonNode, string](isOk: true, value: jsonResult) + return Result[JsonNode, string](isOk: true, okValue: jsonResult) except Exception as e: - return Result[JsonNode, string](isOk: false, error: fmt"Error getting package info: {e.msg}") + return Result[JsonNode, string](isOk: false, errValue: fmt"Error getting package info: {e.msg}") diff --git a/src/nimpak/adapters/git.nim b/src/nimpak/adapters/git.nim index dd60833..304f7fa 100644 --- a/src/nimpak/adapters/git.nim +++ b/src/nimpak/adapters/git.nim @@ -1,17 +1,17 @@ -## Git Source Adapter for NexusForge -## Implements "Obtainium-style" Git-based package resolution -## -## Features: -## - Parse git+https:// URLs with optional tag/branch specifiers -## - Poll GitHub/GitLab APIs for tags and releases -## - Semver matching and wildcard support -## - Shallow clone for efficient fetching +# Git Source Adapter for NexusForge +# Implements "Obtainium-style" Git-based package resolution +# +# Features: +# - Parse git+https:// URLs with optional tag/branch specifiers +# - Poll GitHub/GitLab APIs for tags and releases +# - Semver matching and wildcard support +# - Shallow clone for efficient fetching import std/[strutils, options, json, httpclient, os, osproc, uri, times, sequtils, algorithm] import ../types/grafting_types import ../cas -from ../cas import Result, VoidResult, ok, err, isErr, get +import ../types type GitSourceKind* = enum @@ -468,7 +468,7 @@ proc ingestDirToCas*(cas: var CasManager, sourceDir: string, let storeResult = cas.storeObject(dataBytes) if storeResult.isOk: - let obj = storeResult.value + let obj = storeResult.okValue allHashes.add(file & ":" & obj.hash) result.files.add(file) totalSize += obj.size @@ -488,7 +488,7 @@ proc ingestDirToCas*(cas: var CasManager, sourceDir: string, if manifestResult.isOk: result.success = true - result.casHash = manifestResult.value.hash + result.casHash = manifestResult.okValue.hash result.totalSize = totalSize # ============================================================================= @@ -577,7 +577,7 @@ proc downloadAndIngestAsset*(cas: var CasManager, asset: GitAsset, # Download the asset let downloadResult = downloadReleaseAsset(asset, tempPath, token) if not downloadResult.isOk: - return err[string, string](downloadResult.error) + return err[string, string](downloadResult.errValue) # Ingest into CAS try: @@ -589,7 +589,7 @@ proc downloadAndIngestAsset*(cas: var CasManager, asset: GitAsset, removeFile(tempPath) if storeResult.isOk: - return ok[string, string](storeResult.value.hash) + return ok[string, string](storeResult.okValue.hash) else: return err[string, string]("CAS store failed") except IOError as e: @@ -628,10 +628,10 @@ proc obtainPackage*(cas: var CasManager, source: GitSource, tagPattern: string = # Step 1: Get available tags let tagsResult = fetchTags(source) if not tagsResult.isOk: - result.errors.add("Failed to fetch tags: " & tagsResult.error) + result.errors.add("Failed to fetch tags: " & tagsResult.errValue) return - let matchedTags = filterTags(tagsResult.value, tagPattern) + let matchedTags = filterTags(tagsResult.okValue, tagPattern) if matchedTags.len == 0: result.errors.add("No tags match pattern: " & tagPattern) return @@ -644,7 +644,7 @@ proc obtainPackage*(cas: var CasManager, source: GitSource, tagPattern: string = if preferRelease and source.kind == GitHub: let releasesResult = fetchGitHubReleases(source) if releasesResult.isOk: - for release in releasesResult.value: + for release in releasesResult.okValue: if release.tag == bestTag.name: let asset = findAssetByPattern(release, assetPattern) if asset.isSome: @@ -652,7 +652,7 @@ proc obtainPackage*(cas: var CasManager, source: GitSource, tagPattern: string = actualCacheDir, source.token) if ingestResult.isOk: result.success = true - result.casHash = ingestResult.value + result.casHash = ingestResult.okValue result.fetchMethod = "release" result.files = @[asset.get().name] return diff --git a/src/nimpak/adapters/nix.nim b/src/nimpak/adapters/nix.nim index b753d89..99ccf35 100644 --- a/src/nimpak/adapters/nix.nim +++ b/src/nimpak/adapters/nix.nim @@ -3,7 +3,7 @@ import std/[strutils, json, os, times, osproc, tables, strformat] import ../grafting -from ../cas import Result, ok, err, isErr, get +import ../types type NixAdapter* = ref object of PackageAdapter @@ -351,31 +351,31 @@ proc calculateNixStoreHash(storePath: string): string = "nix-hash-error" -method validatePackage*(adapter: NixAdapter, packageName: string): Result[bool, string] {.base.} = +method validatePackage*(adapter: NixAdapter, packageName: string): Result[bool, string] = ## Validate that a package exists in nixpkgs try: if not isNixAvailable(): - return Result[bool, string](isOk: false, error: "Nix is not installed. Install Nix from https://nixos.org/download.html") + return Result[bool, string](isOk: false, errValue: "Nix is not installed. Install Nix from https://nixos.org/download.html") let info = getNixPackageInfo(adapter, packageName) if info.name == "": - return Result[bool, string](isOk: false, error: fmt"Package '{packageName}' not found in nixpkgs") + return Result[bool, string](isOk: false, errValue: fmt"Package '{packageName}' not found in nixpkgs") - return Result[bool, string](isOk: true, value: true) + return Result[bool, string](isOk: true, okValue: true) except JsonParsingError as e: - return Result[bool, string](isOk: false, error: fmt"Failed to parse Nix output: {e.msg}") + return Result[bool, string](isOk: false, errValue: fmt"Failed to parse Nix output: {e.msg}") except Exception as e: - return Result[bool, string](isOk: false, error: fmt"Validation error: {e.msg}") + return Result[bool, string](isOk: false, errValue: fmt"Validation error: {e.msg}") -method getPackageInfo*(adapter: NixAdapter, packageName: string): Result[JsonNode, string] {.base.} = +method getPackageInfo*(adapter: NixAdapter, packageName: string): Result[JsonNode, string] = ## Get detailed package information from nixpkgs try: let info = getNixPackageInfo(adapter, packageName) if info.name == "": - return Result[JsonNode, string](isOk: false, error: fmt"Package '{packageName}' not found in nixpkgs") + return Result[JsonNode, string](isOk: false, errValue: fmt"Package '{packageName}' not found in nixpkgs") let jsonResult = %*{ "name": info.name, @@ -389,10 +389,10 @@ method getPackageInfo*(adapter: NixAdapter, packageName: string): Result[JsonNod "adapter": adapter.name } - return Result[JsonNode, string](isOk: true, value: jsonResult) + return Result[JsonNode, string](isOk: true, okValue: jsonResult) except Exception as e: - return Result[JsonNode, string](isOk: false, error: fmt"Error getting package info: {e.msg}") + return Result[JsonNode, string](isOk: false, errValue: fmt"Error getting package info: {e.msg}") # Utility functions for Nix integration proc getNixSystemInfo*(): JsonNode = diff --git a/src/nimpak/adapters/pacman.nim b/src/nimpak/adapters/pacman.nim index 61edd91..70ad5b6 100644 --- a/src/nimpak/adapters/pacman.nim +++ b/src/nimpak/adapters/pacman.nim @@ -1,11 +1,11 @@ -## Pacman Database Adapter for NIP -## -## This module provides integration with the existing pacman package manager, -## allowing NIP to read, understand, and manage pacman-installed packages. -## This enables gradual migration from pacman to NIP on Arch Linux systems. +# Pacman Database Adapter for NIP +# +# This module provides integration with the existing pacman package manager, +# allowing NIP to read, understand, and manage pacman-installed packages. +# This enables gradual migration from pacman to NIP on Arch Linux systems. import std/[os, strutils, tables, times, sequtils, options, strformat, hashes, osproc] -from ../cas import VoidResult, Result, ok, get, err +import ../types import ../grafting type @@ -319,10 +319,10 @@ proc syncWithNip*(adapter: var PacmanAdapter): Result[int, string] = # This would integrate with the existing NIP database system syncedCount.inc - return Result[int, string](isOk: true, value: syncedCount) + return Result[int, string](isOk: true, okValue: syncedCount) except Exception as e: - return Result[int, string](isOk: false, error: "Failed to sync with NIP: " & e.msg) + return Result[int, string](isOk: false, errValue: "Failed to sync with NIP: " & e.msg) proc getPackageInfo*(adapter: PacmanAdapter, name: string): string = ## Get detailed package information in human-readable format @@ -390,18 +390,18 @@ proc nipPacmanSync*(): Result[string, string] = let loadResult = adapter.loadPacmanDatabase() if not loadResult.isOk: - return Result[string, string](isOk: false, error: loadResult.errValue) + return Result[string, string](isOk: false, errValue: loadResult.errValue) let syncResult = adapter.syncWithNip() if not syncResult.isOk: - return Result[string, string](isOk: false, error: syncResult.error) + return Result[string, string](isOk: false, errValue: syncResult.errValue) let stats = adapter.getSystemStats() let message = "✅ Synchronized " & $syncResult.get() & " packages\n" & "📊 Total: " & $stats.totalPackages & " packages, " & $(stats.totalSize div (1024*1024)) & " MB" - return Result[string, string](isOk: true, value: message) + return Result[string, string](isOk: true, okValue: message) proc nipPacmanList*(query: string = ""): Result[string, string] = ## NIP command: nip pacman-list [query] @@ -410,7 +410,7 @@ proc nipPacmanList*(query: string = ""): Result[string, string] = let loadResult = adapter.loadPacmanDatabase() if not loadResult.isOk: - return Result[string, string](isOk: false, error: loadResult.errValue) + return Result[string, string](isOk: false, errValue: loadResult.errValue) let packages = if query == "": adapter.listPackages() @@ -429,7 +429,7 @@ proc nipPacmanList*(query: string = ""): Result[string, string] = result.add("\n") result.add("\nTotal: " & $packages.len & " packages") - return Result[string, string](isOk: true, value: result) + return Result[string, string](isOk: true, okValue: result) proc nipPacmanInfo*(packageName: string): Result[string, string] = ## NIP command: nip pacman-info @@ -438,10 +438,10 @@ proc nipPacmanInfo*(packageName: string): Result[string, string] = let loadResult = adapter.loadPacmanDatabase() if not loadResult.isOk: - return Result[string, string](isOk: false, error: loadResult.errValue) + return Result[string, string](isOk: false, errValue: loadResult.errValue) let info = adapter.getPackageInfo(packageName) - return Result[string, string](isOk: true, value: info) + return Result[string, string](isOk: true, okValue: info) proc nipPacmanDeps*(packageName: string): Result[string, string] = ## NIP command: nip pacman-deps @@ -450,7 +450,7 @@ proc nipPacmanDeps*(packageName: string): Result[string, string] = let loadResult = adapter.loadPacmanDatabase() if not loadResult.isOk: - return Result[string, string](isOk: false, error: loadResult.errValue) + return Result[string, string](isOk: false, errValue: loadResult.errValue) var visited: seq[string] = @[] let deps = adapter.getDependencyTree(packageName, visited) @@ -465,7 +465,7 @@ proc nipPacmanDeps*(packageName: string): Result[string, string] = else: result.add("\nTotal dependencies: " & $deps.len) - return Result[string, string](isOk: true, value: result) + return Result[string, string](isOk: true, okValue: result) # Grafting adapter methods for coordinator integration @@ -476,12 +476,12 @@ method validatePackage*(adapter: PacmanAdapter, packageName: string): Result[boo let (output, exitCode) = execCmdEx(fmt"pacman -Ss '^{packageName}$'") if exitCode == 0 and output.len > 0: - return Result[bool, string](isOk: true, value: true) + return Result[bool, string](isOk: true, okValue: true) else: - return Result[bool, string](isOk: true, value: false) + return Result[bool, string](isOk: true, okValue: false) except Exception as e: - return Result[bool, string](isOk: false, error: "Failed to validate package: " & e.msg) + return Result[bool, string](isOk: false, errValue: "Failed to validate package: " & e.msg) proc isPackageInstalled(adapter: PacmanAdapter, packageName: string): bool = ## Check if package is installed locally using pacman -Q diff --git a/src/nimpak/adapters/pkgsrc.nim b/src/nimpak/adapters/pkgsrc.nim index 5fff919..7685571 100644 --- a/src/nimpak/adapters/pkgsrc.nim +++ b/src/nimpak/adapters/pkgsrc.nim @@ -3,7 +3,7 @@ import std/[strutils, json, os, times, osproc, strformat] import ../grafting -from ../cas import Result, ok, err, isErr, get +import ../types type PKGSRCAdapter* = ref object of PackageAdapter @@ -490,9 +490,9 @@ method validatePackage*(adapter: PKGSRCAdapter, packageName: string): Result[boo ## Validate that a package exists in PKGSRC try: let info = findPKGSRCPackage(adapter, packageName) - return Result[bool, string](isOk: true, value: info.name != "") + return Result[bool, string](isOk: true, okValue: info.name != "") except Exception as e: - return Result[bool, string](isOk: false, error: fmt"Validation error: {e.msg}") + return Result[bool, string](isOk: false, errValue: fmt"Validation error: {e.msg}") method getPackageInfo*(adapter: PKGSRCAdapter, packageName: string): Result[JsonNode, string] = ## Get detailed package information from PKGSRC @@ -500,7 +500,7 @@ method getPackageInfo*(adapter: PKGSRCAdapter, packageName: string): Result[Json let info = findPKGSRCPackage(adapter, packageName) if info.name == "": - return Result[JsonNode, string](isOk: false, error: fmt"Package '{packageName}' not found in PKGSRC") + return Result[JsonNode, string](isOk: false, errValue: fmt"Package '{packageName}' not found in PKGSRC") let result = %*{ "name": info.name, @@ -517,10 +517,10 @@ method getPackageInfo*(adapter: PKGSRCAdapter, packageName: string): Result[Json "adapter": adapter.name } - return Result[JsonNode, string](isOk: true, value: result) + return Result[JsonNode, string](isOk: true, okValue: result) except Exception as e: - return Result[JsonNode, string](isOk: false, error: fmt"Error getting package info: {e.msg}") + return Result[JsonNode, string](isOk: false, errValue: fmt"Error getting package info: {e.msg}") # Utility functions proc isPKGSRCAvailable*(adapter: PKGSRCAdapter): bool = diff --git a/src/nimpak/cas.nim b/src/nimpak/cas.nim index 47de384..e1b244f 100644 --- a/src/nimpak/cas.nim +++ b/src/nimpak/cas.nim @@ -1,11 +1,11 @@ -## Content-Addressable Storage (CAS) System -## -## This module implements the foundational content-addressable storage system -## that provides automatic deduplication and cryptographic verification using -## xxHash (xxh3_128) for maximum performance with BLAKE2b legacy fallback. -## -## Hash Algorithm: xxHash xxh3_128 (40-50 GiB/s, 128-bit collision-safe) -## Legacy Support: BLAKE2b-512 (for backward compatibility) +# Content-Addressable Storage (CAS) System +# +# This module implements the foundational content-addressable storage system +# that provides automatic deduplication and cryptographic verification using +# xxHash (xxh3_128) for maximum performance with BLAKE2b legacy fallback. +# +# Hash Algorithm: xxHash xxh3_128 (40-50 GiB/s, 128-bit collision-safe) +# Legacy Support: BLAKE2b-512 (for backward compatibility) import std/[os, tables, sets, strutils, json, sequtils, hashes, options, times, algorithm] {.warning[Deprecated]:off.} @@ -13,37 +13,12 @@ import std/threadpool # For parallel operations {.warning[Deprecated]:on.} import xxhash # Modern high-performance hashing (2-3x faster than BLAKE2b) import nimcrypto/blake2 # Legacy fallback -import ../nip/types +import ./types import ./protection # Read-only protection manager # Result type for error handling - using std/options for now -type - Result*[T, E] = object - case isOk*: bool - of true: - value*: T - of false: - error*: E +# Result types are imported from ./types - VoidResult*[E] = object - case isOk*: bool - of true: - discard - of false: - errValue*: E - -proc ok*[T, E](val: T): Result[T, E] = - Result[T, E](isOk: true, value: val) - -proc err*[T, E](error: E): Result[T, E] = - Result[T, E](isOk: false, error: error) - -proc ok*[E](dummy: typedesc[E]): VoidResult[E] = - VoidResult[E](isOk: true) - -proc isErr*[T, E](r: Result[T, E]): bool = not r.isOk -proc get*[T, E](r: Result[T, E]): T = r.value -proc getError*[T, E](r: Result[T, E]): E = r.error type FormatType* = enum diff --git a/src/nimpak/cli/store_commands.nim b/src/nimpak/cli/store_commands.nim new file mode 100644 index 0000000..3d6be21 --- /dev/null +++ b/src/nimpak/cli/store_commands.nim @@ -0,0 +1,174 @@ +# core/nip/src/nimpak/cli/store_commands.nim +## CLI Commands for Nexus CAS (Content Addressable Storage) + +import std/[options, strutils, strformat, terminal, os] +import ../types +import ../errors +import ../cas +import ../logger + +proc storeHelpCommand() = + echo """ +NIP STORE - Sovereign CAS Interface + +USAGE: + nip store [arguments] + +COMMANDS: + push Store a file in CAS (returns hash) + fetch Retrieve file from CAS by hash + verify Check if object exists and verify integrity + gc Run garbage collection on CAS + stats Show CAS statistics + path Show physical path of object (if exists) + +EXAMPLES: + nip store push mybinary.elf + nip store fetch xxh3-123... /tmp/restored.elf + nip store verify xxh3-123... + nip store stats +""" + +proc storePushCommand*(args: seq[string], verbose: bool): int = + ## Push a file to CAS + if args.len < 1: + errorLog("Usage: nip store push ") + return 1 + + let filePath = args[0] + if not fileExists(filePath): + errorLog(fmt"File not found: {filePath}") + return 1 + + let cas = initCasManager() + + if verbose: showInfo(fmt"Storing '{filePath}'...") + + let res = cas.storeFile(filePath) + if res.isOk: + let obj = res.get() + if verbose: + showInfo(fmt"Stored successfully.") + showInfo(fmt" Original Size: {obj.size} bytes") + showInfo(fmt" Compressed Size: {obj.compressedSize} bytes") + showInfo(fmt" Chunks: {obj.chunks.len}") + + # Output ONLY the hash to stdout for piping support + echo obj.hash + return 0 + else: + errorLog(formatError(res.getError())) + return 1 + +proc storeFetchCommand*(args: seq[string], verbose: bool): int = + ## Fetch a file from CAS + if args.len < 2: + errorLog("Usage: nip store fetch ") + return 1 + + let hash = args[0] + let destPath = args[1] + + # Remove prefix if user typed "fetch cas:" or similar + let cleanHash = if hash.contains(":"): hash.split(":")[1] else: hash + + let cas = initCasManager() + + if verbose: showInfo(fmt"Fetching object {cleanHash} to {destPath}...") + + let res = cas.retrieveFile(cleanHash, destPath) + if res.isOk: + if verbose: showInfo("Success.") + return 0 + else: + errorLog(formatError(res.getError())) + return 1 + +proc storeVerifyCommand*(args: seq[string], verbose: bool): int = + ## Verify object existence and integrity + if args.len < 1: + errorLog("Usage: nip store verify ") + return 1 + + let hash = args[0] + let cas = initCasManager() + + if cas.objectExists(hash): + # Retrieve to verify integrity (checksum check happens during retrieve logic implicitly if we extended it, + # currently retrieveObject just reads. Ideally we should re-hash.) + + # Simple existence check for MVP + showInfo(fmt"Object {hash} exists.") + + # Check if we can read it + let res = cas.retrieveObject(hash) + if res.isOk: + let data = res.get() + let computed = cas.computeHash(data) + if computed == hash: + showInfo("Integrity: VERIFIED (" & $data.len & " bytes)") + return 0 + else: + errorLog(fmt"Integrity: FAILED (Computed: {computed})") + return 1 + else: + errorLog("Corruption: Object exists in index/path but cannot be read.") + return 1 + else: + errorLog(fmt"Object {hash} NOT FOUND.") + return 1 + +proc storeStatsCommand*(verbose: bool): int = + let cas = initCasManager() + # MVP stats + # Since we don't have a persistent counter file in this MVP definition other than 'cas_index.kdl' which we parse manually? + # CasManager has 'CasStats' type but no automatic loadStats() method exposed in cas.nim yet. + # We will just show directory sizes. + + showInfo("CAS Storage Statistics") + showInfo(fmt"Root: {cas.rootPath}") + + # Simple walkdir to count + var count = 0 + var size = 0'i64 + + for kind, path in walkDir(cas.rootPath / "objects", relative=true): + # Recurse... for MVP just simple ls of shards + discard + + showInfo("(Detailed stats pending implementation)") + return 0 + +proc storePathCommand*(args: seq[string], verbose: bool): int = + if args.len < 1: + return 1 + let hash = args[0] + let cas = initCasManager() + let path = getObjectPath(cas.rootPath, hash) + if fileExists(path): + echo path + return 0 + else: + return 1 + +proc dispatchStoreCommand*(args: seq[string], verbose: bool): int = + if args.len == 0: + storeHelpCommand() + return 0 + + let cmd = args[0].toLowerAscii() + let subArgs = if args.len > 1: args[1..^1] else: @[] + + case cmd + of "push": return storePushCommand(subArgs, verbose) + of "fetch", "pull": return storeFetchCommand(subArgs, verbose) + of "verify": return storeVerifyCommand(subArgs, verbose) + of "stats": return storeStatsCommand(verbose) + of "path": return storePathCommand(subArgs, verbose) + of "help": + storeHelpCommand() + return 0 + else: + errorLog(fmt"Unknown store command: {cmd}") + storeHelpCommand() + return 1 diff --git a/src/nimpak/dependency.nim b/src/nimpak/dependency.nim index 75a74f4..49a4c31 100644 --- a/src/nimpak/dependency.nim +++ b/src/nimpak/dependency.nim @@ -2,7 +2,7 @@ # Dependency graph resolution and management system import std/[tables, sets, sequtils, algorithm, strformat] -import ../nip/types +import ./types type DependencyGraph* = object diff --git a/src/nimpak/errors.nim b/src/nimpak/errors.nim index f1822f6..e533237 100644 --- a/src/nimpak/errors.nim +++ b/src/nimpak/errors.nim @@ -1,11 +1,11 @@ -## NimPak Error Handling -## -## Comprehensive error handling utilities for the NimPak system. -## Provides formatted error messages, recovery suggestions, and error chaining. -## Task 37: Implement comprehensive error handling. +# NimPak Error Handling +# +# Comprehensive error handling utilities for the NimPak system. +# Provides formatted error messages, recovery suggestions, and error chaining. +# Task 37: Implement comprehensive error handling. import std/[strformat, strutils, times, tables, terminal] -import ../nip/types +import ./types # ############################################################################ # Error Formatting diff --git a/src/nimpak/graft_coordinator.nim b/src/nimpak/graft_coordinator.nim index a451625..4198d10 100644 --- a/src/nimpak/graft_coordinator.nim +++ b/src/nimpak/graft_coordinator.nim @@ -1,12 +1,12 @@ -## graft_coordinator.nim -## Coordinates grafting from adapters and installation -## Ties together adapters + install_manager for unified grafting +# graft_coordinator.nim +# Coordinates grafting from adapters and installation +# Ties together adapters + install_manager for unified grafting import std/[strformat, strutils, json, os] import install_manager, simple_db, config import adapters/[nix, pacman, pkgsrc, aur] import grafting # For GraftResult type -from cas import get +import types type GraftCoordinator* = ref object @@ -392,10 +392,11 @@ proc parsePackageSpec*(spec: string): tuple[source: GraftSource, name: string] = let name = parts[1] let source = case sourceStr - of "nix": Nix - of "pkgsrc": PKGSRC - of "pacman": Pacman - else: Auto + of "nix": GraftSource.Nix + of "pkgsrc": GraftSource.PKGSRC + of "pacman": GraftSource.Pacman + of "aur": GraftSource.AUR + else: GraftSource.Auto return (source, name) else: diff --git a/src/nimpak/grafting.nim b/src/nimpak/grafting.nim index bfac7c0..57e53b1 100644 --- a/src/nimpak/grafting.nim +++ b/src/nimpak/grafting.nim @@ -2,8 +2,8 @@ # Simplified grafting infrastructure for external package integration import std/[tables, sets, strutils, json, os, times, sequtils, hashes, options] -import ../nip/types -import utils/resultutils +import ./types + import types/grafting_types export grafting_types @@ -39,33 +39,33 @@ proc initGraftingEngine*(configPath: string = ""): Result[GraftingEngine, string try: createDir(engine.cache.cacheDir) except OSError as e: - return Result[GraftingEngine, string](isOk: false, error: "Failed to create cache directory: " & e.msg) + return Result[GraftingEngine, string](isOk: false, errValue: "Failed to create cache directory: " & e.msg) - return Result[GraftingEngine, string](isOk: true, value: engine) + return Result[GraftingEngine, string](isOk: true, okValue: engine) proc registerAdapter*(engine: var GraftingEngine, adapter: PackageAdapter): Result[bool, string] = ## Register a package adapter with the grafting engine if adapter.name in engine.adapters: - return Result[bool, string](isOk: false, error: "Adapter already registered: " & adapter.name) + return Result[bool, string](isOk: false, errValue: "Adapter already registered: " & adapter.name) engine.adapters[adapter.name] = adapter echo "Registered grafting adapter: " & adapter.name - return Result[bool, string](isOk: true, value: true) + return Result[bool, string](isOk: true, okValue: true) proc graftPackage*(engine: var GraftingEngine, source: string, packageName: string): Result[GraftResult, string] = ## Graft a package from an external source if not engine.config.enabled: - return Result[GraftResult, string](isOk: false, error: "Grafting is disabled in configuration") + return Result[GraftResult, string](isOk: false, errValue: "Grafting is disabled in configuration") if source notin engine.adapters: - return Result[GraftResult, string](isOk: false, error: "Unknown grafting source: " & source) + return Result[GraftResult, string](isOk: false, errValue: "Unknown grafting source: " & source) let adapter = engine.adapters[source] if not adapter.enabled: - return Result[GraftResult, string](isOk: false, error: "Adapter disabled: " & source) + return Result[GraftResult, string](isOk: false, errValue: "Adapter disabled: " & source) # Create a simple result for now - let result = GraftResult( + let graftRes = GraftResult( success: true, packageId: packageName, metadata: GraftedPackageMetadata( @@ -89,7 +89,7 @@ proc graftPackage*(engine: var GraftingEngine, source: string, packageName: stri ) echo "Successfully grafted package: " & packageName - return ok[GraftResult](result) + return Result[GraftResult, string](isOk: true, okValue: graftRes) proc listGraftedPackages*(engine: GraftingEngine): seq[GraftedPackageMetadata] = ## List all grafted packages in cache @@ -129,11 +129,11 @@ method graftPackage*(adapter: PackageAdapter, packageName: string, cache: Grafti method validatePackage*(adapter: PackageAdapter, packageName: string): Result[bool, string] {.base.} = ## Base method for validating a package - can be overridden - return ok[bool](true) + return Result[bool, string](isOk: true, okValue: true) method getPackageInfo*(adapter: PackageAdapter, packageName: string): Result[JsonNode, string] {.base.} = ## Base method for getting package information - can be overridden - return ok[JsonNode](%*{"name": packageName, "adapter": adapter.name}) + return Result[JsonNode, string](isOk: true, okValue: %*{"name": packageName, "adapter": adapter.name}) # Utility functions proc calculateGraftHash*(packageName: string, source: string, timestamp: DateTime): string = diff --git a/src/nimpak/grafting_backup.nim b/src/nimpak/grafting_backup.nim index 84006c9..448da24 100644 --- a/src/nimpak/grafting_backup.nim +++ b/src/nimpak/grafting_backup.nim @@ -2,7 +2,7 @@ # Core grafting infrastructure for external package integration import std/[tables, sets, strutils, json, os, times, sequtils, hashes, options] -import ../nip/types +import ./types import utils/resultutils import types/grafting_types diff --git a/src/nimpak/grafting_working.nim b/src/nimpak/grafting_working.nim index 698e405..d6bb064 100644 --- a/src/nimpak/grafting_working.nim +++ b/src/nimpak/grafting_working.nim @@ -2,7 +2,7 @@ # Working grafting infrastructure for external package integration import std/[tables, strutils, json, os, times, sequtils, options, hashes] -import ../nip/types +import ./types import utils/resultutils import types/grafting_types diff --git a/src/nimpak/install.nim b/src/nimpak/install.nim index d5c4289..ac2bf9e 100644 --- a/src/nimpak/install.nim +++ b/src/nimpak/install.nim @@ -2,7 +2,7 @@ # Package installation orchestrator with atomic operations import std/[tables, sequtils, strformat] -import ../nip/types, dependency, transactions, filesystem, cas +import ./types, dependency, transactions, filesystem, cas type InstallStep* = object diff --git a/src/nimpak/install_manager.nim b/src/nimpak/install_manager.nim index 1d0f0cc..492fa91 100644 --- a/src/nimpak/install_manager.nim +++ b/src/nimpak/install_manager.nim @@ -1,9 +1,10 @@ -## install_manager.nim -## Unified installation system for NIP MVP -## Coordinates grafting from adapters and actual system installation +# install_manager.nim +# Unified installation system for NIP MVP +# Coordinates grafting from adapters and actual system installation import std/[os, times, json, strformat, strutils, tables, sequtils, algorithm] import cas +import ./types type InstallConfig* = object diff --git a/src/nimpak/migration.nim b/src/nimpak/migration.nim index c35feac..c8d872e 100644 --- a/src/nimpak/migration.nim +++ b/src/nimpak/migration.nim @@ -4,7 +4,7 @@ ## Task 42: Implement migration tools. import std/[os, strutils, strformat, json, tables, sequtils, times] -import ../nip/types +import ./types import cas import logging diff --git a/src/nimpak/npk_conversion.nim b/src/nimpak/npk_conversion.nim index e832fd3..682d073 100644 --- a/src/nimpak/npk_conversion.nim +++ b/src/nimpak/npk_conversion.nim @@ -2,7 +2,7 @@ # Enhanced NPK conversion with build hash integration import std/[strutils, json, os, times, tables, sequtils, strformat, algorithm, osproc] -import ../nip/types +import ./types import utils/resultutils import types/grafting_types diff --git a/src/nimpak/protection.nim b/src/nimpak/protection.nim index cbaf211..3349d1b 100644 --- a/src/nimpak/protection.nim +++ b/src/nimpak/protection.nim @@ -1,49 +1,35 @@ -## Read-Only Protection Manager -## -## This module implements the read-only protection system for CAS storage, -## ensuring immutability by default with controlled write access elevation. -## -## SECURITY NOTE: chmod-based protection is a UX feature, NOT a security feature! -## In user-mode (~/.local/share/nexus/cas/), chmod 555 only prevents ACCIDENTAL -## deletion/modification. A user who owns the files can bypass this trivially. -## -## Real security comes from: -## 1. Merkle tree verification (cryptographic integrity) -## 2. User namespaces (kernel-enforced read-only mounts during execution) -## 3. Root ownership (system-mode only: /var/lib/nexus/cas/) -## -## See docs/cas-security-architecture.md for full security model. +# Read-Only Protection Manager +# +# This module implements the read-only protection system for CAS storage, +# ensuring immutability by default with controlled write access elevation. +# +# SECURITY NOTE: chmod-based protection is a UX feature, NOT a security feature! +# In user-mode (~/.local/share/nexus/cas/), chmod 555 only prevents ACCIDENTAL +# deletion/modification. A user who owns the files can bypass this trivially. +# +# Real security comes from: +# 1. Merkle tree verification (cryptographic integrity) +# 2. User namespaces (kernel-enforced read-only mounts during execution) +# 3. Root ownership (system-mode only: /var/lib/nexus/cas/) +# +# See docs/cas-security-architecture.md for full security model. import std/[os, times, sequtils, strutils] import xxhash +import ./types type - # Result types for error handling - VoidResult*[E] = object - case isOk*: bool - of true: - discard - of false: - errValue*: E - - # Error types - ErrorCode* = enum - FileWriteError, FileReadError, UnknownError - - CasError* = object of CatchableError - code*: ErrorCode - objectHash*: string - ProtectionManager* = object - casPath*: string ## Path to CAS root directory - auditLog*: string ## Path to audit log file + casPath*: string # Path to CAS root directory + auditLog*: string # Path to audit log file - SecurityError* = object of CatchableError - code*: string - context*: string + SecurityEvent* = object + timestamp*: DateTime + eventType*: string + hash*: string + details*: string + severity*: string # "info", "warning", "critical" -proc ok*[E](dummy: typedesc[E]): VoidResult[E] = - VoidResult[E](isOk: true) proc newProtectionManager*(casPath: string): ProtectionManager = ## Create a new protection manager for the given CAS path @@ -69,35 +55,35 @@ proc logOperation*(pm: ProtectionManager, op: string, path: string, hash: string # (better to allow operation than to fail) discard -proc setReadOnly*(pm: ProtectionManager): VoidResult[CasError] = +proc setReadOnly*(pm: ProtectionManager): VoidResult[NimPakError] = ## Set CAS directory to read-only (chmod 555) try: setFilePermissions(pm.casPath, {fpUserRead, fpUserExec, fpGroupRead, fpGroupExec, fpOthersRead, fpOthersExec}) pm.logOperation("SET_READONLY", pm.casPath) - return ok(CasError) + return ok(NimPakError) except OSError as e: - return VoidResult[CasError](isOk: false, errValue: CasError( + return VoidResult[NimPakError](isOk: false, errValue: NimPakError( code: FileWriteError, msg: "Failed to set read-only permissions: " & e.msg )) -proc setWritable*(pm: ProtectionManager): VoidResult[CasError] = +proc setWritable*(pm: ProtectionManager): VoidResult[NimPakError] = ## Set CAS directory to writable (chmod 755) try: setFilePermissions(pm.casPath, {fpUserRead, fpUserWrite, fpUserExec, fpGroupRead, fpGroupExec, fpOthersRead, fpOthersExec}) pm.logOperation("SET_WRITABLE", pm.casPath) - return ok(CasError) + return ok(NimPakError) except OSError as e: - return VoidResult[CasError](isOk: false, errValue: CasError( + return VoidResult[NimPakError](isOk: false, errValue: NimPakError( code: FileWriteError, msg: "Failed to set writable permissions: " & e.msg )) -proc withWriteAccess*(pm: ProtectionManager, operation: proc()): VoidResult[CasError] = +proc withWriteAccess*(pm: ProtectionManager, operation: proc()): VoidResult[NimPakError] = ## Execute operation with temporary write access, then restore read-only ## This ensures atomic permission elevation and restoration var oldPerms: set[FilePermission] @@ -119,7 +105,7 @@ proc withWriteAccess*(pm: ProtectionManager, operation: proc()): VoidResult[CasE if not setReadOnlyResult.isOk: return setReadOnlyResult - return ok(CasError) + return ok(NimPakError) except Exception as e: # Ensure permissions restored even on error @@ -129,12 +115,12 @@ proc withWriteAccess*(pm: ProtectionManager, operation: proc()): VoidResult[CasE except: discard # Best effort to restore - return VoidResult[CasError](isOk: false, errValue: CasError( + return VoidResult[NimPakError](isOk: false, errValue: NimPakError( code: UnknownError, msg: "Write operation failed: " & e.msg )) -proc ensureReadOnly*(pm: ProtectionManager): VoidResult[CasError] = +proc ensureReadOnly*(pm: ProtectionManager): VoidResult[NimPakError] = ## Ensure CAS directory is in read-only state ## This should be called during initialization return pm.setReadOnly() @@ -152,18 +138,7 @@ proc verifyReadOnly*(pm: ProtectionManager): bool = # Merkle Integrity Verification # This is the PRIMARY security mechanism (not chmod) -type - IntegrityViolation* = object of CatchableError - hash*: string - expectedHash*: string - chunkPath*: string - SecurityEvent* = object - timestamp*: DateTime - eventType*: string - hash*: string - details*: string - severity*: string # "info", "warning", "critical" proc logSecurityEvent*(pm: ProtectionManager, event: SecurityEvent) = ## Log security events (integrity violations, tampering attempts, etc.) @@ -180,7 +155,7 @@ proc logSecurityEvent*(pm: ProtectionManager, event: SecurityEvent) = # If we can't write to audit log, at least try stderr stderr.writeLine("SECURITY EVENT: " & event.eventType & " - " & event.details) -proc verifyChunkIntegrity*(pm: ProtectionManager, data: seq[byte], expectedHash: string): VoidResult[CasError] = +proc verifyChunkIntegrity*(pm: ProtectionManager, data: seq[byte], expectedHash: string): VoidResult[NimPakError] = ## Verify chunk integrity by recalculating hash ## This is the PRIMARY security mechanism - always verify before use try: @@ -197,9 +172,9 @@ proc verifyChunkIntegrity*(pm: ProtectionManager, data: seq[byte], expectedHash: ) pm.logSecurityEvent(event) - return VoidResult[CasError](isOk: false, errValue: CasError( + return VoidResult[NimPakError](isOk: false, errValue: NimPakError( code: UnknownError, - objectHash: expectedHash, + context: "Object Hash: " & expectedHash, msg: "Chunk integrity violation detected! Expected: " & expectedHash & ", Got: " & calculatedHash & ". This chunk may be corrupted or tampered with." )) @@ -214,26 +189,26 @@ proc verifyChunkIntegrity*(pm: ProtectionManager, data: seq[byte], expectedHash: ) pm.logSecurityEvent(event) - return ok(CasError) + return ok(NimPakError) except Exception as e: - return VoidResult[CasError](isOk: false, errValue: CasError( + return VoidResult[NimPakError](isOk: false, errValue: NimPakError( code: UnknownError, msg: "Failed to verify chunk integrity: " & e.msg, - objectHash: expectedHash + context: "Object Hash: " & expectedHash )) -proc verifyChunkIntegrityFromFile*(pm: ProtectionManager, filePath: string, expectedHash: string): VoidResult[CasError] = +proc verifyChunkIntegrityFromFile*(pm: ProtectionManager, filePath: string, expectedHash: string): VoidResult[NimPakError] = ## Verify chunk integrity by reading file and checking hash try: let data = readFile(filePath) let byteData = data.toOpenArrayByte(0, data.len - 1).toSeq() return pm.verifyChunkIntegrity(byteData, expectedHash) except IOError as e: - return VoidResult[CasError](isOk: false, errValue: CasError( + return VoidResult[NimPakError](isOk: false, errValue: NimPakError( code: FileReadError, msg: "Failed to read chunk file for verification: " & e.msg, - objectHash: expectedHash + context: "Object Hash: " & expectedHash )) proc scanCASIntegrity*(pm: ProtectionManager, casPath: string): tuple[verified: int, corrupted: seq[string]] = diff --git a/src/nimpak/signature.nim b/src/nimpak/signature.nim index 38c2bb1..8932176 100644 --- a/src/nimpak/signature.nim +++ b/src/nimpak/signature.nim @@ -15,7 +15,7 @@ import std/[os, strutils, json, base64, tables, times, sets] import ed25519 -import ../nip/types +import ./types type SignatureManager* = object diff --git a/src/nimpak/transactions.nim b/src/nimpak/transactions.nim index 77d68e3..277f090 100644 --- a/src/nimpak/transactions.nim +++ b/src/nimpak/transactions.nim @@ -2,7 +2,7 @@ # Atomic transaction management system import std/[tables, strutils, json, times] -import ../nip/types +import ./types # Transaction management functions proc beginTransaction*(): Transaction = diff --git a/src/nimpak/types_fixed.nim b/src/nimpak/types_fixed.nim index 4c3e1f4..8a41ff7 100644 --- a/src/nimpak/types_fixed.nim +++ b/src/nimpak/types_fixed.nim @@ -1,7 +1,7 @@ -## NimPak Core Types -## -## This module defines the foundational data structures for the NimPak package -## management system, following NexusOS architectural principles. +# NimPak Core Types +# +# This module defines the foundational data structures for the NimPak package +# management system, following NexusOS architectural principles. import std/[times, tables, options, json] @@ -81,13 +81,24 @@ type suggestions*: seq[string] ErrorCode* = enum - PackageNotFound, DependencyConflict, ChecksumMismatch, - PermissionDenied, NetworkError, BuildFailed, - InvalidMetadata, AculViolation, CellNotFound, - FilesystemError, CasError, GraftError, - # CAS-specific errors - ObjectNotFound, CorruptedObject, StorageError, CompressionError, - FileReadError, FileWriteError, UnknownError + # Access Control + PermissionDenied, ElevationRequired, ReadOnlyViolation, + AculViolation, PolicyViolation, TrustViolation, SignatureInvalid, + + # Network & Transport + NetworkError, DownloadFailed, RepositoryUnavailable, TimeoutError, + + # Build & Dependency + BuildFailed, CompilationError, MissingDependency, DependencyConflict, + VersionMismatch, ChecksumMismatch, InvalidMetadata, + + # Storage & Integrity + FilesystemError, CasGeneralError, GraftError, PackageNotFound, CellNotFound, + ObjectNotFound, CorruptedObject, StorageError, CompressionError, StorageFull, + FileReadError, FileWriteError, PackageCorrupted, ReferenceIntegrityError, + + # Runtime & Lifecycle + TransactionFailed, RollbackFailed, GarbageCollectionFailed, UnknownError # ============================================================================= # Package Identification and Streams @@ -405,11 +416,7 @@ type deduplicationStatus*: string # "New" or "Reused" blake2bHash*: string # BLAKE2b hash for enhanced grafting - GraftResult* = object - fragment*: Fragment - extractedPath*: string - originalMetadata*: JsonNode - auditLog*: GraftAuditLog + # ============================================================================= # System Layers and Runtime Control diff --git a/src/nip/archives.nim b/src/nip/archives.nim deleted file mode 100644 index 00192db..0000000 --- a/src/nip/archives.nim +++ /dev/null @@ -1,83 +0,0 @@ -import std/[os, osproc, strformat, logging, tempfiles] -import zstd/compress -import zstd/decompress -import nip/manifest_parser - -type - ArchiveError* = object of CatchableError - -proc runCmd(cmd: string) = - let res = execCmdEx(cmd) - if res.exitCode != 0: - raise newException(ArchiveError, fmt"Command failed: {cmd}{'\n'}Output: {res.output}") - -proc createArchive*(manifest: PackageManifest, sourceDir: string, - outputFile: string) = - ## Create a .nip archive from a source directory and manifest. - ## The archive will contain: - ## - manifest.kdl - ## - files/ (content of sourceDir) - - info(fmt"Creating archive {outputFile} from {sourceDir}") - - let tempDir = createTempDir("nip_build_", "") - defer: removeDir(tempDir) - - # 1. Write manifest to temp root - let manifestPath = tempDir / "manifest.kdl" - writeFile(manifestPath, serializeManifestToKDL(manifest)) - - # 2. Copy source files to temp/files - let filesDir = tempDir / "files" - createDir(filesDir) - copyDirWithPermissions(sourceDir, filesDir) - - # 3. Create Tar (Uncompressed) - let tarFile = tempDir / "archive.tar" - let cmd = fmt"tar -C {tempDir.quoteShell} -cf {tarFile.quoteShell} manifest.kdl files" - runCmd(cmd) - - # 4. Compress with Zstd (Internal) - # TODO: Use streaming for large files - info "Compressing archive (Zstd Internal)..." - let content = readFile(tarFile) - # level 3 is default - let compressedSeq = compress(content, level = 3) - let compressedStr = cast[string](compressedSeq) - writeFile(outputFile, compressedStr) - - info(fmt"Archive created successfully: {outputFile}") - -proc extractArchive*(archivePath: string, targetDir: string) = - ## Extract a .nip archive to targetDir. - ## Decompress using internal Zstd, then untar using shell. - - info(fmt"Extracting archive {archivePath} to {targetDir}") - createDir(targetDir) - - # 1. Decompress (Internal) - info "Decompressing archive (Zstd Internal)..." - let content = readFile(archivePath) - let decompressedSeq = decompress(content) - let decompressedStr = cast[string](decompressedSeq) - - let tarFile = targetDir / "temp_extract.tar" - writeFile(tarFile, decompressedStr) - - # 2. Untar (Shell) - let cmd = fmt"tar -C {targetDir.quoteShell} -xf {tarFile.quoteShell}" - runCmd(cmd) - - removeFile(tarFile) - - info("Extraction complete") - -proc verifyArchive*(archivePath: string): bool = - ## Verify archive integrity (zstd check) - # TODO: Use library verify? For now try decompressing to void - try: - let content = readFile(archivePath) - discard decompress(content) - return true - except: - return false diff --git a/src/nip/cas.nim b/src/nip/cas.nim deleted file mode 100644 index e8a74eb..0000000 --- a/src/nip/cas.nim +++ /dev/null @@ -1,165 +0,0 @@ -## Content-Addressable Storage (CAS) system for NimPak -## -## This module provides the core functionality for storing and retrieving -## content-addressed objects using BLAKE2b-512 hashing (with future support for BLAKE3). -## Objects are stored in a sharded directory structure for scalability. - -import std/[os, strutils, times, posix] -import nimcrypto/hash -import nimcrypto/blake2 -import nip/types - -const - DefaultHashAlgorithm* = "blake2b-512" # Default hash algorithm - ShardingLevels* = 2 # Number of directory levels for sharding - -type - HashAlgorithm* = enum - Blake2b512 = "blake2b-512" - # Blake3 = "blake3" # Will be added when available in Nimble - - CasObject* = object - hash*: Multihash - size*: int64 - compressed*: bool - timestamp*: times.Time - -proc calculateHash*(data: string, algorithm: HashAlgorithm = Blake2b512): Multihash = - ## Calculate the hash of a string using the specified algorithm - case algorithm: - of Blake2b512: - let digest = blake2_512.digest(data) - var hexDigest = "" - for b in digest.data: - hexDigest.add(b.toHex(2).toLowerAscii()) - result = Multihash(hexDigest) - -proc calculateFileHash*(path: string, algorithm: HashAlgorithm = Blake2b512): Multihash = - ## Calculate the hash of a file using the specified algorithm - if not fileExists(path): - raise newException(IOError, "File not found: " & path) - - let data = readFile(path) - result = calculateHash(data, algorithm) - -proc getShardPath*(hash: Multihash, levels: int = ShardingLevels): string = - ## Get the sharded path for a hash - ## e.g., "ab/cd" for hash "abcdef123456..." - let hashStr = string(hash) - var parts: seq[string] = @[] - - for i in 0..// - result = casRoot / "refs" / refType / hash / refId - -proc addReference*(casRoot: string, hash: Multihash, refType, refId: string) = - ## Add a reference to a CAS object - ## refType: "npk", "nip", "nexter" - ## refId: Unique identifier for the reference (e.g. "package-name:version") - let path = getRefPath(casRoot, refType, string(hash), refId) - createDir(path.parentDir) - writeFile(path, "") # Empty file acts as reference - -proc removeReference*(casRoot: string, hash: Multihash, refType, refId: string) = - ## Remove a reference to a CAS object - let path = getRefPath(casRoot, refType, string(hash), refId) - if fileExists(path): - removeFile(path) - # Try to remove parent dir (hash dir) if empty - try: - removeDir(path.parentDir) - except: - discard - -proc hasReferences*(casRoot: string, hash: Multihash): bool = - ## Check if a CAS object has any references - # We need to check all refTypes - let refsDir = casRoot / "refs" - if not dirExists(refsDir): return false - - for kind, path in walkDir(refsDir): - if kind == pcDir: - let hashDir = path / string(hash) - if dirExists(hashDir): - # Check if directory is not empty - for _ in walkDir(hashDir): - return true - return false - -when isMainModule: - # Simple test - echo "Testing CAS functionality..." - let testData = "Hello, NexusOS with Content-Addressable Storage!" - let objHash = calculateHash(testData) - echo "Hash: ", string(objHash) - - # Test sharding - echo "Shard path: ", getShardPath(objHash) \ No newline at end of file diff --git a/src/nip/cli/resolve_command.nim b/src/nip/cli/resolve_command.nim deleted file mode 100644 index fd77237..0000000 --- a/src/nip/cli/resolve_command.nim +++ /dev/null @@ -1,328 +0,0 @@ -## Resolve Command - CLI Interface for Dependency Resolution -## -## This module provides the CLI interface for the dependency resolver, -## allowing users to resolve, explain, and inspect package dependencies. - -import strformat -import tables -import terminal - -# ============================================================================ -# Type Definitions -# ============================================================================ - -import ../resolver/orchestrator -import ../resolver/variant_types -import ../resolver/dependency_graph -import ../resolver/conflict_detection -import std/[options, times] - -type - VersionConstraint* = object - operator*: string - version*: string - -# ============================================================================ -# Helper Functions -# ============================================================================ - -proc loadRepositories*(): seq[Repository] = - ## Load repositories from configuration - result = @[ - Repository(name: "main", url: "https://packages.nexusos.org/main", priority: 100), - Repository(name: "community", url: "https://packages.nexusos.org/community", priority: 50) - ] - - - -proc parseVersionConstraint*(constraint: string): VersionConstraint = - ## Parse version constraint string - result = VersionConstraint(operator: "any", version: constraint) - -proc formatError*(msg: string): string = - ## Format error message - result = fmt"Error: {msg}" - - - -# ============================================================================ -# Command: nip resolve -# ============================================================================ - -proc resolveCommand*(args: seq[string]): int = - ## Handle 'nip resolve ' command - - if args.len < 1: - echo "Usage: nip resolve [constraint] [options]" - echo "" - echo "Options:" - echo " --use-flags= Comma-separated USE flags" - echo " --libc= C library (musl, glibc)" - echo " --allocator= Memory allocator (jemalloc, tcmalloc, default)" - echo " --json Output in JSON format" - return 1 - - let packageName = args[0] - var jsonOutput = false - - # Parse arguments - for arg in args[1..^1]: - if arg == "--json": - jsonOutput = true - - try: - # Initialize Orchestrator - let repos = loadRepositories() - let config = defaultConfig() - let orchestrator = newResolutionOrchestrator(repos, config) - - # Create demand (default for now) - let demand = VariantDemand( - packageName: packageName, - variantProfile: VariantProfile(hash: "any") - ) - - # Resolve - let result = orchestrator.resolve(packageName, "*", demand) - - if result.isOk: - let res = result.value - if jsonOutput: - echo fmt"""{{ - "success": true, - "package": "{packageName}", - "packageCount": {res.packageCount}, - "resolutionTime": {res.resolutionTime}, - "cacheHit": {res.cacheHit}, - "installOrder": [] -}}""" - else: - stdout.styledWrite(fgGreen, "✅ Resolution successful!\n") - echo "" - echo fmt"📦 Package: {packageName}" - echo fmt"⏱️ Time: {res.resolutionTime * 1000:.2f}ms" - echo fmt"📚 Packages: {res.packageCount}" - echo fmt"💾 Cache Hit: {res.cacheHit}" - echo "" - - echo "📋 Resolution Plan:" - for term in res.installOrder: - stdout.styledWrite(fgCyan, fmt" • {term.packageName}") - stdout.write(fmt" ({term.version})") - stdout.styledWrite(fgYellow, fmt" [{term.source}]") - echo "" - echo "" - - else: - let err = result.error - if jsonOutput: - echo fmt"""{{ - "success": false, - "error": "{err.details}" -}}""" - else: - stdout.styledWrite(fgRed, "❌ Resolution Failed!\n") - echo formatError(err) - - return if result.isOk: 0 else: 1 - - except Exception as e: - if jsonOutput: - echo fmt"""{{ - "success": false, - "error": "{e.msg}" -}}""" - else: - stdout.styledWrite(fgRed, "❌ Error!\n") - echo fmt"Error: {e.msg}" - return 1 - -# ============================================================================ -# Command: nip explain -# ============================================================================ - -proc explainCommand*(args: seq[string]): int = - ## Handle 'nip explain ' command - - if args.len < 1: - echo "Usage: nip explain [options]" - return 1 - - let packageName = args[0] - var jsonOutput = false - - for arg in args[1..^1]: - if arg == "--json": - jsonOutput = true - - try: - if jsonOutput: - echo fmt"""{{ - "success": true, - "package": "{packageName}", - "version": "1.0.0", - "variant": "default", - "buildHash": "blake3-abc123", - "source": "main", - "dependencyCount": 0, - "dependencies": [] -}}""" - else: - stdout.styledWrite(fgCyan, fmt"📖 Explaining resolution for: {packageName}\n") - echo "" - echo "Resolution explanation:" - echo fmt" • Package source: main" - echo fmt" • Version selected: 1.0.0" - echo fmt" • Variant: default" - echo fmt" • Dependencies: 0 packages" - echo "" - - return 0 - - except Exception as e: - if jsonOutput: - echo fmt"""{{ - "success": false, - "error": "{e.msg}" -}}""" - else: - stdout.styledWrite(fgRed, "❌ Error!\n") - echo fmt"Error: {e.msg}" - return 1 - -# ============================================================================ -# Command: nip conflicts -# ============================================================================ - -proc conflictsCommand*(args: seq[string]): int = - ## Handle 'nip conflicts' command - - var jsonOutput = false - - for arg in args: - if arg == "--json": - jsonOutput = true - - try: - if jsonOutput: - echo """{"success": true, "conflicts": []}""" - else: - stdout.styledWrite(fgGreen, "✅ No conflicts detected!\n") - echo "" - echo "All installed packages are compatible." - echo "" - - return 0 - - except Exception as e: - if jsonOutput: - echo fmt"""{{ - "success": false, - "error": "{e.msg}" -}}""" - else: - stdout.styledWrite(fgRed, "❌ Error!\n") - echo fmt"Error: {e.msg}" - return 1 - -# ============================================================================ -# Command: nip variants -# ============================================================================ - -proc variantsCommand*(args: seq[string]): int = - ## Handle 'nip variants ' command - - if args.len < 1: - echo "Usage: nip variants [options]" - return 1 - - let packageName = args[0] - var jsonOutput = false - - for arg in args[1..^1]: - if arg == "--json": - jsonOutput = true - - try: - if jsonOutput: - echo fmt"""{{ - "package": "{packageName}", - "variants": {{ - "useFlags": [ - {{"flag": "ssl", "description": "Enable SSL/TLS support", "default": false}}, - {{"flag": "http2", "description": "Enable HTTP/2 support", "default": false}} - ], - "libc": [ - {{"option": "musl", "description": "Lightweight C library", "default": true}}, - {{"option": "glibc", "description": "GNU C library", "default": false}} - ], - "allocator": [ - {{"option": "jemalloc", "description": "High-performance allocator", "default": true}}, - {{"option": "tcmalloc", "description": "Google's thread-caching allocator", "default": false}} - ] - }} -}}""" - else: - stdout.styledWrite(fgCyan, fmt"🎨 Available variants for: {packageName}\n") - echo "" - echo "USE flags:" - echo " • ssl (default) - Enable SSL/TLS support" - echo " • http2 - Enable HTTP/2 support" - echo "" - echo "libc options:" - echo " • musl (default) - Lightweight C library" - echo " • glibc - GNU C library" - echo "" - echo "Allocator options:" - echo " • jemalloc (default) - High-performance allocator" - echo " • tcmalloc - Google's thread-caching allocator" - echo "" - - return 0 - - except Exception as e: - if jsonOutput: - echo fmt"""{{ - "success": false, - "error": "{e.msg}" -}}""" - else: - stdout.styledWrite(fgRed, "❌ Error!\n") - echo fmt"Error: {e.msg}" - return 1 - -# ============================================================================ -# Main CLI Entry Point -# ============================================================================ - -when isMainModule: - import os - - let args = commandLineParams() - - if args.len == 0: - echo "NIP Dependency Resolver" - echo "" - echo "Usage: nip [args]" - echo "" - echo "Commands:" - echo " resolve - Resolve dependencies" - echo " explain - Explain resolution decisions" - echo " conflicts - Show detected conflicts" - echo " variants - Show available variants" - echo "" - quit(1) - - let command = args[0] - let commandArgs = args[1..^1] - - let exitCode = case command: - of "resolve": resolveCommand(commandArgs) - of "explain": explainCommand(commandArgs) - of "conflicts": conflictsCommand(commandArgs) - of "variants": variantsCommand(commandArgs) - else: - echo fmt"Unknown command: {command}" - 1 - - quit(exitCode) diff --git a/src/nip/commands/convert.nim b/src/nip/commands/convert.nim deleted file mode 100644 index 961f747..0000000 --- a/src/nip/commands/convert.nim +++ /dev/null @@ -1,85 +0,0 @@ -import std/[os, strutils, options] -import nimpak/packages -import nimpak/types -import nimpak/cas - -proc runConvertCommand*(args: seq[string]) = - if args.len < 2: - echo "Usage: nip convert " - quit(1) - - let graftedDir = args[1] - - # Load graft result metadata (simulate loading from graftedDir) - # In real implementation, this would parse graft metadata files - # Here, we simulate with placeholders for demonstration - - # TODO: Replace with actual loading/parsing of graft metadata - let dummyFragment = Fragment( - id: PackageId(name: "dummy", version: "0.1.0", stream: Stable), - source: Source( - url: "https://example.com/dummy-0.1.0.tar.gz", - hash: "blake2b-dummyhash", - hashAlgorithm: "blake2b", - sourceMethod: Http, - timestamp: now() - ), - dependencies: @[], - buildSystem: Custom, - metadata: PackageMetadata( - description: "Dummy package for conversion", - license: "MIT", - maintainer: "dummy@example.com", - tags: @[], - runtime: RuntimeProfile( - libc: Musl, - allocator: System, - systemdAware: false, - reproducible: true, - tags: @[] - ) - ), - acul: AculCompliance(required: false, membership: "", attribution: "", buildLog: "") - ) - - let dummyAuditLog = GraftAuditLog( - timestamp: now(), - source: Pacman, - packageName: "dummy", - version: "0.1.0", - downloadedFilename: "dummy-0.1.0.tar.gz", - archiveHash: "blake2b-dummyhash", - hashAlgorithm: "blake2b", - sourceOutput: "Simulated graft source output", - downloadUrl: none(string), - originalSize: 12345, - deduplicationStatus: "New" - ) - - let graftResult = GraftResult( - fragment: dummyFragment, - extractedPath: graftedDir, - originalMetadata: %*{}, - auditLog: dummyAuditLog - ) - - let convertResult = convertGraftToNpk(graftResult) - if convertResult.isErr: - echo "Conversion failed: ", convertResult.getError().msg - quit(1) - - let npk = convertResult.get() - - # Create archive path - let archivePath = graftedDir / (npk.metadata.id.name & "-" & npk.metadata.id.version & ".npk") - - let archiveResult = createNpkArchive(npk, archivePath) - if archiveResult.isErr: - echo "Failed to create NPK archive: ", archiveResult.getError().msg - quit(1) - - echo "Conversion successful. NPK archive created at: ", archivePath - -# Entry point for the command -when isMainModule: - runConvertCommand(commandLineParams()) diff --git a/src/nip/commands/graft.nim b/src/nip/commands/graft.nim deleted file mode 100644 index e69de29..0000000 diff --git a/src/nip/commands/lock.nim b/src/nip/commands/lock.nim deleted file mode 100644 index e69de29..0000000 diff --git a/src/nip/commands/verify.nim b/src/nip/commands/verify.nim deleted file mode 100644 index 85a4a40..0000000 --- a/src/nip/commands/verify.nim +++ /dev/null @@ -1,433 +0,0 @@ -## nip/commands/verify.nim -## Implementation of nip verify command for package integrity verification -## -## This module implements the nip verifyage|--all> command that provides -## comprehensive package integrity verification including hash and signature checks. - -import std/[os, strutils, times, json, sequtils, strformat, algorithm, tables] -import ../../nimpak/security/hash_verifier -import ../../nimpak/cli/core - -type - VerifyOptions* = object - target*: string # Package name or "--all" - checkSignatures*: bool # Verify digital signatures - checkHashes*: bool # Verify file hashes - verbose*: bool # Verbose output - outputFormat*: OutputFormat # Output format - autoRepair*: bool # Attempt automatic repair - showDetails*: bool # Show detailed verification info - - VerificationSummary* = object - totalPackages*: int - verifiedPackages*: int - failedPackages*: int - skippedPackages*: int - integrityPassed*: int - integrityFailed*: int - signaturesPassed*: int - signaturesFailed*: int - duration*: float - timestamp*: times.DateTime - - SimpleVerificationResult* = object - packageName*: string - success*: bool - message*: string - checkType*: string - duration*: float - -proc parseVerifyOptions*(args: seq[string]): VerifyOptions = - ## Parse nip verify command arguments - var options = VerifyOptions( - target: "", - checkSignatures: true, - checkHashes: true, - verbose: false, - outputFormat: OutputHuman, - autoRepair: false, - showDetails: false - ) - - if args.len == 0: - options.target = "--all" - return options - - var i = 0 - while i < args.len: - case args[i]: - of "--all": - options.target = "--all" - of "--no-signatures": - options.checkSignatures = false - of "--no-hashes": - options.checkHashes = false - of "--signatures-only": - options.checkHashes = false - options.checkSignatures = true - of "--hashes-only": - options.checkSignatures = false - options.checkHashes = true - of "--verbose", "-v": - options.verbose = true - of "--details": - options.showDetails = true - of "--auto-repair": - options.autoRepair = true - of "--output": - if i + 1 < args.len: - case args[i + 1].toLower(): - of "json": options.outputFormat = OutputJson - of "yaml": options.outputFormat = OutputYaml - of "kdl": options.outputFormat = OutputKdl - else: options.outputFormat = OutputHuman - i += 1 - else: - # Assume it's a package name - if options.target == "": - options.target = args[i] - i += 1 - - # Default to --all if no target specified - if options.target == "": - options.target = "--all" - - return options - -proc displayVerificationResult*(result: SimpleVerificationResult, options: VerifyOptions) = - ## Display a single verification result in human-readable format - let statusSymbol = if result.success: success("✅") else: error("❌") - - echo fmt"{statusSymbol} {result.checkType}: {result.packageName}" - - if not result.success or options.verbose: - echo fmt" {result.message}" - - if result.duration > 0.0: - echo fmt" Duration: {result.duration:.3f}s" - - echo "" - -proc displayVerificationSummary*(summary: VerificationSummary, options: VerifyOptions) = - ## Display verification summary - echo bold("📋 Verification Summary") - echo "=".repeat(40) - echo "Timestamp: " & $summary.timestamp - echo fmt"Duration: {summary.duration:.2f}s" - echo "" - - echo fmt"Packages: {summary.totalPackages} total, {summary.verifiedPackages} verified, {summary.failedPackages} failed" - - if options.checkHashes: - echo fmt"Hash Checks: {summary.integrityPassed} passed, {summary.integrityFailed} failed" - - if options.checkSignatures: - echo fmt"Signature Checks: {summary.signaturesPassed} passed, {summary.signaturesFailed} failed" - - echo "" - - # Overall status - let overallSuccess = summary.failedPackages == 0 - let statusSymbol = if overallSuccess: success("✅") else: error("❌") - let statusText = if overallSuccess: "PASSED" else: "FAILED" - - echo fmt"Overall Status: {statusSymbol} {statusText}" - -proc verifyPackageHash*(packageName: string, packagePath: string): SimpleVerificationResult = - ## Verify hash of a single package - let startTime = cpuTime() - - try: - if not fileExists(packagePath): - return SimpleVerificationResult( - packageName: packageName, - success: false, - message: fmt"Package file not found: {packagePath}", - checkType: "Hash", - duration: cpuTime() - startTime - ) - - # For now, just check if file exists and is readable - # In a real implementation, we would check against stored hash - let hashResult = computeFileHash(packagePath, HashBlake2b) - - return SimpleVerificationResult( - packageName: packageName, - success: true, - message: fmt"Package hash verified: {packageName}", - checkType: "Hash", - duration: cpuTime() - startTime - ) - - except Exception as e: - return SimpleVerificationResult( - packageName: packageName, - success: false, - message: fmt"Hash verification error: {e.msg}", - checkType: "Hash", - duration: cpuTime() - startTime - ) - -proc verifySpecificPackage*(packageName: string, options: VerifyOptions): seq[SimpleVerificationResult] = - ## Verify a specific package - var results: seq[SimpleVerificationResult] = @[] - - if options.verbose: - showInfo(fmt"Verifying package: {packageName}") - - # Find package file - let packagePath = fmt"/Programs/{packageName}/current/{packageName}.npk" - if not fileExists(packagePath): - # Try to find any version - let packageDir = fmt"/Programs/{packageName}" - if dirExists(packageDir): - var foundVersion = false - for versionDir in walkDirs(packageDir / "*"): - let versionPackagePath = versionDir / (packageName & ".npk") - if fileExists(versionPackagePath): - if options.checkHashes: - results.add(verifyPackageHash(packageName, versionPackagePath)) - foundVersion = true - break - - if not foundVersion: - results.add(SimpleVerificationResult( - packageName: packageName, - success: false, - message: fmt"Package file not found for {packageName}", - checkType: "Hash", - duration: 0.0 - )) - else: - results.add(SimpleVerificationResult( - packageName: packageName, - success: false, - message: fmt"Package directory not found: {packageName}", - checkType: "Hash", - duration: 0.0 - )) - else: - if options.checkHashes: - results.add(verifyPackageHash(packageName, packagePath)) - - return results - -proc verifyAllPackages*(options: VerifyOptions): seq[SimpleVerificationResult] = - ## Verify all installed packages - var results: seq[SimpleVerificationResult] = @[] - - if options.verbose: - showInfo("Verifying all installed packages...") - - # Scan /Programs directory for packages - if not dirExists("/Programs"): - results.add(SimpleVerificationResult( - packageName: "system", - success: false, - message: "/Programs directory not found", - checkType: "System", - duration: 0.0 - )) - return results - - var packageCount = 0 - for packageDir in walkDirs("/Programs/*"): - let packageName = extractFilename(packageDir) - packageCount += 1 - - if options.verbose: - showInfo(fmt"Verifying package {packageCount}: {packageName}") - - # Look for package files in version directories - var foundPackage = false - for versionDir in walkDirs(packageDir / "*"): - let packageFile = versionDir / (packageName & ".npk") - if fileExists(packageFile): - foundPackage = true - - # Hash verification - if options.checkHashes: - results.add(verifyPackageHash(packageName, packageFile)) - - break # Only verify the first found version - - if not foundPackage: - results.add(SimpleVerificationResult( - packageName: packageName, - success: false, - message: fmt"No package file found for {packageName}", - checkType: "Hash", - duration: 0.0 - )) - - return results - - - -proc calculateVerificationSummary*(results: seq[SimpleVerificationResult], duration: float): VerificationSummary = - ## Calculate verification summary from results - var summary = VerificationSummary( - totalPackages: 0, - verifiedPackages: 0, - failedPackages: 0, - skippedPackages: 0, - integrityPassed: 0, - integrityFailed: 0, - signaturesPassed: 0, - signaturesFailed: 0, - duration: duration, - timestamp: now() - ) - - var packageNames: seq[string] = @[] - - for result in results: - # Count unique packages - if result.packageName notin packageNames and result.packageName != "system": - packageNames.add(result.packageName) - - # Count by check type - if result.checkType == "Hash": - if result.success: - summary.integrityPassed += 1 - else: - summary.integrityFailed += 1 - elif result.checkType == "Signature": - if result.success: - summary.signaturesPassed += 1 - else: - summary.signaturesFailed += 1 - - summary.totalPackages = packageNames.len - - # Calculate verified/failed packages - var packageResults: Table[string, bool] = initTable[string, bool]() - for result in results: - if result.packageName != "system": - if result.packageName in packageResults: - # If any check fails for a package, mark it as failed - packageResults[result.packageName] = packageResults[result.packageName] and result.success - else: - packageResults[result.packageName] = result.success - - for packageName, success in packageResults.pairs: - if success: - summary.verifiedPackages += 1 - else: - summary.failedPackages += 1 - - return summary - -proc attemptAutoRepair*(results: seq[SimpleVerificationResult], options: VerifyOptions): seq[string] = - ## Attempt automatic repair of failed verifications - var repairActions: seq[string] = @[] - - if not options.autoRepair: - return repairActions - - showInfo("Attempting automatic repair of failed verifications...") - - for result in results: - if not result.success: - if result.checkType == "Hash": - # For hash failures, we could attempt to re-download or restore from backup - repairActions.add(fmt"Hash failure for {result.packageName}: Consider reinstalling package") - elif result.checkType == "Signature": - # For signature failures, we could attempt to update keyrings - repairActions.add(fmt"Signature failure for {result.packageName}: Consider updating keyrings") - - if repairActions.len > 0: - showWarning(fmt"Auto-repair identified {repairActions.len} potential actions (manual intervention required)") - for action in repairActions: - echo fmt" • {action}" - - return repairActions - -proc nipVerifyCommand*(args: seq[string]): CommandResult = - ## Main implementation of nip verify command - let startTime = cpuTime() - - try: - let options = parseVerifyOptions(args) - - if options.verbose: - showInfo(fmt"Starting verification: {options.target}") - if not options.checkHashes: - showInfo("Hash verification disabled") - if not options.checkSignatures: - showInfo("Signature verification disabled") - - # Run verification - var results: seq[SimpleVerificationResult] = @[] - - if options.target == "--all" or options.target == "all": - results = verifyAllPackages(options) - else: - results = verifySpecificPackage(options.target, options) - - let duration = cpuTime() - startTime - let summary = calculateVerificationSummary(results, duration) - - # Display results - case options.outputFormat: - of OutputHuman: - if options.verbose or results.len <= 20: # Show individual results for small sets - for result in results: - displayVerificationResult(result, options) - - displayVerificationSummary(summary, options) - - # Show auto-repair suggestions - if summary.failedPackages > 0: - let repairActions = attemptAutoRepair(results, options) - if repairActions.len == 0 and not options.autoRepair: - showInfo("Run with --auto-repair to attempt automatic fixes") - - else: - # Structured output - let outputData = %*{ - "summary": %*{ - "total_packages": summary.totalPackages, - "verified_packages": summary.verifiedPackages, - "failed_packages": summary.failedPackages, - "integrity_passed": summary.integrityPassed, - "integrity_failed": summary.integrityFailed, - "signatures_passed": summary.signaturesPassed, - "signatures_failed": summary.signaturesFailed, - "duration": summary.duration, - "timestamp": $summary.timestamp - }, - "results": results.mapIt(%*{ - "check_type": it.checkType, - "package_name": it.packageName, - "success": it.success, - "message": it.message, - "duration": it.duration - }), - "options": %*{ - "target": options.target, - "check_signatures": options.checkSignatures, - "check_hashes": options.checkHashes, - "auto_repair": options.autoRepair - } - } - outputData(outputData) - - # Log verification event (simplified) - if options.verbose: - if summary.failedPackages == 0: - showSuccess(fmt"Package verification completed: {summary.verifiedPackages}/{summary.totalPackages} packages verified") - else: - showWarning(fmt"Package verification completed with issues: {summary.failedPackages}/{summary.totalPackages} packages failed") - - # Return appropriate result - if summary.failedPackages == 0: - return successResult(fmt"Verification completed: {summary.verifiedPackages}/{summary.totalPackages} packages verified successfully") - else: - return errorResult(fmt"Verification failed: {summary.failedPackages}/{summary.totalPackages} packages failed verification", 1) - - except Exception as e: - return errorResult(fmt"Verify command failed: {e.msg}") - -# Export main functions -export nipVerifyCommand, VerifyOptions, parseVerifyOptions, VerificationSummary \ No newline at end of file diff --git a/src/nip/container.nim b/src/nip/container.nim deleted file mode 100644 index 7aaa6bd..0000000 --- a/src/nip/container.nim +++ /dev/null @@ -1,343 +0,0 @@ -## NEXTER Container Namespace and Isolation -## -## **Purpose:** -## Implements container namespace isolation for NEXTER containers. -## Handles network, PID, IPC, UTS namespace creation and management. -## Sets up environment variables and mounts CAS chunks. -## -## **Design Principles:** -## - Lightweight container isolation -## - Namespace-based process isolation -## - Read-only CAS chunk mounts -## - Capability-based security -## -## **Requirements:** -## - Requirement 5.4: Container isolation (network, PID, IPC, UTS) -## - Requirement 5.4: Environment variables and CAS mounts -## - Requirement 5.4: Capability configuration - -import std/[os, times, options, tables, osproc, strutils] -import nip/[nexter_manifest, namespace] - -type - ContainerNamespaceConfig* = object - ## Container namespace configuration - isolationType*: string ## "full", "network", "pid", "ipc", "uts" - capabilities*: seq[string] ## Linux capabilities - mounts*: seq[ContainerMount] - devices*: seq[DeviceSpec] ## Use DeviceSpec from manifest - environment*: Table[string, string] - - ContainerMount* = object - ## Container mount specification - source*: string - target*: string - mountType*: string ## "bind", "tmpfs", "devtmpfs" - readOnly*: bool - options*: seq[string] - - ContainerRuntime* = object - ## Container runtime state - id*: string - name*: string - manifest*: NEXTERManifest - config*: ContainerNamespaceConfig - pid*: int - startTime*: DateTime - status*: ContainerStatus - environment*: Table[string, string] - - ContainerStatus* = enum - ## Container lifecycle status - Created, - Running, - Paused, - Stopped, - Exited, - Error - - ContainerError* = object of CatchableError - code*: ContainerErrorCode - context*: string - suggestions*: seq[string] - - ContainerErrorCode* = enum - NamespaceCreationFailed, - MountFailed, - CapabilityFailed, - EnvironmentSetupFailed, - ProcessExecutionFailed, - InvalidConfiguration - -# ============================================================================ -# Container Configuration -# ============================================================================ - -proc createContainerConfig*(manifest: NEXTERManifest, - casRoot: string): ContainerNamespaceConfig = - ## Create container namespace configuration from manifest - ## - ## **Requirements:** - ## - Requirement 5.4: Create namespace config with isolation settings - ## - ## **Process:** - ## 1. Extract namespace configuration from manifest - ## 2. Set up environment variables - ## 3. Configure mounts for CAS chunks - ## 4. Configure capabilities - ## 5. Configure devices - - var config = ContainerNamespaceConfig( - isolationType: manifest.namespace.isolationType, - capabilities: manifest.namespace.capabilities, - mounts: @[], - devices: manifest.namespace.devices, - environment: manifest.environment - ) - - # Add CAS mount for read-only access to chunks - config.mounts.add(ContainerMount( - source: casRoot / "chunks", - target: "/Cas", - mountType: "bind", - readOnly: true, - options: @["rbind", "ro"] - )) - - # Add standard mounts - config.mounts.add(ContainerMount( - source: "tmpfs", - target: "/tmp", - mountType: "tmpfs", - readOnly: false, - options: @["size=1G", "mode=1777"] - )) - - config.mounts.add(ContainerMount( - source: "tmpfs", - target: "/run", - mountType: "tmpfs", - readOnly: false, - options: @["size=1G", "mode=0755"] - )) - - return config - -# ============================================================================ -# Namespace Setup -# ============================================================================ - -proc setupContainerNamespace*(config: ContainerNamespaceConfig): bool = - ## Set up container namespace isolation - ## - ## **Requirements:** - ## - Requirement 5.4: Create isolated namespaces - ## - ## **Process:** - ## 1. Create user namespace - ## 2. Create mount namespace - ## 3. Create PID namespace (if requested) - ## 4. Create network namespace (if requested) - ## 5. Create IPC namespace (if requested) - ## 6. Create UTS namespace (if requested) - - try: - # Validate isolation type - case config.isolationType: - of "full": - # Full isolation: all namespaces - # This would use unshare() with all namespace flags - discard - of "network": - # Network isolation only - discard - of "pid": - # PID isolation only - discard - of "ipc": - # IPC isolation only - discard - of "uts": - # UTS (hostname) isolation only - discard - else: - return false - - # In a real implementation, we would call unshare() here - # For now, just validate the configuration - return true - - except Exception as e: - return false - -# ============================================================================ -# Mount Management -# ============================================================================ - -proc setupContainerMounts*(config: ContainerNamespaceConfig): bool = - ## Set up container mounts - ## - ## **Requirements:** - ## - Requirement 5.4: Mount CAS chunks and configure filesystem - ## - ## **Process:** - ## 1. Create mount points - ## 2. Mount CAS chunks read-only - ## 3. Mount tmpfs for temporary storage - ## 4. Mount devices if configured - - try: - for mount in config.mounts: - # Create target directory if needed - if not dirExists(mount.target): - createDir(mount.target) - - # Mount based on type - case mount.mountType: - of "bind": - # Bind mount - let flags = if mount.readOnly: "rbind,ro" else: "rbind" - let cmd = "mount -o " & flags & " " & mount.source & " " & mount.target - let exitCode = execCmd(cmd) - if exitCode != 0: - return false - - of "tmpfs": - # Tmpfs mount - let options = mount.options.join(",") - let cmd = "mount -t tmpfs -o " & options & " tmpfs " & mount.target - let exitCode = execCmd(cmd) - if exitCode != 0: - return false - - of "devtmpfs": - # Device tmpfs mount - let options = mount.options.join(",") - let cmd = "mount -t devtmpfs -o " & options & " devtmpfs " & mount.target - let exitCode = execCmd(cmd) - if exitCode != 0: - return false - - else: - return false - - return true - - except Exception as e: - return false - -# ============================================================================ -# Capability Management -# ============================================================================ - -proc setupContainerCapabilities*(config: ContainerNamespaceConfig): bool = - ## Set up container capabilities - ## - ## **Requirements:** - ## - Requirement 5.4: Configure Linux capabilities - ## - ## **Process:** - ## 1. Parse capability list - ## 2. Drop unnecessary capabilities - ## 3. Keep only required capabilities - - try: - if config.capabilities.len == 0: - # No capabilities specified - drop all - let cmd = "setcap -r /proc/self/exe" - discard execCmd(cmd) - else: - # Set specific capabilities - let capString = config.capabilities.join(",") - let cmd = "setcap cap_" & capString & "+ep /proc/self/exe" - let exitCode = execCmd(cmd) - if exitCode != 0: - return false - - return true - - except Exception as e: - return false - -# ============================================================================ -# Environment Setup -# ============================================================================ - -proc setupContainerEnvironment*(config: ContainerNamespaceConfig): bool = - ## Set up container environment variables - ## - ## **Requirements:** - ## - Requirement 5.4: Configure environment variables - ## - ## **Process:** - ## 1. Parse environment variables from config - ## 2. Set environment variables in current process - ## 3. Prepare for child process inheritance - - try: - for key, value in config.environment.pairs: - putEnv(key, value) - - return true - - except Exception as e: - return false - -# ============================================================================ -# Container Runtime -# ============================================================================ - -var containerCounter = 0 - -proc createContainerRuntime*(name: string, manifest: NEXTERManifest, - config: ContainerNamespaceConfig): ContainerRuntime = - ## Create container runtime state - ## - ## **Requirements:** - ## - Requirement 5.4: Initialize container runtime - - containerCounter += 1 - return ContainerRuntime( - id: "container-" & $getTime().toUnix() & "-" & $containerCounter, - name: name, - manifest: manifest, - config: config, - pid: 0, - startTime: now(), - status: Created, - environment: config.environment - ) - -proc getContainerStatus*(runtime: ContainerRuntime): ContainerStatus = - ## Get current container status - if runtime.pid > 0: - # Check if process is still running - let cmd = "kill -0 " & $runtime.pid - let exitCode = execCmd(cmd) - if exitCode == 0: - return Running - else: - return Exited - else: - return runtime.status - -# ============================================================================ -# Formatting -# ============================================================================ - -proc `$`*(config: ContainerNamespaceConfig): string = - ## Format container config as string - result = "Container Config:\n" - result.add(" Isolation: " & config.isolationType & "\n") - result.add(" Capabilities: " & config.capabilities.join(", ") & "\n") - result.add(" Mounts: " & $config.mounts.len & "\n") - result.add(" Devices: " & $config.devices.len & "\n") - result.add(" Environment: " & $config.environment.len & " variables\n") - -proc `$`*(runtime: ContainerRuntime): string = - ## Format container runtime as string - result = "Container: " & runtime.name & "\n" - result.add(" ID: " & runtime.id & "\n") - result.add(" PID: " & $runtime.pid & "\n") - result.add(" Status: " & $runtime.status & "\n") - result.add(" Started: " & runtime.startTime.format("yyyy-MM-dd HH:mm:ss") & "\n") diff --git a/src/nip/container_management.nim b/src/nip/container_management.nim deleted file mode 100644 index 20a9a0e..0000000 --- a/src/nip/container_management.nim +++ /dev/null @@ -1,325 +0,0 @@ -## NEXTER Container Management -## -## **Purpose:** -## Implements container lifecycle management including stopping, status checking, -## log access, and restart functionality. -## -## **Design Principles:** -## - Clean lifecycle management -## - Non-blocking status queries -## - Comprehensive log access -## - Graceful shutdown with timeout -## -## **Requirements:** -## - Requirement 5.4: Container management (stop, status, logs, restart) - -import std/[os, times, options, tables, osproc, strutils, posix] -import nip/[nexter_manifest, container_startup] - -type - ContainerManager* = object - ## Container manager for lifecycle operations - containerName*: string - process*: ContainerProcess - config*: ContainerStartupConfig - logs*: seq[string] - createdAt*: DateTime - stoppedAt*: Option[DateTime] - - ContainerLog* = object - ## Container log entry - timestamp*: DateTime - level*: LogLevel - message*: string - - LogLevel* = enum - ## Log level - Debug, - Info, - Warning, - Error - - ContainerStats* = object - ## Container statistics - name*: string - status*: ProcessStatus - uptime*: int64 ## Seconds - pid*: int - memoryUsage*: int64 ## Bytes - cpuUsage*: float ## Percentage - restartCount*: int - - ContainerManagementError* = object of CatchableError - code*: ManagementErrorCode - context*: string - suggestions*: seq[string] - - ManagementErrorCode* = enum - ContainerNotRunning, - ProcessTerminationFailed, - LogAccessFailed, - StatsUnavailable, - RestartFailed - -# ============================================================================ -# Container Manager Creation -# ============================================================================ - -proc createContainerManager*(name: string, process: ContainerProcess, - config: ContainerStartupConfig): ContainerManager = - ## Create container manager - ## - ## **Requirements:** - ## - Requirement 5.4: Initialize container manager - - return ContainerManager( - containerName: name, - process: process, - config: config, - logs: @[], - createdAt: now(), - stoppedAt: none[DateTime]() - ) - -# ============================================================================ -# Container Stopping -# ============================================================================ - -proc stopContainer*(manager: var ContainerManager, timeout: int = 30): bool = - ## Stop container gracefully - ## - ## **Requirements:** - ## - Requirement 5.4: Stop running container - ## - ## **Process:** - ## 1. Send SIGTERM to process - ## 2. Wait for graceful shutdown (timeout seconds) - ## 3. Send SIGKILL if still running - ## 4. Update container status - - if manager.process.pid <= 0: - return false - - try: - # Send SIGTERM for graceful shutdown - let termResult = kill(Pid(manager.process.pid), SIGTERM) - if termResult != 0: - # Process might already be dead - manager.process.status = Stopped - manager.stoppedAt = some(now()) - return true - - # Wait for graceful shutdown - var waited = 0 - while waited < timeout: - # Check if process is still running - let checkResult = kill(Pid(manager.process.pid), 0) - if checkResult != 0: - # Process has exited - manager.process.status = Stopped - manager.stoppedAt = some(now()) - return true - - # Sleep a bit and try again - sleep(100) - waited += 100 - - # Process didn't stop gracefully, force kill - let killResult = kill(Pid(manager.process.pid), SIGKILL) - if killResult == 0: - manager.process.status = Stopped - manager.stoppedAt = some(now()) - return true - else: - return false - - except Exception as e: - return false - -proc restartContainer*(manager: var ContainerManager): bool = - ## Restart container - ## - ## **Requirements:** - ## - Requirement 5.4: Restart container - ## - ## **Process:** - ## 1. Stop current container - ## 2. Start new container with same config - ## 3. Update manager state - - # Stop current container - if not stopContainer(manager): - return false - - # Wait a bit for cleanup - sleep(500) - - # Start new container - let newProcess = startContainer(manager.config) - - if newProcess.status == Failed: - return false - - # Update manager - manager.process = newProcess - manager.stoppedAt = none[DateTime]() - - return true - -# ============================================================================ -# Container Status -# ============================================================================ - -proc getContainerStatus*(manager: ContainerManager): ProcessStatus = - ## Get current container status - ## - ## **Requirements:** - ## - Requirement 5.4: Query container status - - if manager.process.pid <= 0: - return manager.process.status - - # Check if process is still running - try: - let checkResult = kill(Pid(manager.process.pid), 0) - if checkResult == 0: - return Running - else: - return Stopped - except: - return Stopped - -proc getContainerStats*(manager: ContainerManager): ContainerStats = - ## Get container statistics - ## - ## **Requirements:** - ## - Requirement 5.4: Get container statistics - ## - ## **Returns:** - ## Container statistics including uptime, memory, CPU usage - - let status = getContainerStatus(manager) - let uptime = if status == Running: - (now() - manager.createdAt).inSeconds - else: - if manager.stoppedAt.isSome: - (manager.stoppedAt.get() - manager.createdAt).inSeconds - else: - 0 - - return ContainerStats( - name: manager.containerName, - status: status, - uptime: uptime, - pid: manager.process.pid, - memoryUsage: 0, # Would require /proc parsing in real implementation - cpuUsage: 0.0, # Would require /proc parsing in real implementation - restartCount: 0 # Would need to track restarts - ) - -proc isContainerRunning*(manager: ContainerManager): bool = - ## Check if container is running - ## - ## **Requirements:** - ## - Requirement 5.4: Query running status - - return getContainerStatus(manager) == Running - -# ============================================================================ -# Container Logs -# ============================================================================ - -proc addLog*(manager: var ContainerManager, level: LogLevel, message: string) = - ## Add log entry to container - ## - ## **Requirements:** - ## - Requirement 5.4: Log container operations - - manager.logs.add(message) - -proc getContainerLogs*(manager: ContainerManager, level: LogLevel = Debug): seq[string] = - ## Get container logs - ## - ## **Requirements:** - ## - Requirement 5.4: Access container logs - ## - ## **Returns:** - ## All logs at or above specified level - - return manager.logs - -proc clearContainerLogs*(manager: var ContainerManager) = - ## Clear container logs - ## - ## **Requirements:** - ## - Requirement 5.4: Manage container logs - - manager.logs = @[] - -proc getLastLogs*(manager: ContainerManager, count: int = 10): seq[string] = - ## Get last N log entries - ## - ## **Requirements:** - ## - Requirement 5.4: Access recent logs - - let startIdx = max(0, manager.logs.len - count) - return manager.logs[startIdx..^1] - -# ============================================================================ -# Container Uptime -# ============================================================================ - -proc getContainerUptime*(manager: ContainerManager): int64 = - ## Get container uptime in seconds - ## - ## **Requirements:** - ## - Requirement 5.4: Query container uptime - - let stats = getContainerStats(manager) - return stats.uptime - -proc getContainerUptimeFormatted*(manager: ContainerManager): string = - ## Get container uptime as formatted string - ## - ## **Requirements:** - ## - Requirement 5.4: Format uptime for display - - let uptime = getContainerUptime(manager) - let days = uptime div 86400 - let hours = (uptime mod 86400) div 3600 - let minutes = (uptime mod 3600) div 60 - let seconds = uptime mod 60 - - if days > 0: - return $days & "d " & $hours & "h " & $minutes & "m" - elif hours > 0: - return $hours & "h " & $minutes & "m " & $seconds & "s" - elif minutes > 0: - return $minutes & "m " & $seconds & "s" - else: - return $seconds & "s" - -# ============================================================================ -# Formatting -# ============================================================================ - -proc `$`*(manager: ContainerManager): string = - ## Format container manager as string - let status = getContainerStatus(manager) - let uptime = getContainerUptimeFormatted(manager) - - result = "Container: " & manager.containerName & "\n" - result.add(" Status: " & $status & "\n") - result.add(" PID: " & $manager.process.pid & "\n") - result.add(" Uptime: " & uptime & "\n") - result.add(" Logs: " & $manager.logs.len & " entries\n") - -proc `$`*(stats: ContainerStats): string = - ## Format container stats as string - result = "Container Stats: " & stats.name & "\n" - result.add(" Status: " & $stats.status & "\n") - result.add(" PID: " & $stats.pid & "\n") - result.add(" Uptime: " & $stats.uptime & "s\n") - result.add(" Memory: " & $(stats.memoryUsage div 1024 div 1024) & "MB\n") - result.add(" CPU: " & formatFloat(stats.cpuUsage, ffDecimal, 2) & "%\n") - result.add(" Restarts: " & $stats.restartCount & "\n") diff --git a/src/nip/container_startup.nim b/src/nip/container_startup.nim deleted file mode 100644 index fa53cd7..0000000 --- a/src/nip/container_startup.nim +++ /dev/null @@ -1,379 +0,0 @@ -## NEXTER Container Startup and Lifecycle Management -## -## **Purpose:** -## Implements container startup, execution, and lifecycle management. -## Handles process creation, working directory setup, user switching, and command execution. -## -## **Design Principles:** -## - Lightweight process management -## - Proper environment setup -## - User and working directory configuration -## - Entrypoint and command execution -## -## **Requirements:** -## - Requirement 5.4: Container startup with configuration -## - Requirement 5.4: Working directory and user setup -## - Requirement 5.4: Command execution - -import std/[os, times, options, tables, osproc, strutils, posix] -import nip/[nexter_manifest, container] - -type - ContainerStartupConfig* = object - ## Container startup configuration - command*: seq[string] - workingDir*: string - user*: Option[string] - entrypoint*: Option[string] - environment*: Table[string, string] - - ContainerProcess* = object - ## Container process information - pid*: int - startTime*: DateTime - status*: ProcessStatus - exitCode*: Option[int] - output*: string - error*: string - - ProcessStatus* = enum - ## Process lifecycle status - Starting, - Running, - Paused, - Stopped, - Exited, - Failed - - ContainerStartupError* = object of CatchableError - code*: StartupErrorCode - context*: string - suggestions*: seq[string] - - StartupErrorCode* = enum - InvalidCommand, - WorkingDirectoryNotFound, - UserNotFound, - ProcessExecutionFailed, - EnvironmentSetupFailed, - EntrypointNotFound - -# ============================================================================ -# Startup Configuration -# ============================================================================ - -proc createStartupConfig*(manifest: NEXTERManifest): ContainerStartupConfig = - ## Create startup configuration from manifest - ## - ## **Requirements:** - ## - Requirement 5.4: Extract startup configuration from manifest - - return ContainerStartupConfig( - command: manifest.startup.command, - workingDir: manifest.startup.workingDir, - user: manifest.startup.user, - entrypoint: manifest.startup.entrypoint, - environment: manifest.environment - ) - -# ============================================================================ -# Startup Process -# ============================================================================ - -proc validateStartupConfig*(config: ContainerStartupConfig): bool = - ## Validate startup configuration - ## - ## **Requirements:** - ## - Requirement 5.4: Validate configuration before startup - ## - ## **Checks:** - ## 1. Command is not empty - ## 2. Working directory exists or can be created - ## 3. User exists (if specified) - ## 4. Entrypoint exists (if specified) - - # Check command - if config.command.len == 0: - return false - - # Check working directory - if config.workingDir.len > 0 and not dirExists(config.workingDir): - # Try to create it - try: - createDir(config.workingDir) - except: - return false - - # Check user (if specified) - if config.user.isSome: - let username = config.user.get() - # In a real implementation, we would check if user exists - # For now, just validate it's not empty - if username.len == 0: - return false - - # Check entrypoint (if specified) - if config.entrypoint.isSome: - let entrypoint = config.entrypoint.get() - if entrypoint.len == 0: - return false - - return true - -proc setupWorkingDirectory*(config: ContainerStartupConfig): bool = - ## Set up working directory for container - ## - ## **Requirements:** - ## - Requirement 5.4: Set working directory - ## - ## **Process:** - ## 1. Create working directory if needed - ## 2. Change to working directory - ## 3. Verify directory is accessible - - try: - if config.workingDir.len == 0: - return true - - # Create directory if needed - if not dirExists(config.workingDir): - createDir(config.workingDir) - - # Change to working directory - setCurrentDir(config.workingDir) - - return true - - except Exception as e: - return false - -proc setupUser*(config: ContainerStartupConfig): bool = - ## Set up user for container process - ## - ## **Requirements:** - ## - Requirement 5.4: Switch to specified user - ## - ## **Process:** - ## 1. Get user ID from username - ## 2. Switch to user (if not already that user) - ## 3. Verify user switch successful - - try: - if config.user.isNone: - return true - - let username = config.user.get() - if username.len == 0: - return true - - # In a real implementation, we would use getpwnam() to get user info - # and setuid() to switch users. For now, just validate. - # This requires elevated privileges to work properly. - - return true - - except Exception as e: - return false - -proc setupEnvironment*(config: ContainerStartupConfig): bool = - ## Set up environment variables for container - ## - ## **Requirements:** - ## - Requirement 5.4: Configure environment variables - ## - ## **Process:** - ## 1. Clear existing environment (optional) - ## 2. Set environment variables from config - ## 3. Verify environment is set - - try: - for key, value in config.environment.pairs: - putEnv(key, value) - - return true - - except Exception as e: - return false - -# ============================================================================ -# Container Execution -# ============================================================================ - -proc startContainer*(config: ContainerStartupConfig): ContainerProcess = - ## Start container with given configuration - ## - ## **Requirements:** - ## - Requirement 5.4: Start container process - ## - ## **Process:** - ## 1. Validate configuration - ## 2. Set up working directory - ## 3. Set up user - ## 4. Set up environment - ## 5. Execute command or entrypoint - ## 6. Return process information - - let startTime = now() - - # Validate configuration - if not validateStartupConfig(config): - return ContainerProcess( - pid: -1, - startTime: startTime, - status: Failed, - exitCode: some(-1), - output: "", - error: "Invalid startup configuration" - ) - - # Set up working directory - if not setupWorkingDirectory(config): - return ContainerProcess( - pid: -1, - startTime: startTime, - status: Failed, - exitCode: some(-1), - output: "", - error: "Failed to set up working directory" - ) - - # Set up user - if not setupUser(config): - return ContainerProcess( - pid: -1, - startTime: startTime, - status: Failed, - exitCode: some(-1), - output: "", - error: "Failed to set up user" - ) - - # Set up environment - if not setupEnvironment(config): - return ContainerProcess( - pid: -1, - startTime: startTime, - status: Failed, - exitCode: some(-1), - output: "", - error: "Failed to set up environment" - ) - - # Determine command to execute - var cmdToExecute: seq[string] = @[] - if config.entrypoint.isSome: - cmdToExecute.add(config.entrypoint.get()) - if config.command.len > 0: - cmdToExecute.add(config.command) - - if cmdToExecute.len == 0: - return ContainerProcess( - pid: -1, - startTime: startTime, - status: Failed, - exitCode: some(-1), - output: "", - error: "No command or entrypoint specified" - ) - - # Execute command - try: - let process = startProcess(cmdToExecute[0], args=cmdToExecute[1..^1]) - let pid = process.processID() - - return ContainerProcess( - pid: pid, - startTime: startTime, - status: Running, - exitCode: none[int](), - output: "", - error: "" - ) - - except Exception as e: - return ContainerProcess( - pid: -1, - startTime: startTime, - status: Failed, - exitCode: some(-1), - output: "", - error: "Process execution failed: " & e.msg - ) - -# ============================================================================ -# Process Management -# ============================================================================ - -proc waitForContainer*(process: var ContainerProcess): int = - ## Wait for container process to complete - ## - ## **Requirements:** - ## - Requirement 5.4: Wait for process completion - ## - ## **Process:** - ## 1. Wait for process to exit - ## 2. Capture exit code - ## 3. Update process status - - if process.pid <= 0: - return -1 - - try: - # In a real implementation, we would use waitpid() to wait for the process - # For now, just return a placeholder - process.status = Exited - process.exitCode = some(0) - return 0 - - except Exception as e: - process.status = Failed - process.exitCode = some(-1) - return -1 - -proc getContainerLogs*(process: ContainerProcess): string = - ## Get container process logs - ## - ## **Requirements:** - ## - Requirement 5.4: Access container logs - ## - ## **Returns:** - ## Combined stdout and stderr from process - - return process.output & process.error - -proc getContainerStatus*(process: ContainerProcess): ProcessStatus = - ## Get current container process status - ## - ## **Requirements:** - ## - Requirement 5.4: Query process status - - if process.pid <= 0: - return process.status - - # In a real implementation, we would check if process is still running - # using kill(pid, 0) or similar - return process.status - -# ============================================================================ -# Formatting -# ============================================================================ - -proc `$`*(config: ContainerStartupConfig): string = - ## Format startup config as string - result = "Container Startup Config:\n" - result.add(" Command: " & config.command.join(" ") & "\n") - result.add(" Working Dir: " & config.workingDir & "\n") - if config.user.isSome: - result.add(" User: " & config.user.get() & "\n") - if config.entrypoint.isSome: - result.add(" Entrypoint: " & config.entrypoint.get() & "\n") - result.add(" Environment: " & $config.environment.len & " variables\n") - -proc `$`*(process: ContainerProcess): string = - ## Format container process as string - result = "Container Process:\n" - result.add(" PID: " & $process.pid & "\n") - result.add(" Status: " & $process.status & "\n") - result.add(" Started: " & process.startTime.format("yyyy-MM-dd HH:mm:ss") & "\n") - if process.exitCode.isSome: - result.add(" Exit Code: " & $process.exitCode.get() & "\n") diff --git a/src/nip/doctor.nim b/src/nip/doctor.nim deleted file mode 100644 index ad787ec..0000000 --- a/src/nip/doctor.nim +++ /dev/null @@ -1,516 +0,0 @@ -## nip/doctor.nim -## Implementationnip doctor command for system health checks -## -## This module implements the nip doctor command that provides comprehensive -## system health diagnostics including integrity checks, keyring health, and more. - -import std/[os, strutils, times, json, sequtils, strformat, algorithm, tables] -import ../nimpak/security/[integrity_monitor, hash_verifier, signature_verifier_working, keyring_manager, event_logger] -import ../nimpak/cli/core - -type - DoctorOptions* = object - integrityCheck*: bool - keyringCheck*: bool - performanceCheck*: bool - autoRepair*: bool - verbose*: bool - outputFormat*: OutputFormat - - HealthCheckCategory* = enum - HealthIntegrity = "integrity" - HealthKeyring = "keyring" - HealthPerformance = "performance" - HealthConfiguration = "configuration" - HealthStorage = "storage" - - SystemHealthReport* = object - overallStatus*: string - categories*: seq[CategoryHealth] - recommendations*: seq[string] - statistics*: JsonNode - timestamp*: times.DateTime - duration*: float - - CategoryHealth* = object - category*: HealthCheckCategory - status*: string - score*: float - issues*: seq[string] - details*: JsonNode - -# Helper functions for disk space and directory size -proc getFreeDiskSpace*(path: string): int64 = - ## Get free disk space for a path (placeholder implementation) - try: - # This is a simplified implementation - # In a real implementation, you'd use system calls - return 10_000_000_000 # 10GB placeholder - except: - return 0 - -proc getDirSize*(path: string): int64 = - ## Get total size of directory (placeholder implementation) - try: - var totalSize: int64 = 0 - for file in walkDirRec(path): - totalSize += getFileSize(file) - return totalSize - except: - return 0 - -proc parseDoctorOptions*(args: seq[string]): DoctorOptions = - ## Parse nip doctor command arguments - var options = DoctorOptions( - integrityCheck: false, - keyringCheck: false, - performanceCheck: false, - autoRepair: false, - verbose: false, - outputFormat: OutputHuman - ) - - # If no specific checks requested, enable all - if args.len == 0: - options.integrityCheck = true - options.keyringCheck = true - options.performanceCheck = true - return options - - var i = 0 - while i < args.len: - case args[i]: - of "--integrity": - options.integrityCheck = true - of "--keyring": - options.keyringCheck = true - of "--performance": - options.performanceCheck = true - of "--auto-repair": - options.autoRepair = true - of "--verbose", "-v": - options.verbose = true - of "--output": - if i + 1 < args.len: - case args[i + 1].toLower(): - of "json": options.outputFormat = OutputJson - of "yaml": options.outputFormat = OutputYaml - of "kdl": options.outputFormat = OutputKdl - else: options.outputFormat = OutputHuman - i += 1 - else: - # If no specific flags, enable all checks - if not (options.integrityCheck or options.keyringCheck or options.performanceCheck): - options.integrityCheck = true - options.keyringCheck = true - options.performanceCheck = true - i += 1 - - return options - -proc runIntegrityHealthCheck*(options: DoctorOptions): CategoryHealth = - ## Run comprehensive integrity health check - let startTime = cpuTime() - - var categoryHealth = CategoryHealth( - category: HealthIntegrity, - status: "unknown", - score: 0.0, - issues: @[], - details: newJObject() - ) - - try: - if options.verbose: - showInfo("Running integrity health check...") - - # Initialize integrity monitor - let monitor = newIntegrityMonitor(getDefaultIntegrityConfig()) - let integrityResult = runIntegrityHealthCheck(monitor) - - # Extract statistics from integrity check - let stats = integrityResult.details - categoryHealth.details = stats - - # Determine health score based on results - let totalPackages = stats["statistics"]["packages_checked"].getInt() - let integrityPassed = stats["statistics"]["integrity_passed"].getInt() - let signaturesPassed = stats["statistics"]["signatures_verified"].getInt() - let totalIssues = stats["total_issues"].getInt() - - if totalPackages > 0: - let integrityScore = integrityPassed.float / totalPackages.float - let signatureScore = if totalPackages > 0: signaturesPassed.float / totalPackages.float else: 1.0 - categoryHealth.score = (integrityScore + signatureScore) / 2.0 - else: - categoryHealth.score = 0.0 - - # Determine status - if integrityResult.success and totalIssues == 0: - categoryHealth.status = "healthy" - elif totalIssues <= 5: # Configurable threshold - categoryHealth.status = "warning" - categoryHealth.issues.add(fmt"Found {totalIssues} integrity issues") - else: - categoryHealth.status = "critical" - categoryHealth.issues.add(fmt"Found {totalIssues} integrity issues (above threshold)") - - # Add specific issues from the integrity check - if stats.hasKey("issues"): - for issue in stats["issues"]: - categoryHealth.issues.add(issue.getStr()) - - if options.verbose: - showSuccess(fmt"Integrity check completed: {categoryHealth.status}") - - except Exception as e: - categoryHealth.status = "error" - categoryHealth.score = 0.0 - categoryHealth.issues.add(fmt"Integrity check failed: {e.msg}") - errorLog(fmt"Integrity health check error: {e.msg}") - - return categoryHealth - -proc runKeyringHealthCheck*(options: DoctorOptions): CategoryHealth = - ## Run keyring health check - var categoryHealth = CategoryHealth( - category: HealthKeyring, - status: "unknown", - score: 0.0, - issues: @[], - details: newJObject() - ) - - try: - if options.verbose: - showInfo("Running keyring health check...") - - # Initialize keyring manager - let config = getDefaultKeyringConfig() - var keyringManager = newKeyringManager(config) - keyringManager.loadAllKeyrings() - - # Get keyring statistics - let stats = keyringManager.getKeyringStatistics() - categoryHealth.details = stats - - let totalKeys = stats["total_keys"].getInt() - let validKeys = stats["valid_keys"].getInt() - let expiredKeys = stats["expired_keys"].getInt() - let revokedKeys = stats["revoked_keys"].getInt() - - # Calculate health score - if totalKeys > 0: - categoryHealth.score = validKeys.float / totalKeys.float - else: - categoryHealth.score = 0.0 - categoryHealth.issues.add("No keys found in keyring") - - # Determine status - if expiredKeys == 0 and revokedKeys == 0 and totalKeys > 0: - categoryHealth.status = "healthy" - elif expiredKeys > 0 or revokedKeys > 0: - categoryHealth.status = "warning" - if expiredKeys > 0: - categoryHealth.issues.add(fmt"Found {expiredKeys} expired keys") - if revokedKeys > 0: - categoryHealth.issues.add(fmt"Found {revokedKeys} revoked keys") - else: - categoryHealth.status = "critical" - - if options.verbose: - showSuccess(fmt"Keyring check completed: {categoryHealth.status}") - - except Exception as e: - categoryHealth.status = "error" - categoryHealth.score = 0.0 - categoryHealth.issues.add(fmt"Keyring check failed: {e.msg}") - errorLog(fmt"Keyring health check error: {e.msg}") - - return categoryHealth - -proc runPerformanceHealthCheck*(options: DoctorOptions): CategoryHealth = - ## Run performance health check - var categoryHealth = CategoryHealth( - category: HealthPerformance, - status: "unknown", - score: 0.0, - issues: @[], - details: newJObject() - ) - - try: - if options.verbose: - showInfo("Running performance health check...") - - # Check disk space - let programsSpace = getFreeDiskSpace("/Programs") - let cacheSpace = getFreeDiskSpace("/var/cache/nip") - - # Check package count and sizes - var packageCount = 0 - var totalSize: int64 = 0 - - if dirExists("/Programs"): - for packageDir in walkDirs("/Programs/*"): - inc packageCount - totalSize += getDirSize(packageDir) - - # Performance metrics - let stats = %*{ - "package_count": packageCount, - "total_size_bytes": totalSize, - "programs_free_space": programsSpace, - "cache_free_space": cacheSpace, - "avg_package_size": if packageCount > 0: totalSize div packageCount else: 0 - } - - categoryHealth.details = stats - - # Calculate performance score based on available space and package efficiency - var score = 1.0 - - # Penalize if low disk space - if programsSpace < 1_000_000_000: # Less than 1GB - score -= 0.3 - categoryHealth.issues.add("Low disk space in /Programs") - - if cacheSpace < 500_000_000: # Less than 500MB - score -= 0.2 - categoryHealth.issues.add("Low disk space in cache") - - categoryHealth.score = max(0.0, score) - - # Determine status - if categoryHealth.issues.len == 0: - categoryHealth.status = "healthy" - elif categoryHealth.score > 0.7: - categoryHealth.status = "warning" - else: - categoryHealth.status = "critical" - - if options.verbose: - showSuccess(fmt"Performance check completed: {categoryHealth.status}") - - except Exception as e: - categoryHealth.status = "error" - categoryHealth.score = 0.0 - categoryHealth.issues.add(fmt"Performance check failed: {e.msg}") - errorLog(fmt"Performance health check error: {e.msg}") - - return categoryHealth - -proc generateRecommendations*(categories: seq[CategoryHealth]): seq[string] = - ## Generate recommendations based on health check results - var recommendations: seq[string] = @[] - - for category in categories: - case category.category: - of HealthIntegrity: - if category.status == "critical": - recommendations.add("Run 'nip verify --all --auto-repair' to fix integrity issues") - elif category.status == "warning": - recommendations.add("Consider running 'nip verify --all' to check specific issues") - - of HealthKeyring: - if category.status == "warning" or category.status == "critical": - recommendations.add("Update keyring with 'nip key update' to refresh expired keys") - recommendations.add("Remove revoked keys with 'nip key cleanup'") - - of HealthPerformance: - if category.status == "critical": - recommendations.add("Free up disk space or move packages to larger storage") - recommendations.add("Run 'nip clean' to remove unnecessary cache files") - elif category.status == "warning": - recommendations.add("Consider cleaning package cache with 'nip clean --cache'") - - else: - discard - - if recommendations.len == 0: - recommendations.add("System health is good - no immediate actions required") - - return recommendations - -proc displayHealthReport*(report: SystemHealthReport, options: DoctorOptions) = - ## Display health report in human-readable format - echo bold("🩺 NimPak System Health Report") - echo "=".repeat(50) - echo "Generated: " & report.timestamp.format("yyyy-MM-dd HH:mm:ss") - echo fmt"Duration: {report.duration:.2f}s" - echo "" - - # Overall status - let statusSymbol = case report.overallStatus: - of "healthy": success("✅") - of "warning": warning("⚠️") - of "critical": error("🚨") - else: "❓" - - echo fmt"Overall Status: {statusSymbol} {report.overallStatus.toUpper()}" - echo "" - - # Category details - for category in report.categories: - let categorySymbol = case category.status: - of "healthy": success("✅") - of "warning": warning("⚠️") - of "critical": error("🚨") - of "error": error("❌") - else: "❓" - - echo fmt"{categorySymbol} {($category.category).capitalizeAscii()}: {category.status} (score: {category.score:.2f})" - - if category.issues.len > 0: - for issue in category.issues: - echo fmt" • {issue}" - - if options.verbose and category.details != nil: - echo " Details:" - for key, value in category.details.pairs: - echo fmt" {key}: {value}" - - echo "" - - # Recommendations - if report.recommendations.len > 0: - echo bold("💡 Recommendations:") - for i, rec in report.recommendations: - echo fmt" {i + 1}. {rec}" - echo "" - - # Statistics summary - if options.verbose and report.statistics != nil: - echo bold("📊 System Statistics:") - for key, value in report.statistics.pairs: - echo fmt" {key}: {value}" - -proc runSystemHealthCheck*(options: DoctorOptions): SystemHealthReport = - ## Run comprehensive system health check - let startTime = cpuTime() - - var report = SystemHealthReport( - overallStatus: "unknown", - categories: @[], - recommendations: @[], - statistics: newJObject(), - timestamp: now(), - duration: 0.0 - ) - - try: - showInfo("🩺 Starting comprehensive system health check...") - - # Run individual health checks - if options.integrityCheck: - report.categories.add(runIntegrityHealthCheck(options)) - - if options.keyringCheck: - report.categories.add(runKeyringHealthCheck(options)) - - if options.performanceCheck: - report.categories.add(runPerformanceHealthCheck(options)) - - # Calculate overall status - var totalScore = 0.0 - var criticalCount = 0 - var warningCount = 0 - var healthyCount = 0 - - for category in report.categories: - totalScore += category.score - case category.status: - of "critical", "error": inc criticalCount - of "warning": inc warningCount - of "healthy": inc healthyCount - - let avgScore = if report.categories.len > 0: totalScore / report.categories.len.float else: 0.0 - - # Determine overall status - if criticalCount > 0: - report.overallStatus = "critical" - elif warningCount > 0: - report.overallStatus = "warning" - elif healthyCount > 0: - report.overallStatus = "healthy" - else: - report.overallStatus = "unknown" - - # Generate recommendations - report.recommendations = generateRecommendations(report.categories) - - # Compile statistics - report.statistics = %*{ - "categories_checked": report.categories.len, - "healthy_categories": healthyCount, - "warning_categories": warningCount, - "critical_categories": criticalCount, - "average_score": avgScore, - "check_duration": cpuTime() - startTime - } - - report.duration = cpuTime() - startTime - - showSuccess(fmt"Health check completed: {report.overallStatus}") - - except Exception as e: - report.overallStatus = "error" - report.recommendations.add(fmt"Health check failed: {e.msg}") - errorLog(fmt"System health check error: {e.msg}") - - return report - -proc nipDoctorCommand*(args: seq[string]): CommandResult = - ## Main implementation of nip doctor command - try: - let options = parseDoctorOptions(args) - - # Run health check - let report = runSystemHealthCheck(options) - - # Display results - case options.outputFormat: - of OutputHuman: - displayHealthReport(report, options) - else: - let reportJson = %*{ - "overall_status": report.overallStatus, - "categories": report.categories.mapIt(%*{ - "category": $it.category, - "status": it.status, - "score": it.score, - "issues": it.issues, - "details": it.details - }), - "recommendations": report.recommendations, - "statistics": report.statistics, - "timestamp": $report.timestamp, - "duration": report.duration - } - outputData(reportJson) - - # Log health check event - let severity = case report.overallStatus: - of "healthy": SeverityInfo - of "warning": SeverityWarning - of "critical": SeverityCritical - else: SeverityError - - logGlobalSecurityEvent(EventSystemHealthCheck, severity, "nip-doctor", - fmt"System health check completed: {report.overallStatus}") - - # Return appropriate result - case report.overallStatus: - of "healthy": - return successResult("System health check passed - all systems healthy") - of "warning": - return successResult("System health check completed with warnings") - of "critical": - return errorResult("System health check found critical issues", 1) - else: - return errorResult("System health check encountered errors", 2) - - except Exception as e: - return errorResult(fmt"Doctor command failed: {e.msg}") - -export nipDoctorCommand, DoctorOptions, parseDoctorOptions, SystemHealthReport \ No newline at end of file diff --git a/src/nip/graft.nim b/src/nip/graft.nim deleted file mode 100644 index e63b9d5..0000000 --- a/src/nip/graft.nim +++ /dev/null @@ -1,222 +0,0 @@ -import os -import osproc -import times -import blake2 -import nimpak/types -import strutils - -type - GraftError* = object of CatchableError - - GraftAuditLog* = object - timestamp*: string - source*: string - packageName*: string - version*: string - downloadedFilename*: string - blake2bHash*: string - hashAlgorithm*: string - sourceOutput*: string - archiveSize*: int64 - extractionTime*: float - fileCount*: int - deduplicationStatus*: string - originalArchivePath*: string - -proc calculateBlake2b*(filePath: string): string = - ## Calculate BLAKE2b hash of a file and return it in the format "blake2b-[hash]" - try: - let fileContent = readFile(filePath) - var ctx: Blake2b - blake2b_init(ctx, 32) # 32 bytes = 256 bits - blake2b_update(ctx, fileContent, fileContent.len) - let hash = blake2b_final(ctx) - result = "blake2b-" & $hash - except IOError as e: - raise newException(GraftError, "Failed to read file for hashing: " & filePath & " - " & e.msg) - except Exception as e: - raise newException(GraftError, "Failed to calculate BLAKE2b hash: " & e.msg) - -proc archiveExists*(cacheDir: string, blake2bHash: string): bool = - ## Check if an archive with the given BLAKE2b hash already exists in cache - let hashFile = joinPath(cacheDir, blake2bHash & ".hash") - result = fileExists(hashFile) - -proc reuseExistingArchive*(cacheDir: string, blake2bHash: string): string = - ## Get the path to an existing archive with the given BLAKE2b hash - let hashFile = joinPath(cacheDir, blake2bHash & ".hash") - if fileExists(hashFile): - result = readFile(hashFile).strip() - else: - raise newException(GraftError, "Archive hash file not found: " & hashFile) - -proc storeArchiveHash*(cacheDir: string, archivePath: string, blake2bHash: string) = - ## Store the mapping between BLAKE2b hash and archive path - let hashFile = joinPath(cacheDir, blake2bHash & ".hash") - writeFile(hashFile, archivePath) - -proc parseVersionFromFilename*(filename: string): string = - ## Parse version from pacman package filename (e.g., "neofetch-7.1.0-2-any.pkg.tar.zst" -> "7.1.0-2") - try: - # Handle empty or invalid filenames - if filename.len == 0: - return "unknown" - - # Remove file extension - let nameWithoutExt = filename.replace(".pkg.tar.zst", "").replace(".pkg.tar.xz", "") - - # Split by dashes and find version pattern - let parts = nameWithoutExt.split("-") - if parts.len >= 3: - # Typical format: packagename-version-release-arch - # Find the first part that looks like a version (contains digits and dots) - for i in 1.. 0 and (parts[i].contains('.') or parts[i][0].isDigit): - # Combine version and release if available - if i + 1 < parts.len - 1: # Has release number - result = parts[i] & "-" & parts[i + 1] - else: - result = parts[i] - return - - # Fallback: return everything after first dash, before last dash - if parts.len >= 2: - let fallback = parts[1..^2].join("-") - if fallback.len > 0: - result = fallback - else: - result = "unknown" - else: - result = "unknown" - except: - result = "unknown" - -proc detectPackageVersion*(packageName: string): string = - ## Detect package version using pacman - try: - let cmd = "pacman -Si " & packageName & " | grep '^Version' | awk '{print $3}'" - let (output, exitCode) = execCmdEx(cmd) - if exitCode == 0 and output.strip().len > 0 and not output.contains("error:") and not output.contains("not found"): - result = output.strip() - else: - result = "latest" - except: - result = "latest" - -proc graftPacman*(packageName: string, version: string = ""): PackageId = - let programsDir = "/tmp/nexus/Programs" - let cacheDir = "/tmp/nexus/cache" - - # Auto-detect version if not provided - var actualVersion = version - if actualVersion == "" or actualVersion == "latest": - actualVersion = detectPackageVersion(packageName) - echo "Auto-detected version for ", packageName, ": ", actualVersion - - let pkgDir = joinPath(programsDir, packageName, actualVersion) - createDir(pkgDir) - createDir(cacheDir) - - # Check for existing archive (deduplication) - let downloadedFilename = packageName & "-" & actualVersion & "-any.pkg.tar.zst" - let downloadedPkgPath = joinPath(cacheDir, downloadedFilename) - var calculatedBlake2b = "" - var deduplicationStatus = "New" - var pacmanOutput = "" - - if fileExists(downloadedPkgPath): - calculatedBlake2b = calculateBlake2b(downloadedPkgPath) - deduplicationStatus = "Reused" - echo "Found existing archive: ", downloadedPkgPath, " (BLAKE2b: ", calculatedBlake2b, ")" - else: - # Download package using pacman - let pacmanCmd = "pacman -Sw " & packageName & " --noconfirm --cachedir " & cacheDir - let (output, pacmanExit) = execCmdEx(pacmanCmd) - pacmanOutput = output - if pacmanExit != 0: - raise newException(GraftError, "Failed to download " & packageName & ": " & pacmanOutput) - - # Verify file exists - if not fileExists(downloadedPkgPath): - raise newException(GraftError, "Downloaded file not found: " & downloadedPkgPath) - - # Calculate BLAKE2b hash - calculatedBlake2b = calculateBlake2b(downloadedPkgPath) - - # Store hash mapping for future deduplication - storeArchiveHash(cacheDir, downloadedPkgPath, calculatedBlake2b) - - # Extract package with timing - let extractionStartTime = cpuTime() - let tarCmd = "tar -xvf " & downloadedPkgPath & " -C " & pkgDir - let (tarOutput, tarExit) = execCmdEx(tarCmd) - let extractionEndTime = cpuTime() - let extractionTime = extractionEndTime - extractionStartTime - - if tarExit != 0: - raise newException(GraftError, "Failed to extract " & downloadedPkgPath & ": " & tarOutput) - - # Count extracted files - var fileCount = 0 - for kind, path in walkDir(pkgDir, relative=true): - if kind == pcFile: - inc fileCount - - # Get archive size - let archiveSize = getFileSize(downloadedPkgPath) - - # Create comprehensive GraftAuditLog - let auditLog = GraftAuditLog( - timestamp: now().format("yyyy-MM-dd'T'HH:mm:sszzz"), - source: "pacman", - packageName: packageName, - version: actualVersion, - downloadedFilename: downloadedFilename, - blake2bHash: calculatedBlake2b, - hashAlgorithm: "blake2b", - sourceOutput: pacmanOutput, - archiveSize: archiveSize, - extractionTime: extractionTime, - fileCount: fileCount, - deduplicationStatus: deduplicationStatus, - originalArchivePath: downloadedPkgPath - ) - - # Write enhanced graft.log - let graftLogPath = joinPath(pkgDir, "graft.log") - var logFile = open(graftLogPath, fmWrite) - logFile.writeLine("Graft Log for " & packageName & "-" & actualVersion) - logFile.writeLine("=============================") - logFile.writeLine("Timestamp: " & auditLog.timestamp) - logFile.writeLine("Source: " & auditLog.source) - logFile.writeLine("Package: " & auditLog.packageName) - logFile.writeLine("Version: " & auditLog.version) - logFile.writeLine("Downloaded Filename: " & auditLog.downloadedFilename) - logFile.writeLine("Archive Size: " & $auditLog.archiveSize & " bytes") - logFile.writeLine("BLAKE2b Hash: " & auditLog.blake2bHash) - logFile.writeLine("Hash Algorithm: " & auditLog.hashAlgorithm) - logFile.writeLine("Original Archive Path: " & auditLog.originalArchivePath) - logFile.writeLine("Deduplication Status: " & auditLog.deduplicationStatus) - logFile.writeLine("") - logFile.writeLine("Pacman Download Output:") - logFile.writeLine("======================") - logFile.writeLine(auditLog.sourceOutput) - logFile.writeLine("") - logFile.writeLine("Package Extraction Summary:") - logFile.writeLine("==========================") - logFile.writeLine("Files Extracted: " & $auditLog.fileCount) - logFile.writeLine("Extraction Time: " & $auditLog.extractionTime & "s") - logFile.writeLine("Target Directory: " & pkgDir) - logFile.writeLine("BLAKE2b Verification: PASSED") - logFile.close() - - result = PackageId(name: packageName, version: actualVersion, stream: Stable) - -when isMainModule: - try: - let pkg = graftPacman("neofetch", "7.1.0") - echo "Grafted: ", $pkg - echo "Location: /tmp/nexus/Programs/neofetch/7.1.0" - echo "Log: /tmp/nexus/Programs/neofetch/7.1.0/graft.log" - except GraftError as e: - echo "Error: ", e.msg diff --git a/src/nip/installer.nim b/src/nip/installer.nim deleted file mode 100644 index e69de29..0000000 diff --git a/src/nip/integrity.nim b/src/nip/integrity.nim deleted file mode 100644 index 5722d2d..0000000 --- a/src/nip/integrity.nim +++ /dev/null @@ -1,578 +0,0 @@ -## Integrity Manager - Merkle Tree Verification for Content Addressable Storage -## -## **Crypto-Anarchist Zeal Applied to Package Management** -## Trust the math, not the source. Verify everything. -## -## Core Philosophy: -## - Content is king, hashes are truth -## - CAS provides inherent caching via path-based verification -## - Parallel hash calculation for performance -## - Audit trail for all verification events -## - Zero tolerance for corruption -## -## **Canonical Leaf Hashing:** -## The Merkle tree uses path-aware hashing to ensure determinism: -## CanonicalHash = Hash(RelativePath || ContentHash) -## This guarantees that moving a file changes the package structure hash. - -import std/[os, strutils, algorithm, tables, hashes] -import std/[times, asyncdispatch, threadpool] -import nimcrypto/[hash, blake2] -import nip/unified_storage -import nip/manifest_parser - -type - # ========================================================================== - # Core Types - # ============================================================================ - - IntegrityError* = object of CatchableError - ## Integrity verification failure - path*: string - expectedHash*: string - actualHash*: string - errorType*: IntegrityErrorType - - IntegrityErrorType* = enum - ## Types of integrity failures - HashMismatch, ## Calculated hash doesn't match expected - FileNotFound, ## Referenced file missing - PermissionDenied, ## Cannot read file for verification - CorruptedData, ## File exists but appears corrupted - InvalidHash, ## Hash format invalid - CASInconsistent ## CAS structure inconsistent - - CanonicalLeaf* = object - ## Canonical leaf node with path-aware hashing - relativePath*: string ## Relative path from root - contentHash*: string ## Hash of file content - canonicalHash*: string ## Hash(relativePath || contentHash) - size*: int64 ## File size in bytes - - MerkleNode* = object - ## Node in Merkle tree - path*: string ## File/directory path - hash*: string ## Content hash (hex encoded) - size*: int64 ## Size in bytes - isDirectory*: bool ## True if directory node - children*: seq[MerkleNode] ## Child nodes (for directories) - - MerkleTree* = object - ## Complete Merkle tree for a package - root*: MerkleNode - rootHash*: string ## The Merkle root (this goes in manifest) - totalFiles*: int - totalSize*: int64 - algorithm*: string ## "blake2b" (TODO: xxh3-128) - - VerificationResult* = object - ## Result of integrity verification - success*: bool - path*: string - expectedHash*: string - actualHash*: string - verifiedFiles*: int - totalFiles*: int - duration*: float ## Verification time in seconds - errors*: seq[IntegrityError] - - IntegrityCache* = object - ## Cache for hash calculations - fileHashes*: Table[string, string] ## path -> content hash - dirHashes*: Table[string, string] ## path -> merkle root - lastModified*: Table[string, int64] ## path -> mtime - - IntegrityManager* = object - ## Main integrity verification manager - casRoot*: string ## CAS root directory - cache*: IntegrityCache ## Hash cache - auditLog*: string ## Audit log file path - parallelism*: int ## Number of parallel workers - chunkSize*: int ## Chunk size for large files - strictMode*: bool ## Fail on any hash mismatch - -# ============================================================================ -# Hash Calculation (BLAKE2b placeholder for xxh3-128) -# ============================================================================ - -proc calculateHash*(data: string): string = - ## Calculate hash of data - ## TODO: Switch to xxh3-128 when available - ## Returns hash in format: "blake2b-" - let digest = blake2_512.digest(data) - var hexDigest = "" - for b in digest.data: - hexDigest.add(b.toHex(2).toLowerAscii()) - result = "blake2b-" & hexDigest - -proc calculateFileHash*(path: string, chunkSize: int = 65536): string = - ## Calculate hash of file using chunked reading - if not fileExists(path): - raise newException(IntegrityError, "File not found: " & path) - - let data = readFile(path) - result = calculateHash(data) - -proc hashString*(s: string): string = - ## Calculate hash of string - return calculateHash(s) - -# ============================================================================ -# Canonical Leaf Hashing - The Foundation of Determinism -# ============================================================================ - -proc calculateCanonicalHash*(relativePath: string, contentHash: string): string = - ## Calculate canonical hash: Hash(RelativePath || ContentHash) - ## This ensures that file location is part of the hash - ## - ## **Critical for CAS determinism:** - ## - Same content in different locations = different canonical hash - ## - Moving a file changes the package structure hash - ## - Prevents hash collisions from identical files in different dirs - let canonicalInput = relativePath & "|" & contentHash - return hashString(canonicalInput) - -# ============================================================================ -# Parallel Hashing Worker -# ============================================================================ - -proc parallelHashWorker(path: string, relativePath: string): CanonicalLeaf {.gcsafe.} = - ## Worker to calculate canonical leaf hash concurrently - ## This is the expensive operation that benefits from parallelization - ## - ## **Performance Critical:** - ## - File I/O (reading content) - ## - Hash calculation (CPU-bound) - ## - Both benefit from parallel execution - - # 1. Calculate file content hash (expensive I/O + CPU) - let contentHash = calculateFileHash(path) - let fileSize = getFileSize(path) - - # 2. Calculate canonical hash = Hash(path || content_hash) - let canonicalHash = calculateCanonicalHash(relativePath, contentHash) - - return CanonicalLeaf( - relativePath: relativePath, - contentHash: contentHash, - canonicalHash: canonicalHash, - size: fileSize - ) - -proc collectCanonicalLeaves*(rootPath: string, cache: var IntegrityCache, - parallel: bool = true): seq[CanonicalLeaf] = - ## Collect all files as canonical leaves with path-aware hashing - ## This is the foundation of deterministic Merkle tree construction - ## - ## **Algorithm:** - ## 1. Walk directory tree recursively - ## 2. For each file: calculate content hash (parallel if enabled) - ## 3. Calculate canonical hash = Hash(path || content_hash) - ## 4. Sort by relative path for absolute determinism - ## - ## **Parallelization:** - ## - Uses spawn/threadpool for concurrent file hashing - ## - Significant speedup for large packages (10-100+ files) - ## - Falls back to sequential for small packages - var leaves: seq[CanonicalLeaf] = @[] - - # Normalize root path - let normalizedRoot = rootPath.normalizedPath() - let rootLen = normalizedRoot.len + 1 # Include trailing separator - - # Collect all file paths first - var filePaths: seq[tuple[fullPath: string, relativePath: string]] = @[] - for path in walkDirRec(normalizedRoot, yieldFilter = {pcFile}): - let relativePath = if path.len > rootLen: - path[rootLen..^1] - else: - extractFilename(path) - filePaths.add((fullPath: path, relativePath: relativePath)) - - # Decide on parallelization strategy - let useParallel = parallel and filePaths.len > 10 # Parallel for 10+ files - - if useParallel: - # Parallel processing using spawn - var futures: seq[FlowVar[CanonicalLeaf]] = @[] - - for (fullPath, relativePath) in filePaths: - # Check cache first - let info = getFileInfo(fullPath) - - if relativePath in cache.fileHashes and - relativePath in cache.lastModified and - cache.lastModified[relativePath] == info.lastWriteTime.toUnix(): - # Cache hit - use cached values - let contentHash = cache.fileHashes[relativePath] - let canonicalHash = calculateCanonicalHash(relativePath, contentHash) - leaves.add(CanonicalLeaf( - relativePath: relativePath, - contentHash: contentHash, - canonicalHash: canonicalHash, - size: info.size - )) - else: - # Cache miss - spawn parallel worker - futures.add(spawn parallelHashWorker(fullPath, relativePath)) - - # Collect results from parallel workers - for future in futures: - let leaf = ^future # Wait for result - leaves.add(leaf) - - # Update cache - cache.fileHashes[leaf.relativePath] = leaf.contentHash - cache.lastModified[leaf.relativePath] = getFileInfo( - normalizedRoot / leaf.relativePath - ).lastWriteTime.toUnix() - - else: - # Sequential processing (small packages or parallel disabled) - for (fullPath, relativePath) in filePaths: - # Check cache - var contentHash: string - let info = getFileInfo(fullPath) - - if relativePath in cache.fileHashes and - relativePath in cache.lastModified and - cache.lastModified[relativePath] == info.lastWriteTime.toUnix(): - # Cache hit - contentHash = cache.fileHashes[relativePath] - else: - # Cache miss - calculate - contentHash = calculateFileHash(fullPath) - cache.fileHashes[relativePath] = contentHash - cache.lastModified[relativePath] = info.lastWriteTime.toUnix() - - # Calculate canonical hash - let canonicalHash = calculateCanonicalHash(relativePath, contentHash) - - leaves.add(CanonicalLeaf( - relativePath: relativePath, - contentHash: contentHash, - canonicalHash: canonicalHash, - size: info.size - )) - - # Sort leaves by relative path for absolute determinism - # This is CRITICAL - must happen after all parallel work completes - leaves.sort(proc(a, b: CanonicalLeaf): int = cmp(a.relativePath, b.relativePath)) - - return leaves - -# ============================================================================ -# Merkle Tree Construction from Canonical Leaves -# ============================================================================ - -proc buildMerkleTreeFromLeaves*(leaves: seq[CanonicalLeaf]): MerkleNode = - ## Build Merkle tree from flat list of canonical leaves - ## Uses bottom-up construction with deterministic ordering - ## - ## **Algorithm:** - ## 1. Start with sorted canonical leaves - ## 2. Pair adjacent nodes and hash: Hash(left || right) - ## 3. Repeat until single root node remains - ## 4. Handle odd nodes by promoting to next level - - if leaves.len == 0: - # Empty tree - return MerkleNode( - path: "", - hash: hashString(""), - size: 0, - isDirectory: true, - children: @[] - ) - - if leaves.len == 1: - # Single leaf - return as root - let leaf = leaves[0] - return MerkleNode( - path: leaf.relativePath, - hash: leaf.canonicalHash, - size: leaf.size, - isDirectory: false, - children: @[] - ) - - # Multiple leaves - build tree bottom-up - var currentLevel: seq[MerkleNode] = @[] - - # Create leaf nodes from canonical leaves - for leaf in leaves: - currentLevel.add(MerkleNode( - path: leaf.relativePath, - hash: leaf.canonicalHash, - size: leaf.size, - isDirectory: false, - children: @[] - )) - - # Build tree by pairing nodes - while currentLevel.len > 1: - var nextLevel: seq[MerkleNode] = @[] - - var i = 0 - while i < currentLevel.len: - if i + 1 < currentLevel.len: - # Pair two nodes - let left = currentLevel[i] - let right = currentLevel[i + 1] - - # Combine hashes: Hash(leftHash || rightHash) - let combinedHash = hashString(left.hash & right.hash) - - nextLevel.add(MerkleNode( - path: "", # Internal nodes don't have paths - hash: combinedHash, - size: left.size + right.size, - isDirectory: true, - children: @[left, right] - )) - - i += 2 - else: - # Odd node - promote to next level - nextLevel.add(currentLevel[i]) - i += 1 - - currentLevel = nextLevel - - return currentLevel[0] - -proc buildMerkleTree*(rootPath: string, cache: var IntegrityCache): MerkleTree = - ## Build Merkle tree for a directory using canonical leaf hashing - ## This is the main entry point for build_hash calculation - ## - ## **Returns:** MerkleTree with rootHash suitable for manifest - - if not dirExists(rootPath): - raise newException(IntegrityError, "Directory not found: " & rootPath) - - # Collect canonical leaves (path-aware hashing) - let leaves = collectCanonicalLeaves(rootPath, cache) - - # Build tree from leaves - let root = buildMerkleTreeFromLeaves(leaves) - - # Calculate statistics - var totalSize: int64 = 0 - for leaf in leaves: - totalSize += leaf.size - - result = MerkleTree( - root: root, - rootHash: root.hash, - totalFiles: leaves.len, - totalSize: totalSize, - algorithm: "blake2b" # TODO: xxh3-128 - ) - -# ============================================================================ -# Verification Functions -# ============================================================================ - -proc verifyContent*(rootPath: string, expectedHash: string, manager: var IntegrityManager): VerificationResult = - ## Verify content against expected hash - ## This is the main verification entry point - let startTime = cpuTime() - - var res = VerificationResult( - path: rootPath, - expectedHash: expectedHash, - success: false, - verifiedFiles: 0, - totalFiles: 0, - errors: @[] - ) - - try: - # Build Merkle tree and calculate actual hash - let tree = buildMerkleTree(rootPath, manager.cache) - res.actualHash = tree.rootHash - res.totalFiles = tree.totalFiles - - # Compare hashes - if res.actualHash == expectedHash: - res.success = true - res.verifiedFiles = tree.totalFiles - else: - res.errors.add(IntegrityError( - path: rootPath, - expectedHash: expectedHash, - actualHash: res.actualHash, - errorType: HashMismatch, - msg: "Merkle root hash mismatch" - )) - - except IntegrityError as e: - var err = IntegrityError( - path: e.path, - expectedHash: e.expectedHash, - actualHash: e.actualHash, - errorType: e.errorType, - msg: e.msg - ) - res.errors.add(err) - except OSError as e: - res.errors.add(IntegrityError( - path: rootPath, - expectedHash: expectedHash, - actualHash: "", - errorType: FileNotFound, - msg: "Path not found: " & e.msg - )) - - res.duration = cpuTime() - startTime - - # Log verification result (defined later in file) - # logVerificationResult(res, manager) - - return res - -proc verifyManifestHashes*(manifest: PackageManifest, manager: var IntegrityManager): seq[VerificationResult] = - ## Verify all hashes in a manifest - result = @[] - - # Verify build hash if present - if manifest.buildHash.len > 0: - let buildResult = verifyContent("build", manifest.buildHash, manager) - result.add(buildResult) - - # Verify source hash if present - if manifest.sourceHash.len > 0: - let sourceResult = verifyContent("source", manifest.sourceHash, manager) - result.add(sourceResult) - - # Verify artifact hash if present - if manifest.artifactHash.len > 0: - let artifactResult = verifyContent("artifact", manifest.artifactHash, manager) - result.add(artifactResult) - -# ============================================================================ -# Audit Logging -# ============================================================================ - -proc logVerificationResult*(result: VerificationResult, manager: IntegrityManager) = - ## Log verification result to audit trail - let timestamp = now().format("yyyy-MM-dd HH:mm:ss") - let status = if result.success: "SUCCESS" else: "FAILURE" - let logLine = "$1 [$2] $3: $4 (expected: $5, actual: $6, files: $7/$8, duration: $9s)" % [ - timestamp, status, result.path, - if result.success: "VERIFIED" else: "HASH_MISMATCH", - result.expectedHash, result.actualHash, - $result.verifiedFiles, $result.totalFiles, - result.duration.formatFloat(ffDecimal, 3) - ] - - # Append to audit log - try: - let logFile = open(manager.auditLog, fmAppend) - defer: logFile.close() - logFile.writeLine(logLine) - - # Also log errors - for error in result.errors: - let errorLine = "$1 [ERROR] $2: $3 - $4" % [ - timestamp, error.path, $error.errorType, error.msg - ] - logFile.writeLine(errorLine) - except IOError: - discard # Logging failure shouldn't break verification - -# ============================================================================ -# Manager Construction -# ============================================================================ - -proc newIntegrityManager*(casRoot: string, auditLog: string = "", - parallelism: int = 4, strictMode: bool = true): IntegrityManager = - ## Create new integrity manager - result = IntegrityManager( - casRoot: casRoot, - auditLog: if auditLog.len > 0: auditLog else: casRoot / "integrity.log", - parallelism: parallelism, - chunkSize: 65536, # 64KB chunks - strictMode: strictMode, - cache: IntegrityCache( - fileHashes: initTable[string, string](), - dirHashes: initTable[string, string](), - lastModified: initTable[string, int64]() - ) - ) - - # Ensure audit log directory exists - createDir(parentDir(result.auditLog)) - -proc clearCache*(manager: var IntegrityManager) = - ## Clear integrity cache - manager.cache.fileHashes.clear() - manager.cache.dirHashes.clear() - manager.cache.lastModified.clear() - -# ============================================================================ -# Convenience Functions -# ============================================================================ - -proc calculateBuildHash*(packagePath: string): string = - ## Calculate build_hash for a package directory - ## This is what goes in the manifest - var cache = IntegrityCache( - fileHashes: initTable[string, string](), - dirHashes: initTable[string, string](), - lastModified: initTable[string, int64]() - ) - let tree = buildMerkleTree(packagePath, cache) - result = tree.rootHash - -proc verifyPackage*(packagePath: string, manifest: PackageManifest, - manager: var IntegrityManager): bool = - ## Verify a package against its manifest - if manifest.buildHash.len == 0: - return false # No hash to verify against - - let result = verifyContent(packagePath, manifest.buildHash, manager) - return result.success - -# ============================================================================ -# Pretty Printing -# ============================================================================ - -proc `$`*(tree: MerkleTree): string = - ## Convert Merkle tree to human-readable string - result = "MerkleTree:\n" - result.add(" Root Hash: " & tree.rootHash & "\n") - result.add(" Algorithm: " & tree.algorithm & "\n") - result.add(" Total Files: " & $tree.totalFiles & "\n") - result.add(" Total Size: " & $tree.totalSize & " bytes\n") - -proc `$`*(res: VerificationResult): string = - ## Convert verification result to human-readable string - let status = if res.success: "✅ SUCCESS" else: "❌ FAILURE" - result = status & "\n" - result.add(" Path: " & res.path & "\n") - result.add(" Expected: " & res.expectedHash & "\n") - result.add(" Actual: " & res.actualHash & "\n") - result.add(" Files: " & $res.verifiedFiles & "/" & $res.totalFiles & "\n") - result.add(" Duration: " & res.duration.formatFloat(ffDecimal, 3) & "s\n") - - if res.errors.len > 0: - result.add(" Errors:\n") - for err in res.errors: - result.add(" - " & err.msg & "\n") - -when isMainModule: - echo "Integrity Manager - Merkle Tree Verification" - echo "Hash Algorithm: blake2b (TODO: xxh3-128)" - echo "" - echo "**Canonical Leaf Hashing:**" - echo " CanonicalHash = Hash(RelativePath || ContentHash)" - echo " This ensures deterministic, path-aware verification." - echo "" - - # Example usage - let testContent = "Hello, Merkle Tree!" - let stringHash = hashString(testContent) - echo "String hash: " & stringHash - echo "" - echo "Trust the math, not the source." diff --git a/src/nip/lockfile.nim b/src/nip/lockfile.nim deleted file mode 100644 index e69de29..0000000 diff --git a/src/nip/manifest.nim b/src/nip/manifest.nim deleted file mode 100644 index e69de29..0000000 diff --git a/src/nip/manifest_parser.nim b/src/nip/manifest_parser.nim deleted file mode 100644 index 5fdc646..0000000 --- a/src/nip/manifest_parser.nim +++ /dev/null @@ -1,2057 +0,0 @@ -## Manifest Parser - Format-Agnostic Package Manifest Parsing -## -## **Systems Engineering Approach:** -## - Strict-by-default validation with whitelist-based field controlpport. -## - Format-agnostic: KDL (human-friendly) and JSON (machine-friendly) -## - Semantic versioning enforcement (not just strings) -## - Platform/Architecture constraint awareness -## - UTCP protocol support for AI accessibility -## - Zero tolerance for contamination (unknown fields = rejection) -## -## **Core Philosophy:** -## The manifest is the contract. If it lies, we reject it. -## If it's ambiguous, we reject it. If it's incomplete, we reject it. - -import std/[strutils, options, sets, json, sequtils, tables, algorithm] -import nimpak/kdl_parser -import nip/platform -import nip/xxh - -type - # ============================================================================ - # Format Types - # ============================================================================ - - ManifestFormat* = enum - ## Supported manifest formats (wire format) - FormatKDL = "kdl" ## Human-friendly KDL format - FormatJSON = "json" ## Machine-friendly JSON format - FormatAuto = "auto" ## Auto-detect from content - - FormatType* = enum - ## Package format types (semantic meaning) - NPK = "npk" ## Nexus Package Kit (Standard distribution) - NIP = "nip" ## Nexus Installed Package (Local state) - NEXTER = "nexter" ## Nexus Container (Opaque runtime) - - # ============================================================================ - # Validation & Error Handling - # ============================================================================ - - ManifestValidationMode* = enum - ## Validation strictness levels - ValidationStrict ## Reject unknown fields, enforce all constraints (DEFAULT) - ValidationLenient ## Warn on unknown fields, allow missing optional fields - ValidationMinimal ## Only validate required fields (unsafe, testing only) - - ManifestErrorCode* = enum - ## Specific error codes for precise diagnostics - InvalidFormat, ## Syntax error in wire format - MissingField, ## Required field absent - InvalidValue, ## Field present but value invalid - StrictViolation, ## Unknown field detected (contamination) - SemVerViolation, ## Version string not valid semver - SchemaError, ## Structural schema violation - HashMismatch, ## Integrity hash mismatch - PlatformIncompat, ## Platform/arch constraint violation - DependencyError ## Dependency specification invalid - - ManifestError* = object of CatchableError - ## Detailed error with context and suggestions - code*: ManifestErrorCode - field*: string ## Field that caused the error - line*: int ## Line number (if available) - context*: string ## Human-readable context - suggestions*: seq[string] ## Actionable suggestions - - # ============================================================================ - # Core Manifest Types - # ============================================================================ - - PackageManifest* = object - ## Complete package manifest (the contract) - # Identity - format*: FormatType - name*: string - version*: SemanticVersion ## Parsed, not string - description*: Option[string] - homepage*: Option[string] - license*: string - - # Dependencies - dependencies*: seq[DependencySpec] - buildDependencies*: seq[DependencySpec] - optionalDependencies*: seq[DependencySpec] - - # Build configuration - buildSystem*: Option[string] - buildFlags*: seq[string] - configureFlags*: seq[string] - - # Platform constraints (THE PHYSICAL WORLD) - supportedOS*: seq[string] ## e.g., ["linux", "freebsd"] - supportedArchitectures*: seq[string] ## e.g., ["x86_64", "aarch64"] - requiredCapabilities*: seq[string] ## e.g., ["user_namespaces"] - - # Runtime configuration - libc*: Option[string] - allocator*: Option[string] - - # Integrity (cryptographic truth) - buildHash*: string ## BLAKE3 hash of build configuration - sourceHash*: string ## BLAKE3 hash of source - artifactHash*: string ## BLAKE3 hash of final artifact - - # Metadata - author*: Option[string] - timestamp*: Option[string] - tags*: seq[string] - maintainers*: seq[string] - - # UTCP support (AI accessibility) - utcpEndpoint*: Option[string] ## Remote query endpoint - utcpVersion*: Option[string] ## UTCP protocol version - - # System Integration - files*: seq[FileSpec] - users*: seq[UserSpec] - groups*: seq[GroupSpec] - services*: seq[ServiceSpec] - - # Security / Sandbox (NIP) - sandbox*: Option[SandboxConfig] - - # Desktop Integration (NIP) - desktop*: Option[DesktopIntegration] - - DesktopIntegration* = object - ## Desktop environment integration - displayName*: string ## Human readable name (e.g. "Firefox Web Browser") - icon*: Option[string] ## Icon name or path - categories*: seq[string] ## Menu categories (e.g. "Network;WebBrowser") - keywords*: seq[string] ## Search keywords - mimeTypes*: seq[string] ## Supported MIME types - terminal*: bool ## Run in terminal? - startupNotify*: bool ## Support startup notification? - startupWMClass*: Option[string] ## For window grouping (StartupWMClass) - - SandboxLevel* = enum - SandboxStrict = "strict" ## Maximum isolation (default) - SandboxStandard = "standard" ## Standard desktop app isolation - SandboxRelaxed = "relaxed" ## Minimal isolation (use with caution) - SandboxNone = "none" ## No isolation (requires user override) - - SandboxConfig* = object - ## Sandboxing configuration for NIPs - level*: SandboxLevel - - # Linux Specific - seccompProfile*: Option[string] ## "default", "strict", or custom path - capabilities*: seq[string] ## e.g. "CAP_NET_ADMIN" (usually to drop) - namespaces*: seq[string] ## e.g. "net", "ipc", "pid" - - # BSD Specific (OpenBSD/DragonflyBSD) - pledge*: Option[string] ## e.g. "stdio rpath wpath inet" - unveil*: seq[string] ## e.g. "/tmp:rwc" - - DependencySpec* = object - ## Package dependency specification - name*: string - versionConstraint*: VersionConstraint - optional*: bool - features*: seq[string] - - FileSpec* = object - ## File specification for installation - path*: string - hash*: string - size*: int64 - permissions*: string # e.g. "755" - - UserSpec* = object - ## System user specification - name*: string - uid*: Option[int] - group*: string - shell*: string - home*: string - - GroupSpec* = object - ## System group specification - name*: string - gid*: Option[int] - - ServiceSpec* = object - ## System service specification - name*: string - content*: string # Content of the service file - enabled*: bool - - SemanticVersion* = object - ## Semantic version (major.minor.patch-prerelease+build) - ## NOT just a string - this is a structured type - major*: int - minor*: int - patch*: int - prerelease*: string - build*: string - - VersionConstraint* = object - ## Version constraint specification - operator*: VersionOperator - version*: SemanticVersion - - VersionOperator* = enum - ## Version comparison operators - OpExact = "=" ## Exact version match - OpGreater = ">" ## Greater than - OpGreaterEq = ">=" ## Greater than or equal - OpLess = "<" ## Less than - OpLessEq = "<=" ## Less than or equal - OpTilde = "~" ## Compatible version (~1.2.3 = >=1.2.3 <1.3.0) - OpCaret = "^" ## Compatible version (^1.2.3 = >=1.2.3 <2.0.0) - OpAny = "*" ## Any version - - # ============================================================================ - # Validation Rules (The Enforcers) - # ============================================================================ - - ValidationRule* = object - ## A validation rule that can be applied to manifest data - name*: string - description*: string - ## Validate function receives parsed data and accumulates errors - validate*: proc(data: JsonNode, errors: var seq[ManifestError]): bool - - ParserConfig* = object - ## Parser configuration - format*: FormatType - wireFormat*: ManifestFormat ## KDL or JSON - strictMode*: bool - allowedFields*: HashSet[string] ## The Whitelist (The Bouncer) - - ManifestParser* = object - ## Parser state and configuration - config*: ParserConfig - rules*: seq[ValidationRule] - warnings*: seq[string] - -# ============================================================================ -# Known Fields (The Whitelist) -# ============================================================================ - -const BASE_ALLOWED_FIELDS = [ - # Identity - "name", "version", "description", "homepage", "license", "author", - # Dependencies - "dependencies", "build_dependencies", "optional_dependencies", - # Build - "build_system", "build_flags", "configure_flags", - # Platform - "os", "arch", "supported_os", "supported_architectures", - "required_capabilities", - # Runtime - "libc", "allocator", - # Integrity - "build_hash", "source_hash", "artifact_hash", - # Metadata - "timestamp", "tags", "maintainers", - # UTCP - "utcp_endpoint", "utcp_version" -].toHashSet() - -const NPK_SPECIFIC_FIELDS = [ - "files", "install_scripts", "post_install" -].toHashSet() - -const NIP_SPECIFIC_FIELDS = [ - "desktop", "permissions", "installed_path", "sandbox" -].toHashSet() - -const NEXTER_SPECIFIC_FIELDS = [ - "container", "env", "entrypoint", "volumes" -].toHashSet() - -const KNOWN_DEPENDENCY_FIELDS = [ - "name", "version", "optional", "features" -].toHashSet() - -# Valid platform values -const VALID_OS = [ - "linux", "freebsd", "openbsd", "netbsd", "dragonfly", - "macos", "windows", "android", "ios" -].toHashSet() - -const VALID_ARCHITECTURES = [ - "x86_64", "aarch64", "armv8", "armv7", "armv6", "i686", - "riscv64", "riscv32", "powerpc64", "powerpc64le", - "s390x", "mips64", "mips64el" -].toHashSet() - -# ============================================================================ -# Format Detection -# ============================================================================ - -proc detectFormat*(content: string): ManifestFormat = - ## Auto-detect manifest format from content - let trimmed = content.strip() - - if trimmed.len > 0 and (trimmed[0] == '{' or trimmed[0] == '['): - return FormatJSON - - # KDL typically starts with a node name - if trimmed.len > 0 and trimmed[0] != '{': - return FormatKDL - - raise newException(ManifestError, "Unable to detect manifest format") - -# ============================================================================ -# Semantic Versioning (NOT JUST STRINGS) -# ============================================================================ - -proc isValidSemVer*(v: string): bool = - ## Quick check if string looks like semver (X.Y.Z) - if v.len == 0: return false - let parts = v.split({'.', '-', '+'}) - if parts.len < 3: return false - - # Ensure first three are numbers - try: - discard parseInt(parts[0]) - discard parseInt(parts[1]) - discard parseInt(parts[2]) - return true - except ValueError: - return false - -proc parseSemanticVersion*(version: string): SemanticVersion = - ## Parse semantic version string (major.minor.patch-prerelease+build) - ## This is NOT just string validation - we parse into structured data - - if not isValidSemVer(version): - raise newException(ManifestError, - "Invalid semantic version: " & version & " (expected X.Y.Z)") - - var parts = version.split('-', maxsplit = 1) - var versionPart = parts[0] - var prerelease = "" - var build = "" - - if parts.len > 1: - var prereleaseAndBuild = parts[1].split('+', maxsplit = 1) - prerelease = prereleaseAndBuild[0] - if prereleaseAndBuild.len > 1: - build = prereleaseAndBuild[1] - else: - # Check for build metadata without prerelease - parts = versionPart.split('+', maxsplit = 1) - versionPart = parts[0] - if parts.len > 1: - build = parts[1] - - let versionNumbers = versionPart.split('.') - if versionNumbers.len != 3: - raise newException(ManifestError, - "Invalid semantic version: " & version & " (expected major.minor.patch)") - - try: - result = SemanticVersion( - major: parseInt(versionNumbers[0]), - minor: parseInt(versionNumbers[1]), - patch: parseInt(versionNumbers[2]), - prerelease: prerelease, - build: build - ) - except ValueError as e: - raise newException(ManifestError, - "Invalid semantic version numbers: " & version & " (" & e.msg & ")") - -proc `$`*(v: SemanticVersion): string = - ## Convert semantic version to string - result = $v.major & "." & $v.minor & "." & $v.patch - if v.prerelease.len > 0: - result.add("-" & v.prerelease) - if v.build.len > 0: - result.add("+" & v.build) - -proc compareVersions*(a, b: SemanticVersion): int = - ## Compare two semantic versions (-1: ab) - ## Follows semver 2.0.0 specification - - # Compare major.minor.patch - if a.major != b.major: - return cmp(a.major, b.major) - if a.minor != b.minor: - return cmp(a.minor, b.minor) - if a.patch != b.patch: - return cmp(a.patch, b.patch) - - # Prerelease comparison - if a.prerelease.len == 0 and b.prerelease.len > 0: - return 1 # Release > prerelease - if a.prerelease.len > 0 and b.prerelease.len == 0: - return -1 # Prerelease < release - if a.prerelease != b.prerelease: - return cmp(a.prerelease, b.prerelease) - - # Build metadata is ignored in version precedence - return 0 - -proc `<`*(a, b: SemanticVersion): bool = compareVersions(a, b) < 0 -proc `<=`*(a, b: SemanticVersion): bool = compareVersions(a, b) <= 0 -proc `==`*(a, b: SemanticVersion): bool = compareVersions(a, b) == 0 -proc `>`*(a, b: SemanticVersion): bool = compareVersions(a, b) > 0 -proc `>=`*(a, b: SemanticVersion): bool = compareVersions(a, b) >= 0 - - -proc parseVersionConstraint*(constraint: string): VersionConstraint = - ## Parse version constraint (e.g., ">=1.2.3", "~1.0.0", "^2.0.0") - let trimmed = constraint.strip() - - if trimmed == "*": - return VersionConstraint(operator: OpAny, version: SemanticVersion()) - - var operator: VersionOperator - var versionStr: string - - if trimmed.startsWith(">="): - operator = OpGreaterEq - versionStr = trimmed[2..^1].strip() - elif trimmed.startsWith("<="): - operator = OpLessEq - versionStr = trimmed[2..^1].strip() - elif trimmed.startsWith(">"): - operator = OpGreater - versionStr = trimmed[1..^1].strip() - elif trimmed.startsWith("<"): - operator = OpLess - versionStr = trimmed[1..^1].strip() - elif trimmed.startsWith("~"): - operator = OpTilde - versionStr = trimmed[1..^1].strip() - elif trimmed.startsWith("^"): - operator = OpCaret - versionStr = trimmed[1..^1].strip() - elif trimmed.startsWith("="): - operator = OpExact - versionStr = trimmed[1..^1].strip() - else: - # No operator means exact match - operator = OpExact - versionStr = trimmed - - let version = parseSemanticVersion(versionStr) - result = VersionConstraint(operator: operator, version: version) - -proc satisfiesConstraint*(version: SemanticVersion, - constraint: VersionConstraint): bool = - ## Check if a version satisfies a constraint - case constraint.operator: - of OpAny: - return true - of OpExact: - return version == constraint.version - of OpGreater: - return version > constraint.version - of OpGreaterEq: - return version >= constraint.version - of OpLess: - return version < constraint.version - of OpLessEq: - return version <= constraint.version - of OpTilde: - # ~1.2.3 means >=1.2.3 <1.3.0 - if version < constraint.version: - return false - return version.major == constraint.version.major and - version.minor == constraint.version.minor - of OpCaret: - # ^1.2.3 means >=1.2.3 <2.0.0 - if version < constraint.version: - return false - return version.major == constraint.version.major - -proc parseSandboxLevel*(s: string): SandboxLevel = - case s.toLowerAscii(): - of "strict": return SandboxStrict - of "standard": return SandboxStandard - of "relaxed": return SandboxRelaxed - of "none": return SandboxNone - else: raise newException(ManifestError, "Invalid sandbox level: " & s) - -# ============================================================================ -# Validation Rules (The Enforcers) -# ============================================================================ - -proc createStrictStructRule*(allowed: HashSet[string]): ValidationRule = - ## The Bouncer: Kicks out any field not on the whitelist - ## This is your primary defense against contamination - return ValidationRule( - name: "strict_whitelist", - description: "Ensures only authorized fields are present", - validate: proc(data: JsonNode, errors: var seq[ManifestError]): bool = - var valid = true - for key in data.keys: - if key notin allowed: - valid = false - errors.add(ManifestError( - code: StrictViolation, - field: key, - context: "Unauthorized field found: '" & key & "'", - suggestions: @[ - "Remove the field", - "Check spelling against spec", - "This field may be format-specific"] - )) - return valid - ) - -proc createSemVerRule*(fieldName: string): ValidationRule = - ## Enforce semantic versioning on a field - return ValidationRule( - name: "semver_" & fieldName, - description: "Field '" & fieldName & "' must be valid SemVer (X.Y.Z)", - validate: proc(data: JsonNode, errors: var seq[ManifestError]): bool = - if fieldName notin data: return true # Required check handles missing - - let val = data[fieldName].getStr() - if not isValidSemVer(val): - errors.add(ManifestError( - code: SemVerViolation, - field: fieldName, - context: "'" & val & "' is not a valid SemVer", - suggestions: @[ - "Use format X.Y.Z (e.g., 1.0.0)", - "Prerelease: 1.0.0-alpha", - "Build metadata: 1.0.0+20130313144700" - ] - )) - return false - return true - ) - -proc createPlatformConstraintRule*(): ValidationRule = - ## Validate OS and Architecture constraints - return ValidationRule( - name: "platform_constraints", - description: "Validates OS and Architecture targets", - validate: proc(data: JsonNode, errors: var seq[ManifestError]): bool = - var valid = true - - # Validate OS field - if "os" in data or "supported_os" in data: - let osField = if "os" in data: "os" else: "supported_os" - let osNode = data[osField] - - if osNode.kind != JArray: - errors.add(ManifestError( - code: InvalidValue, - field: osField, - context: "OS field must be an array", - suggestions: @["Use array format: [\"linux\", \"freebsd\"]"] - )) - valid = false - elif osNode.len == 0: - errors.add(ManifestError( - code: InvalidValue, - field: osField, - context: "OS array cannot be empty", - suggestions: @["Specify at least one OS"] - )) - valid = false - else: - # Validate each OS value - for osVal in osNode: - let os = osVal.getStr() - if os notin VALID_OS: - errors.add(ManifestError( - code: PlatformIncompat, - field: osField, - context: "Invalid OS: " & os, - suggestions: @["Valid OS: " & $VALID_OS] - )) - valid = false - - # Validate Architecture field - if "arch" in data or "supported_architectures" in data: - let archField = if "arch" in data: "arch" else: "supported_architectures" - let archNode = data[archField] - - if archNode.kind != JArray: - errors.add(ManifestError( - code: InvalidValue, - field: archField, - context: "Architecture field must be an array", - suggestions: @["Use array format: [\"x86_64\", \"aarch64\"]"] - )) - valid = false - elif archNode.len == 0: - errors.add(ManifestError( - code: InvalidValue, - field: archField, - context: "Architecture array cannot be empty", - suggestions: @["Specify at least one architecture"] - )) - valid = false - else: - # Validate each architecture value - for archVal in archNode: - let arch = archVal.getStr() - if arch notin VALID_ARCHITECTURES: - errors.add(ManifestError( - code: PlatformIncompat, - field: archField, - context: "Invalid architecture: " & arch, - suggestions: @["Valid architectures: " & $VALID_ARCHITECTURES] - )) - valid = false - - return valid - ) - -proc createRequiredFieldsRule*(fields: seq[string]): ValidationRule = - ## Ensure required fields are present - return ValidationRule( - name: "required_fields", - description: "Ensures all required fields are present", - validate: proc(data: JsonNode, errors: var seq[ManifestError]): bool = - var valid = true - for field in fields: - if field notin data: - errors.add(ManifestError( - code: MissingField, - field: field, - context: "Required field '" & field & "' is missing", - suggestions: @["Add the field to the manifest"] - )) - valid = false - return valid - ) - -proc createDependencyRule*(): ValidationRule = - ## Validate dependency specifications - return ValidationRule( - name: "dependencies", - description: "Validates dependency specifications", - validate: proc(data: JsonNode, errors: var seq[ManifestError]): bool = - var valid = true - - for depField in ["dependencies", "build_dependencies", - "optional_dependencies"]: - if depField notin data: continue - - let deps = data[depField] - if deps.kind != JArray: - errors.add(ManifestError( - code: SchemaError, - field: depField, - context: "Dependencies must be an array", - suggestions: @["Use array format"] - )) - valid = false - continue - - for dep in deps: - if dep.kind != JObject: - errors.add(ManifestError( - code: SchemaError, - field: depField, - context: "Each dependency must be an object", - suggestions: @["Use object format: {\"name\": \"pkg\", \"version\": \">=1.0.0\"}"] - )) - valid = false - continue - - # Check required dependency fields - if "name" notin dep: - errors.add(ManifestError( - code: MissingField, - field: depField & ".name", - context: "Dependency missing 'name' field", - suggestions: @["Add name field"] - )) - valid = false - - # Validate version constraint if present - if "version" in dep: - let versionStr = dep["version"].getStr() - try: - discard parseVersionConstraint(versionStr) - except ManifestError as e: - errors.add(ManifestError( - code: DependencyError, - field: depField & ".version", - context: "Invalid version constraint: " & versionStr, - suggestions: @[ - "Use valid constraint: >=1.0.0, ~1.2.0, ^2.0.0", - e.msg - ] - )) - valid = false - - return valid - ) - -# ============================================================================ -# Parser Construction -# ============================================================================ - -proc newManifestParser*(format: FormatType, - wireFormat: ManifestFormat = FormatAuto, - strict: bool = true): ManifestParser = - ## Create a new manifest parser with format-specific rules - var parser = ManifestParser( - config: ParserConfig( - format: format, - wireFormat: wireFormat, - strictMode: strict, - allowedFields: BASE_ALLOWED_FIELDS - ), - rules: @[], - warnings: @[] - ) - - # Add format-specific allowed fields - case format: - of NPK: - for field in NPK_SPECIFIC_FIELDS: - parser.config.allowedFields.incl(field) - of NIP: - for field in NIP_SPECIFIC_FIELDS: - parser.config.allowedFields.incl(field) - of NEXTER: - for field in NEXTER_SPECIFIC_FIELDS: - parser.config.allowedFields.incl(field) - - # Add common validation rules - parser.rules.add(createRequiredFieldsRule(@["name", "version", "license"])) - parser.rules.add(createSemVerRule("version")) - parser.rules.add(createPlatformConstraintRule()) - parser.rules.add(createDependencyRule()) - - # Add strict mode rule (The Bouncer) - LAST - if strict: - parser.rules.add(createStrictStructRule(parser.config.allowedFields)) - - return parser - -# ============================================================================ -# Platform Validation -# ============================================================================ - -proc checkPlatformCompatibility*(manifest: PackageManifest, - caps: PlatformCapabilities): bool = - ## Check if package is compatible with current platform - - # Check OS - if manifest.supportedOS.len > 0: - let currentOS = $caps.osType - if currentOS notin manifest.supportedOS: - return false - - # Check required capabilities - for cap in manifest.requiredCapabilities: - case cap: - of "user_namespaces": - if not caps.hasUserNamespaces: - return false - of "jails": - if not caps.hasJails: - return false - of "unveil": - if not caps.hasUnveil: - return false - else: - # Unknown capability - be conservative - return false - - return true - - -# ============================================================================ -# JSON Parsing (Machine-Friendly) -# ============================================================================ - -proc parseManifestFromJSON*(content: string, - parser: var ManifestParser): PackageManifest = - ## Parse package manifest from JSON format - ## This is the machine-friendly format for automated systems - - let jsonNode = try: - parseJson(content) - except JsonParsingError as e: - raise newException(ManifestError, "JSON parse error: " & e.msg) - - if jsonNode.kind != JObject: - raise newException(ManifestError, "JSON manifest must be an object") - - # Run validation rules - var errors: seq[ManifestError] = @[] - for rule in parser.rules: - discard rule.validate(jsonNode, errors) - - if errors.len > 0: - # Collect all errors into one exception - var msg = "Manifest validation failed:\n" - for err in errors: - msg.add(" [" & $err.code & "] " & err.context & "\n") - for suggestion in err.suggestions: - msg.add(" → " & suggestion & "\n") - raise newException(ManifestError, msg) - - # Extract manifest data (safe after validation) - var manifest = PackageManifest( - format: parser.config.format, - name: jsonNode["name"].getStr(), - version: parseSemanticVersion(jsonNode["version"].getStr()), - license: jsonNode["license"].getStr() - ) - - # Optional fields - if jsonNode.hasKey("description"): - manifest.description = some(jsonNode["description"].getStr()) - - if jsonNode.hasKey("homepage"): - manifest.homepage = some(jsonNode["homepage"].getStr()) - - if jsonNode.hasKey("author"): - manifest.author = some(jsonNode["author"].getStr()) - - if jsonNode.hasKey("timestamp"): - manifest.timestamp = some(jsonNode["timestamp"].getStr()) - - if jsonNode.hasKey("libc"): - manifest.libc = some(jsonNode["libc"].getStr()) - - if jsonNode.hasKey("allocator"): - manifest.allocator = some(jsonNode["allocator"].getStr()) - - if jsonNode.hasKey("build_system"): - manifest.buildSystem = some(jsonNode["build_system"].getStr()) - - # Integrity hashes - if jsonNode.hasKey("build_hash"): - manifest.buildHash = jsonNode["build_hash"].getStr() - - if jsonNode.hasKey("source_hash"): - manifest.sourceHash = jsonNode["source_hash"].getStr() - - if jsonNode.hasKey("artifact_hash"): - manifest.artifactHash = jsonNode["artifact_hash"].getStr() - - # UTCP support - if jsonNode.hasKey("utcp_endpoint"): - manifest.utcpEndpoint = some(jsonNode["utcp_endpoint"].getStr()) - - if jsonNode.hasKey("utcp_version"): - manifest.utcpVersion = some(jsonNode["utcp_version"].getStr()) - - # Platform constraints - if jsonNode.hasKey("os") or jsonNode.hasKey("supported_os"): - let osField = if jsonNode.hasKey("os"): "os" else: "supported_os" - for os in jsonNode[osField]: - manifest.supportedOS.add(os.getStr()) - - if jsonNode.hasKey("arch") or jsonNode.hasKey("supported_architectures"): - let archField = if jsonNode.hasKey("arch"): "arch" else: "supported_architectures" - for arch in jsonNode[archField]: - manifest.supportedArchitectures.add(arch.getStr()) - - if jsonNode.hasKey("required_capabilities"): - for cap in jsonNode["required_capabilities"]: - manifest.requiredCapabilities.add(cap.getStr()) - - # Dependencies - if jsonNode.hasKey("dependencies"): - for dep in jsonNode["dependencies"]: - var depSpec = DependencySpec( - name: dep["name"].getStr(), - optional: false - ) - if dep.hasKey("version"): - depSpec.versionConstraint = parseVersionConstraint(dep[ - "version"].getStr()) - if dep.hasKey("optional"): - depSpec.optional = dep["optional"].getBool() - if dep.hasKey("features"): - for feature in dep["features"]: - depSpec.features.add(feature.getStr()) - manifest.dependencies.add(depSpec) - - if jsonNode.hasKey("build_dependencies"): - for dep in jsonNode["build_dependencies"]: - var depSpec = DependencySpec( - name: dep["name"].getStr(), - optional: false - ) - if dep.hasKey("version"): - depSpec.versionConstraint = parseVersionConstraint(dep[ - "version"].getStr()) - manifest.buildDependencies.add(depSpec) - - if jsonNode.hasKey("optional_dependencies"): - for dep in jsonNode["optional_dependencies"]: - var depSpec = DependencySpec( - name: dep["name"].getStr(), - optional: true - ) - if dep.hasKey("version"): - depSpec.versionConstraint = parseVersionConstraint(dep[ - "version"].getStr()) - manifest.optionalDependencies.add(depSpec) - - # Build configuration - if jsonNode.hasKey("build_flags"): - for flag in jsonNode["build_flags"]: - manifest.buildFlags.add(flag.getStr()) - - if jsonNode.hasKey("configure_flags"): - for flag in jsonNode["configure_flags"]: - manifest.configureFlags.add(flag.getStr()) - - # Metadata - if jsonNode.hasKey("tags"): - for tag in jsonNode["tags"]: - manifest.tags.add(tag.getStr()) - - if jsonNode.hasKey("maintainers"): - for maintainer in jsonNode["maintainers"]: - manifest.maintainers.add(maintainer.getStr()) - - # System Integration - if jsonNode.hasKey("files"): - for file in jsonNode["files"]: - manifest.files.add(FileSpec( - path: file["path"].getStr(), - hash: file["hash"].getStr(), - size: file["size"].getBiggestInt(), - permissions: file.getOrDefault("permissions").getStr("644") - )) - - if jsonNode.hasKey("users"): - for user in jsonNode["users"]: - var userSpec = UserSpec( - name: user["name"].getStr(), - group: user.getOrDefault("group").getStr(user["name"].getStr()), - shell: user.getOrDefault("shell").getStr("/bin/false"), - home: user.getOrDefault("home").getStr("/var/empty") - ) - if user.hasKey("uid"): - userSpec.uid = some(user["uid"].getInt()) - manifest.users.add(userSpec) - - if jsonNode.hasKey("groups"): - for group in jsonNode["groups"]: - var groupSpec = GroupSpec(name: group["name"].getStr()) - if group.hasKey("gid"): - groupSpec.gid = some(group["gid"].getInt()) - manifest.groups.add(groupSpec) - - if jsonNode.hasKey("services"): - for service in jsonNode["services"]: - manifest.services.add(ServiceSpec( - name: service["name"].getStr(), - content: service.getOrDefault("content").getStr(""), - enabled: service.getOrDefault("enabled").getBool(true) - )) - - # Security / Sandbox - if jsonNode.hasKey("sandbox"): - let sbNode = jsonNode["sandbox"] - var config = SandboxConfig( - level: parseSandboxLevel(sbNode.getOrDefault("level").getStr("strict")) - ) - - # Linux - if sbNode.hasKey("linux"): - let linuxNode = sbNode["linux"] - if linuxNode.hasKey("seccomp"): - config.seccompProfile = some(linuxNode["seccomp"].getStr()) - - if linuxNode.hasKey("capabilities"): - for cap in linuxNode["capabilities"]: - config.capabilities.add(cap.getStr()) - - if linuxNode.hasKey("namespaces"): - for ns in linuxNode["namespaces"]: - config.namespaces.add(ns.getStr()) - - # BSD - if sbNode.hasKey("bsd"): - let bsdNode = sbNode["bsd"] - if bsdNode.hasKey("pledge"): - config.pledge = some(bsdNode["pledge"].getStr()) - - if bsdNode.hasKey("unveil"): - for path in bsdNode["unveil"]: - config.unveil.add(path.getStr()) - - manifest.sandbox = some(config) - - # Desktop Integration - if jsonNode.hasKey("desktop"): - let dtNode = jsonNode["desktop"] - var dt = DesktopIntegration( - displayName: dtNode.getOrDefault("display_name").getStr(manifest.name), - terminal: dtNode.getOrDefault("terminal").getBool(false), - startupNotify: dtNode.getOrDefault("startup_notify").getBool(true) - ) - - if dtNode.hasKey("icon"): - dt.icon = some(dtNode["icon"].getStr()) - - if dtNode.hasKey("startup_wm_class"): - dt.startupWMClass = some(dtNode["startup_wm_class"].getStr()) - - if dtNode.hasKey("categories"): - for cat in dtNode["categories"]: - dt.categories.add(cat.getStr()) - - if dtNode.hasKey("keywords"): - for kw in dtNode["keywords"]: - dt.keywords.add(kw.getStr()) - - if dtNode.hasKey("mime_types"): - for mt in dtNode["mime_types"]: - dt.mimeTypes.add(mt.getStr()) - - manifest.desktop = some(dt) - - return manifest - -# ============================================================================ -# KDL Parsing (Human-Friendly) - NATIVE IMPLEMENTATION -# ============================================================================ - -proc parseManifestFromKDL*(content: string, - parser: var ManifestParser): PackageManifest = - ## Parse package manifest from KDL format using NATIVE KDL structures - ## No JSON conversion - direct KDL parsing for maximum efficiency - - let doc = parseKdlString(content) - - # Find package node - let packageNode = doc.findNode("package") - if packageNode.isNone: - raise newException(ManifestError, "Missing 'package' node in KDL manifest") - - let pkg = packageNode.get() - - # Track seen fields for strict validation - var seenFields: HashSet[string] - - # Initialize manifest - var manifest = PackageManifest(format: parser.config.format) - - # Extract name from first argument (REQUIRED) - if pkg.args.len > 0: - manifest.name = pkg.getArgString(0) - seenFields.incl("name") - else: - raise newException(ManifestError, "Missing package name") - - # Extract properties (NATIVE KDL) - # Extract properties (NATIVE KDL) - if pkg.hasProp("version"): - let versionStr = pkg.getPropString("version") - manifest.version = parseSemanticVersion(versionStr) - seenFields.incl("version") - else: - # Check for child node "version" - let verNode = pkg.findChild("version") - if verNode.isSome: - let versionStr = verNode.get().getArgString(0) - manifest.version = parseSemanticVersion(versionStr) - seenFields.incl("version") - else: - raise newException(ManifestError, "Missing package version") - - if pkg.hasProp("license"): - manifest.license = pkg.getPropString("license") - seenFields.incl("license") - else: - # Check for child node "license" - let licNode = pkg.findChild("license") - if licNode.isSome: - manifest.license = licNode.get().getArgString(0) - seenFields.incl("license") - else: - raise newException(ManifestError, "Missing package license") - - # Optional properties - if pkg.hasProp("description"): - manifest.description = some(pkg.getPropString("description")) - seenFields.incl("description") - - if pkg.hasProp("homepage"): - manifest.homepage = some(pkg.getPropString("homepage")) - seenFields.incl("homepage") - - if pkg.hasProp("author"): - manifest.author = some(pkg.getPropString("author")) - seenFields.incl("author") - - if pkg.hasProp("timestamp"): - manifest.timestamp = some(pkg.getPropString("timestamp")) - seenFields.incl("timestamp") - - if pkg.hasProp("libc"): - manifest.libc = some(pkg.getPropString("libc")) - seenFields.incl("libc") - - if pkg.hasProp("allocator"): - manifest.allocator = some(pkg.getPropString("allocator")) - seenFields.incl("allocator") - - if pkg.hasProp("build_system"): - manifest.buildSystem = some(pkg.getPropString("build_system")) - seenFields.incl("build_system") - - # Integrity hashes - if pkg.hasProp("build_hash"): - manifest.buildHash = pkg.getPropString("build_hash") - seenFields.incl("build_hash") - - if pkg.hasProp("source_hash"): - manifest.sourceHash = pkg.getPropString("source_hash") - seenFields.incl("source_hash") - - if pkg.hasProp("artifact_hash"): - manifest.artifactHash = pkg.getPropString("artifact_hash") - seenFields.incl("artifact_hash") - - # UTCP support - if pkg.hasProp("utcp_endpoint"): - manifest.utcpEndpoint = some(pkg.getPropString("utcp_endpoint")) - seenFields.incl("utcp_endpoint") - - if pkg.hasProp("utcp_version"): - manifest.utcpVersion = some(pkg.getPropString("utcp_version")) - seenFields.incl("utcp_version") - - # Parse child nodes (NATIVE KDL) - for child in pkg.children: - seenFields.incl(child.name) - - case child.name: - of "dependencies": - for dep in child.children: - if dep.args.len == 0: - raise newException(ManifestError, "Dependency missing name") - - var depSpec = DependencySpec( - name: dep.getArgString(0), - optional: false - ) - - if dep.hasProp("version"): - depSpec.versionConstraint = parseVersionConstraint(dep.getPropString("version")) - - if dep.hasProp("optional"): - depSpec.optional = dep.getPropBool("optional") - - if dep.hasProp("features"): - let featuresStr = dep.getPropString("features") - depSpec.features = featuresStr.split(',').mapIt(it.strip()) - - manifest.dependencies.add(depSpec) - - of "build_dependencies": - for dep in child.children: - if dep.args.len == 0: - raise newException(ManifestError, "Build dependency missing name") - - var depSpec = DependencySpec( - name: dep.getArgString(0), - optional: false - ) - - if dep.hasProp("version"): - depSpec.versionConstraint = parseVersionConstraint(dep.getPropString("version")) - - manifest.buildDependencies.add(depSpec) - - of "optional_dependencies": - for dep in child.children: - if dep.args.len == 0: - raise newException(ManifestError, "Optional dependency missing name") - - var depSpec = DependencySpec( - name: dep.getArgString(0), - optional: true - ) - - if dep.hasProp("version"): - depSpec.versionConstraint = parseVersionConstraint(dep.getPropString("version")) - - manifest.optionalDependencies.add(depSpec) - - of "build_flags": - for flag in child.children: - if flag.args.len > 0: - manifest.buildFlags.add(flag.getArgString(0)) - - of "configure_flags": - for flag in child.children: - if flag.args.len > 0: - manifest.configureFlags.add(flag.getArgString(0)) - - of "os", "supported_os": - for os in child.children: - if os.args.len > 0: - let osStr = os.getArgString(0) - if parser.config.strictMode and osStr notin VALID_OS: - raise newException(ManifestError, "Invalid OS: " & osStr) - manifest.supportedOS.add(osStr) - - of "arch", "supported_architectures": - for arch in child.children: - if arch.args.len > 0: - let archStr = arch.getArgString(0) - if parser.config.strictMode and archStr notin VALID_ARCHITECTURES: - raise newException(ManifestError, "Invalid architecture: " & archStr) - manifest.supportedArchitectures.add(archStr) - - of "required_capabilities": - for cap in child.children: - if cap.args.len > 0: - manifest.requiredCapabilities.add(cap.getArgString(0)) - - of "tags": - for tag in child.children: - if tag.args.len > 0: - manifest.tags.add(tag.getArgString(0)) - - of "maintainers": - for maintainer in child.children: - if maintainer.args.len > 0: - manifest.maintainers.add(maintainer.getArgString(0)) - - of "files": - for file in child.children: - if file.args.len == 0: - raise newException(ManifestError, "File missing path") - - var fileSpec = FileSpec( - path: file.getArgString(0), - hash: file.getPropString("hash"), - size: file.getPropInt("size"), - permissions: file.getPropString("permissions", "644") - ) - manifest.files.add(fileSpec) - - of "users": - for user in child.children: - if user.args.len == 0: - raise newException(ManifestError, "User missing name") - - var userSpec = UserSpec( - name: user.getArgString(0), - group: user.getPropString("group", user.getArgString(0)), - shell: user.getPropString("shell", "/bin/false"), - home: user.getPropString("home", "/var/empty") - ) - if user.hasProp("uid"): - userSpec.uid = some(user.getPropInt("uid").int) - - manifest.users.add(userSpec) - - of "groups": - for group in child.children: - if group.args.len == 0: - raise newException(ManifestError, "Group missing name") - - var groupSpec = GroupSpec(name: group.getArgString(0)) - if group.hasProp("gid"): - groupSpec.gid = some(group.getPropInt("gid").int) - - manifest.groups.add(groupSpec) - - of "services": - for service in child.children: - if service.args.len == 0: - raise newException(ManifestError, "Service missing name") - - var serviceSpec = ServiceSpec( - name: service.getArgString(0), - content: service.getPropString("content", ""), - enabled: service.getPropBool("enabled", true) - ) - manifest.services.add(serviceSpec) - - manifest.services.add(serviceSpec) - - of "sandbox": - var config = SandboxConfig( - level: parseSandboxLevel(child.getPropString("level", "strict")) - ) - - for sbChild in child.children: - case sbChild.name: - of "linux": - if sbChild.hasProp("seccomp"): - config.seccompProfile = some(sbChild.getPropString("seccomp")) - - for linuxChild in sbChild.children: - case linuxChild.name: - of "capabilities": - for cap in linuxChild.args: - config.capabilities.add(cap.getString()) - of "namespaces": - for ns in linuxChild.args: - config.namespaces.add(ns.getString()) - - of "bsd": - if sbChild.hasProp("pledge"): - config.pledge = some(sbChild.getPropString("pledge")) - - for bsdChild in sbChild.children: - case bsdChild.name: - of "unveil": - for path in bsdChild.args: - config.unveil.add(path.getString()) - - manifest.sandbox = some(config) - - of "desktop": - var dt = DesktopIntegration( - displayName: child.getPropString("display_name", manifest.name), - terminal: child.getPropBool("terminal", false), - startupNotify: child.getPropBool("startup_notify", true) - ) - - if child.hasProp("icon"): - dt.icon = some(child.getPropString("icon")) - - if child.hasProp("startup_wm_class"): - dt.startupWMClass = some(child.getPropString("startup_wm_class")) - - for dtChild in child.children: - case dtChild.name: - of "categories": - for cat in dtChild.args: - dt.categories.add(cat.getString()) - of "keywords": - for kw in dtChild.args: - dt.keywords.add(kw.getString()) - of "mime_types": - for mt in dtChild.args: - dt.mimeTypes.add(mt.getString()) - - manifest.desktop = some(dt) - - else: - if parser.config.strictMode and child.name notin - parser.config.allowedFields: - raise newException(ManifestError, "Unknown field: " & child.name) - else: - parser.warnings.add("Unknown field: " & child.name) - - # Strict mode: check for unknown properties - if parser.config.strictMode: - for key in keys(pkg.props): - if key notin parser.config.allowedFields: - raise newException(ManifestError, "Unknown property: " & key) - - return manifest - -# ============================================================================ -# High-Level API -# ============================================================================ - -proc parseManifest*(content: string, - format: FormatType = NPK, - wireFormat: ManifestFormat = FormatAuto, - validationMode: ManifestValidationMode = ValidationStrict): PackageManifest = - ## Parse package manifest from string (auto-detects wire format) - var parser = newManifestParser(format, wireFormat, validationMode == ValidationStrict) - - let actualFormat = if wireFormat == FormatAuto: - detectFormat(content) - else: - wireFormat - - case actualFormat: - of FormatKDL: - return parseManifestFromKDL(content, parser) - of FormatJSON: - return parseManifestFromJSON(content, parser) - of FormatAuto: - raise newException(ManifestError, "Format detection failed") - -proc parseManifestFile*(path: string, - format: FormatType = NPK, - wireFormat: ManifestFormat = FormatAuto, - validationMode: ManifestValidationMode = ValidationStrict): PackageManifest = - ## Parse package manifest from file - let content = readFile(path) - return parseManifest(content, format, wireFormat, validationMode) - -proc validateManifest*(manifest: PackageManifest): seq[string] = - ## Validate manifest and return list of issues (empty if valid) - result = @[] - - # Validate name - if manifest.name.len == 0: - result.add("Package name cannot be empty") - - # Version is already validated during parsing (it's a SemanticVersion) - - # Validate license - if manifest.license.len == 0: - result.add("License cannot be empty") - - # Platform compatibility is checked separately via checkPlatformCompatibility - -# ============================================================================ -# Serialization -# ============================================================================ - -proc serializeManifestToJSON*(manifest: PackageManifest): string = - ## Serialize manifest to JSON format (machine-friendly) - var jsonObj = %* { - "name": manifest.name, - "version": $manifest.version, - "license": manifest.license - } - - if manifest.description.isSome: - jsonObj["description"] = %manifest.description.get() - - if manifest.homepage.isSome: - jsonObj["homepage"] = %manifest.homepage.get() - - if manifest.author.isSome: - jsonObj["author"] = %manifest.author.get() - - if manifest.timestamp.isSome: - jsonObj["timestamp"] = %manifest.timestamp.get() - - if manifest.libc.isSome: - jsonObj["libc"] = %manifest.libc.get() - - if manifest.allocator.isSome: - jsonObj["allocator"] = %manifest.allocator.get() - - if manifest.buildSystem.isSome: - jsonObj["build_system"] = %manifest.buildSystem.get() - - # Integrity - if manifest.buildHash.len > 0: - jsonObj["build_hash"] = %manifest.buildHash - - if manifest.sourceHash.len > 0: - jsonObj["source_hash"] = %manifest.sourceHash - - if manifest.artifactHash.len > 0: - jsonObj["artifact_hash"] = %manifest.artifactHash - - # UTCP - if manifest.utcpEndpoint.isSome: - jsonObj["utcp_endpoint"] = %manifest.utcpEndpoint.get() - - if manifest.utcpVersion.isSome: - jsonObj["utcp_version"] = %manifest.utcpVersion.get() - - # Platform - if manifest.supportedOS.len > 0: - jsonObj["supported_os"] = %manifest.supportedOS - - if manifest.supportedArchitectures.len > 0: - jsonObj["supported_architectures"] = %manifest.supportedArchitectures - - if manifest.requiredCapabilities.len > 0: - jsonObj["required_capabilities"] = %manifest.requiredCapabilities - - # Dependencies - if manifest.dependencies.len > 0: - var deps = newJArray() - for dep in manifest.dependencies: - var depObj = %* {"name": dep.name} - if dep.versionConstraint.operator != OpAny: - depObj["version"] = %($dep.versionConstraint.operator & - $dep.versionConstraint.version) - if dep.optional: - depObj["optional"] = %true - if dep.features.len > 0: - depObj["features"] = %dep.features - deps.add(depObj) - jsonObj["dependencies"] = deps - - # System Integration - if manifest.files.len > 0: - var files = newJArray() - for file in manifest.files: - files.add(%*{ - "path": file.path, - "hash": file.hash, - "size": file.size, - "permissions": file.permissions - }) - jsonObj["files"] = files - - if manifest.users.len > 0: - var users = newJArray() - for user in manifest.users: - var userObj = %*{ - "name": user.name, - "group": user.group, - "shell": user.shell, - "home": user.home - } - if user.uid.isSome: - userObj["uid"] = %user.uid.get() - users.add(userObj) - jsonObj["users"] = users - - if manifest.groups.len > 0: - var groups = newJArray() - for group in manifest.groups: - var groupObj = %*{"name": group.name} - if group.gid.isSome: - groupObj["gid"] = %group.gid.get() - groups.add(groupObj) - jsonObj["groups"] = groups - - if manifest.services.len > 0: - var services = newJArray() - for service in manifest.services: - services.add(%*{ - "name": service.name, - "content": service.content, - "enabled": service.enabled - }) - jsonObj["services"] = services - - # Security / Sandbox - if manifest.sandbox.isSome: - let sb = manifest.sandbox.get() - var sbObj = %*{"level": $sb.level} - - var linuxObj = newJObject() - if sb.seccompProfile.isSome: - linuxObj["seccomp"] = %sb.seccompProfile.get() - if sb.capabilities.len > 0: - linuxObj["capabilities"] = %sb.capabilities - if sb.namespaces.len > 0: - linuxObj["namespaces"] = %sb.namespaces - if linuxObj.len > 0: - sbObj["linux"] = linuxObj - - var bsdObj = newJObject() - if sb.pledge.isSome: - bsdObj["pledge"] = %sb.pledge.get() - if sb.unveil.len > 0: - bsdObj["unveil"] = %sb.unveil - if bsdObj.len > 0: - sbObj["bsd"] = bsdObj - - jsonObj["sandbox"] = sbObj - - # Desktop Integration - if manifest.desktop.isSome: - let dt = manifest.desktop.get() - var dtObj = %*{ - "display_name": dt.displayName, - "terminal": dt.terminal, - "startup_notify": dt.startupNotify - } - - if dt.icon.isSome: - dtObj["icon"] = %dt.icon.get() - if dt.startupWMClass.isSome: - dtObj["startup_wm_class"] = %dt.startupWMClass.get() - if dt.categories.len > 0: - dtObj["categories"] = %dt.categories - if dt.keywords.len > 0: - dtObj["keywords"] = %dt.keywords - if dt.mimeTypes.len > 0: - dtObj["mime_types"] = %dt.mimeTypes - - jsonObj["desktop"] = dtObj - - return $jsonObj - -proc serializeManifestToKDL*(manifest: PackageManifest): string = - ## Serialize manifest to KDL format (human-friendly) - ## Complete implementation with all fields for perfect roundtrip - result = "package \"" & manifest.name & "\" {\n" - - # Required fields - result.add(" version \"" & $manifest.version & "\"\n") - result.add(" license \"" & manifest.license & "\"\n") - - # Optional identity fields - if manifest.description.isSome: - result.add(" description \"" & manifest.description.get() & "\"\n") - - if manifest.homepage.isSome: - result.add(" homepage \"" & manifest.homepage.get() & "\"\n") - - if manifest.author.isSome: - result.add(" author \"" & manifest.author.get() & "\"\n") - - if manifest.timestamp.isSome: - result.add(" timestamp \"" & manifest.timestamp.get() & "\"\n") - - # Runtime configuration - if manifest.libc.isSome: - result.add(" libc \"" & manifest.libc.get() & "\"\n") - - if manifest.allocator.isSome: - result.add(" allocator \"" & manifest.allocator.get() & "\"\n") - - if manifest.buildSystem.isSome: - result.add(" build_system \"" & manifest.buildSystem.get() & "\"\n") - - # Integrity hashes - if manifest.buildHash.len > 0: - result.add(" build_hash \"" & manifest.buildHash & "\"\n") - - if manifest.sourceHash.len > 0: - result.add(" source_hash \"" & manifest.sourceHash & "\"\n") - - if manifest.artifactHash.len > 0: - result.add(" artifact_hash \"" & manifest.artifactHash & "\"\n") - - # UTCP support - if manifest.utcpEndpoint.isSome: - result.add(" utcp_endpoint \"" & manifest.utcpEndpoint.get() & "\"\n") - - if manifest.utcpVersion.isSome: - result.add(" utcp_version \"" & manifest.utcpVersion.get() & "\"\n") - - # Dependencies (child nodes) - if manifest.dependencies.len > 0: - result.add("\n dependencies {\n") - for dep in manifest.dependencies: - result.add(" \"" & dep.name & "\"") - if dep.versionConstraint.operator != OpAny: - result.add(" version=\"" & $dep.versionConstraint.operator & - $dep.versionConstraint.version & "\"") - if dep.optional: - result.add(" optional=true") - if dep.features.len > 0: - result.add(" features=\"" & dep.features.join(",") & "\"") - result.add("\n") - result.add(" }\n") - - # Build dependencies - if manifest.buildDependencies.len > 0: - result.add("\n build_dependencies {\n") - for dep in manifest.buildDependencies: - result.add(" \"" & dep.name & "\"") - if dep.versionConstraint.operator != OpAny: - result.add(" version=\"" & $dep.versionConstraint.operator & - $dep.versionConstraint.version & "\"") - result.add("\n") - result.add(" }\n") - - # Optional dependencies - if manifest.optionalDependencies.len > 0: - result.add("\n optional_dependencies {\n") - for dep in manifest.optionalDependencies: - result.add(" \"" & dep.name & "\"") - if dep.versionConstraint.operator != OpAny: - result.add(" version=\"" & $dep.versionConstraint.operator & - $dep.versionConstraint.version & "\"") - if dep.features.len > 0: - result.add(" features=\"" & dep.features.join(",") & "\"") - result.add("\n") - result.add(" }\n") - - # Build configuration - if manifest.buildFlags.len > 0: - result.add("\n build_flags {\n") - for flag in manifest.buildFlags: - result.add(" \"" & flag & "\"\n") - result.add(" }\n") - - if manifest.configureFlags.len > 0: - result.add("\n configure_flags {\n") - for flag in manifest.configureFlags: - result.add(" \"" & flag & "\"\n") - result.add(" }\n") - - # Platform constraints - if manifest.supportedOS.len > 0: - result.add("\n supported_os {\n") - for os in manifest.supportedOS: - result.add(" \"" & os & "\"\n") - result.add(" }\n") - - if manifest.supportedArchitectures.len > 0: - result.add("\n supported_architectures {\n") - for arch in manifest.supportedArchitectures: - result.add(" \"" & arch & "\"\n") - result.add(" }\n") - - if manifest.requiredCapabilities.len > 0: - result.add("\n required_capabilities {\n") - for cap in manifest.requiredCapabilities: - result.add(" \"" & cap & "\"\n") - result.add(" }\n") - - # Metadata - if manifest.tags.len > 0: - result.add("\n tags {\n") - for tag in manifest.tags: - result.add(" \"" & tag & "\"\n") - result.add(" }\n") - - if manifest.maintainers.len > 0: - result.add("\n maintainers {\n") - for maintainer in manifest.maintainers: - result.add(" \"" & maintainer & "\"\n") - result.add(" }\n") - - # System Integration - if manifest.files.len > 0: - result.add("\n files {\n") - for file in manifest.files: - result.add(" file \"" & file.path & "\" hash=\"" & file.hash & - "\" size=" & $file.size & " permissions=\"" & file.permissions & "\"\n") - result.add(" }\n") - - if manifest.users.len > 0: - result.add("\n users {\n") - for user in manifest.users: - result.add(" \"" & user.name & "\" group=\"" & user.group & - "\" shell=\"" & user.shell & "\" home=\"" & user.home & "\"") - if user.uid.isSome: - result.add(" uid=" & $user.uid.get()) - result.add("\n") - result.add(" }\n") - - if manifest.groups.len > 0: - result.add("\n groups {\n") - for group in manifest.groups: - result.add(" \"" & group.name & "\"") - if group.gid.isSome: - result.add(" gid=" & $group.gid.get()) - result.add("\n") - result.add(" }\n") - - if manifest.services.len > 0: - result.add("\n services {\n") - for service in manifest.services: - result.add(" \"" & service.name & "\" enabled=" & $service.enabled & - " content=" & service.content.escape() & "\n") - result.add(" }\n") - - # Security / Sandbox - if manifest.sandbox.isSome: - let sb = manifest.sandbox.get() - result.add("\n sandbox level=\"" & $sb.level & "\" {\n") - - # Linux - if sb.seccompProfile.isSome or sb.capabilities.len > 0 or - sb.namespaces.len > 0: - result.add(" linux") - if sb.seccompProfile.isSome: - result.add(" seccomp=\"" & sb.seccompProfile.get() & "\"") - result.add(" {\n") - - if sb.capabilities.len > 0: - result.add(" capabilities") - for cap in sb.capabilities: - result.add(" \"" & cap & "\"") - result.add("\n") - - if sb.namespaces.len > 0: - result.add(" namespaces") - for ns in sb.namespaces: - result.add(" \"" & ns & "\"") - result.add("\n") - result.add(" }\n") - - # BSD - if sb.pledge.isSome or sb.unveil.len > 0: - result.add(" bsd") - if sb.pledge.isSome: - result.add(" pledge=\"" & sb.pledge.get() & "\"") - result.add(" {\n") - - if sb.unveil.len > 0: - result.add(" unveil") - for path in sb.unveil: - result.add(" \"" & path & "\"") - result.add("\n") - result.add(" }\n") - - result.add(" }\n") - - # Desktop Integration - if manifest.desktop.isSome: - let dt = manifest.desktop.get() - result.add("\n desktop display_name=\"" & dt.displayName & "\" terminal=" & - $dt.terminal & " startup_notify=" & $dt.startupNotify) - if dt.icon.isSome: - result.add(" icon=\"" & dt.icon.get() & "\"") - if dt.startupWMClass.isSome: - result.add(" startup_wm_class=\"" & dt.startupWMClass.get() & "\"") - result.add(" {\n") - - if dt.categories.len > 0: - result.add(" categories") - for cat in dt.categories: - result.add(" \"" & cat & "\"") - result.add("\n") - - if dt.keywords.len > 0: - result.add(" keywords") - for kw in dt.keywords: - result.add(" \"" & kw & "\"") - result.add("\n") - - if dt.mimeTypes.len > 0: - result.add(" mime_types") - for mt in dt.mimeTypes: - result.add(" \"" & mt & "\"") - result.add("\n") - - result.add(" }\n") - - result.add("}\n") - -# ============================================================================ -# Manifest Hash Calculation -# ============================================================================ - -proc calculateManifestHash*(manifest: PackageManifest): string = - ## Calculate deterministic xxh3-128 hash of manifest - ## - ## **Purpose:** Provides a unique identifier for a specific manifest configuration - ## **Algorithm:** xxh3-128 (fast, 128-bit collision resistance) - ## **Determinism:** Same manifest always produces same hash - ## - ## **Hash Components (in order):** - ## 1. Format type - ## 2. Package identity (name, version) - ## 3. Dependencies (sorted by name for determinism) - ## 4. Build configuration (sorted flags) - ## 5. Platform constraints (sorted) - ## 6. Integrity hashes (build, source, artifact) - ## 7. Metadata (sorted) - ## - ## **Requirements:** 6.5, 7.5 - ## **Property:** Manifest Hash Determinism (Property 9) - - var components: seq[string] = @[] - - # 1. Format type (ensures different formats have different hashes) - components.add($manifest.format) - - # 2. Package identity - components.add(manifest.name) - components.add($manifest.version) - components.add(manifest.license) - - # 3. Optional identity fields (sorted for determinism) - if manifest.description.isSome: - components.add("description:" & manifest.description.get()) - if manifest.homepage.isSome: - components.add("homepage:" & manifest.homepage.get()) - if manifest.author.isSome: - components.add("author:" & manifest.author.get()) - - # 4. Dependencies (sorted by name for determinism) - var depStrings: seq[string] = @[] - for dep in manifest.dependencies: - var depStr = "dep:" & dep.name - if dep.versionConstraint.operator != OpAny: - depStr.add(":" & $dep.versionConstraint.operator & - $dep.versionConstraint.version) - if dep.optional: - depStr.add(":optional") - if dep.features.len > 0: - depStr.add(":features=" & dep.features.sorted().join(",")) - depStrings.add(depStr) - components.add(depStrings.sorted().join("|")) - - # 5. Build dependencies (sorted) - var buildDepStrings: seq[string] = @[] - for dep in manifest.buildDependencies: - var depStr = "builddep:" & dep.name - if dep.versionConstraint.operator != OpAny: - depStr.add(":" & $dep.versionConstraint.operator & - $dep.versionConstraint.version) - buildDepStrings.add(depStr) - components.add(buildDepStrings.sorted().join("|")) - - # 6. Optional dependencies (sorted) - var optDepStrings: seq[string] = @[] - for dep in manifest.optionalDependencies: - var depStr = "optdep:" & dep.name - if dep.versionConstraint.operator != OpAny: - depStr.add(":" & $dep.versionConstraint.operator & - $dep.versionConstraint.version) - if dep.features.len > 0: - depStr.add(":features=" & dep.features.sorted().join(",")) - optDepStrings.add(depStr) - components.add(optDepStrings.sorted().join("|")) - - # 7. Build configuration (sorted flags for determinism) - if manifest.buildSystem.isSome: - components.add("buildsystem:" & manifest.buildSystem.get()) - components.add("buildflags:" & manifest.buildFlags.sorted().join(" ")) - components.add("configureflags:" & manifest.configureFlags.sorted().join(" ")) - - # 8. Runtime configuration - if manifest.libc.isSome: - components.add("libc:" & manifest.libc.get()) - if manifest.allocator.isSome: - components.add("allocator:" & manifest.allocator.get()) - - # 9. Platform constraints (sorted for determinism) - components.add("os:" & manifest.supportedOS.sorted().join(",")) - components.add("arch:" & manifest.supportedArchitectures.sorted().join(",")) - components.add("caps:" & manifest.requiredCapabilities.sorted().join(",")) - - # 10. Integrity hashes (these are already deterministic) - components.add("buildhash:" & manifest.buildHash) - components.add("sourcehash:" & manifest.sourceHash) - components.add("artifacthash:" & manifest.artifactHash) - - # 11. Metadata (sorted for determinism) - if manifest.timestamp.isSome: - components.add("timestamp:" & manifest.timestamp.get()) - components.add("tags:" & manifest.tags.sorted().join(",")) - components.add("maintainers:" & manifest.maintainers.sorted().join(",")) - - # 12. UTCP support - if manifest.utcpEndpoint.isSome: - components.add("utcp:" & manifest.utcpEndpoint.get()) - if manifest.utcpVersion.isSome: - components.add("utcpver:" & manifest.utcpVersion.get()) - - # 13. System Integration (sorted for determinism) - var fileStrings: seq[string] = @[] - for file in manifest.files: - fileStrings.add("file:" & file.path & ":" & file.hash & ":" & $file.size & - ":" & file.permissions) - components.add(fileStrings.sorted().join("|")) - - var userStrings: seq[string] = @[] - for user in manifest.users: - var s = "user:" & user.name & ":" & user.group & ":" & user.shell & ":" & user.home - if user.uid.isSome: s.add(":" & $user.uid.get()) - userStrings.add(s) - components.add(userStrings.sorted().join("|")) - - var groupStrings: seq[string] = @[] - for group in manifest.groups: - var s = "group:" & group.name - if group.gid.isSome: s.add(":" & $group.gid.get()) - groupStrings.add(s) - components.add(groupStrings.sorted().join("|")) - - var serviceStrings: seq[string] = @[] - for service in manifest.services: - serviceStrings.add("service:" & service.name & ":" & $service.enabled & - ":" & service.content) - components.add(serviceStrings.sorted().join("|")) - - # 14. Security / Sandbox - if manifest.sandbox.isSome: - let sb = manifest.sandbox.get() - var sbStr = "sandbox:" & $sb.level - - if sb.seccompProfile.isSome: - sbStr.add(":seccomp=" & sb.seccompProfile.get()) - if sb.capabilities.len > 0: - sbStr.add(":caps=" & sb.capabilities.sorted().join(",")) - if sb.namespaces.len > 0: - sbStr.add(":ns=" & sb.namespaces.sorted().join(",")) - - if sb.pledge.isSome: - sbStr.add(":pledge=" & sb.pledge.get()) - if sb.unveil.len > 0: - sbStr.add(":unveil=" & sb.unveil.sorted().join(",")) - - components.add(sbStr) - - # 15. Desktop Integration - if manifest.desktop.isSome: - let dt = manifest.desktop.get() - var dtStr = "desktop:" & dt.displayName & ":" & $dt.terminal & ":" & - $dt.startupNotify - - if dt.icon.isSome: - dtStr.add(":icon=" & dt.icon.get()) - if dt.startupWMClass.isSome: - dtStr.add(":wmclass=" & dt.startupWMClass.get()) - if dt.categories.len > 0: - dtStr.add(":cats=" & dt.categories.sorted().join(",")) - if dt.keywords.len > 0: - dtStr.add(":kws=" & dt.keywords.sorted().join(",")) - if dt.mimeTypes.len > 0: - dtStr.add(":mimes=" & dt.mimeTypes.sorted().join(",")) - - components.add(dtStr) - - # Calculate hash from all components - let input = components.join("|") - let hash = calculateXXH3(input) - - return $hash - -proc verifyManifestHash*(manifest: PackageManifest, - expectedHash: string): bool = - ## Verify that a manifest matches the expected hash - ## Returns true if hash matches, false otherwise - let calculatedHash = calculateManifestHash(manifest) - return calculatedHash == expectedHash - -# ============================================================================ -# Convenience -# ============================================================================ - -proc `$`*(manifest: PackageManifest): string = - ## Convert manifest to human-readable string - result = "Package: " & manifest.name & " v" & $manifest.version & "\n" - result.add("License: " & manifest.license & "\n") - result.add("Format: " & $manifest.format & "\n") - - if manifest.description.isSome: - result.add("Description: " & manifest.description.get() & "\n") - - if manifest.dependencies.len > 0: - result.add("Dependencies: " & $manifest.dependencies.len & "\n") - - if manifest.supportedOS.len > 0: - result.add("OS: " & manifest.supportedOS.join(", ") & "\n") - - if manifest.supportedArchitectures.len > 0: - result.add("Architectures: " & manifest.supportedArchitectures.join(", ") & "\n") - -when isMainModule: - echo "Manifest Parser - Systems Engineering Approach" - echo "Format-agnostic, strict validation, platform-aware" - echo "" - - # Example JSON manifest - let jsonExample = """ -{ - "name": "example-package", - "version": "1.2.3", - "license": "MIT", - "description": "An example package", - "supported_os": ["linux", "freebsd"], - "supported_architectures": ["x86_64", "aarch64"], - "dependencies": [ - {"name": "dep1", "version": ">=1.0.0"}, - {"name": "dep2", "version": "~2.0.0", "optional": true} - ], - "build_hash": "blake3-abc123", - "utcp_endpoint": "https://packages.nexusos.org/utcp/v1/manifest/example-package" -} -""" - - echo "Parsing JSON manifest..." - let manifest = parseManifest(jsonExample, NPK, FormatJSON, ValidationStrict) - echo manifest - echo "" - - echo "Serializing to KDL..." - echo serializeManifestToKDL(manifest) diff --git a/src/nip/metadata.nim b/src/nip/metadata.nim deleted file mode 100644 index a11bd7f..0000000 --- a/src/nip/metadata.nim +++ /dev/null @@ -1,266 +0,0 @@ -## Package Metadata Generation Module -## -## This module implements metadata.json generation for all package formats (.npk, .nip, .nexter). -## It provides complete provenance tracking from source to installation, including: -## - Source origin and maintainer information -## - Build configuration and compiler details -## - Complete audit trail with timestamps -## - Dependency tracking with build hashes -## -## Requirements: 7.1, 7.2, 7.3, 7.4, 7.5 - -import std/[times, json, options, strutils] - -type - FormatType* = enum - ## Package format type - NPK = "NPK" ## System package - NIP = "NIP" ## User application - NEXTER = "NEXTER" ## Container - - SourceInfo* = object - ## Source origin information (Requirement 7.1) - origin*: string ## Source repository or download URL - maintainer*: string ## Package maintainer - upstreamUrl*: string ## Upstream project URL - sourceHash*: string ## xxh3 hash of source code - - BuildInfo* = object - ## Build configuration information (Requirement 7.2) - compilerVersion*: string ## Compiler version used - compilerFlags*: seq[string] ## Compiler flags used - targetArchitecture*: string ## Target CPU architecture - buildHash*: string ## xxh3 build hash - buildTimestamp*: DateTime ## When the build occurred - - ProvenanceStep* = object - ## Single step in provenance chain (Requirement 7.3) - timestamp*: DateTime - action*: string ## Action performed (e.g., "source_download", "build", "installation") - hash*: string ## xxh3 hash of result - verifiedBy*: string ## Tool/version that verified this step - - ProvenanceChain* = object - ## Complete provenance chain from source to installation (Requirement 7.3) - sourceDownload*: ProvenanceStep - build*: ProvenanceStep - installation*: ProvenanceStep - - DependencyInfo* = object - ## Dependency information with build hash - name*: string - version*: string - buildHash*: string ## xxh3 build hash of dependency - - PackageMetadata* = object - ## Complete package metadata (Requirements 7.1-7.5) - packageName*: string - version*: string - formatType*: FormatType - source*: SourceInfo - buildInfo*: BuildInfo - provenance*: Option[ProvenanceChain] - dependencies*: seq[DependencyInfo] - createdAt*: DateTime - -proc generateMetadata*( - packageName: string, - version: string, - formatType: FormatType, - source: SourceInfo, - buildInfo: BuildInfo, - provenance: Option[ProvenanceChain] = none(ProvenanceChain), - dependencies: seq[DependencyInfo] = @[] -): PackageMetadata = - ## Generate complete package metadata - ## - ## This function creates a PackageMetadata object with all required information - ## for provenance tracking and audit trails. - ## - ## Requirements: - ## - 7.1: Includes source origin, maintainer, upstream URL, build timestamp - ## - 7.2: Includes compiler version, flags, target architecture, build hash - ## - 7.3: Records complete chain from source to installation (if provided) - ## - 7.4: Provides full audit trail - ## - 7.5: Uses xxh3 for build hashes - - result = PackageMetadata( - packageName: packageName, - version: version, - formatType: formatType, - source: source, - buildInfo: buildInfo, - provenance: provenance, - dependencies: dependencies, - createdAt: now() - ) - -proc toJson*(metadata: PackageMetadata): string = - ## Serialize metadata to JSON format (Requirement 7.4) - ## - ## This enables querying and audit trail access. - - var jsonObj = %* { - "packageName": metadata.packageName, - "version": metadata.version, - "formatType": $metadata.formatType, - "source": { - "origin": metadata.source.origin, - "maintainer": metadata.source.maintainer, - "upstreamUrl": metadata.source.upstreamUrl, - "sourceHash": metadata.source.sourceHash - }, - "buildInfo": { - "compilerVersion": metadata.buildInfo.compilerVersion, - "compilerFlags": metadata.buildInfo.compilerFlags, - "targetArchitecture": metadata.buildInfo.targetArchitecture, - "buildHash": metadata.buildInfo.buildHash, - "buildTimestamp": metadata.buildInfo.buildTimestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'") - }, - "dependencies": newJArray(), - "createdAt": metadata.createdAt.format("yyyy-MM-dd'T'HH:mm:ss'Z'") - } - - # Add dependencies - for dep in metadata.dependencies: - jsonObj["dependencies"].add(%* { - "name": dep.name, - "version": dep.version, - "buildHash": dep.buildHash - }) - - # Add provenance chain if present - if metadata.provenance.isSome: - let prov = metadata.provenance.get() - jsonObj["provenance"] = %* { - "sourceDownload": { - "timestamp": prov.sourceDownload.timestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'"), - "action": prov.sourceDownload.action, - "hash": prov.sourceDownload.hash, - "verifiedBy": prov.sourceDownload.verifiedBy - }, - "build": { - "timestamp": prov.build.timestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'"), - "action": prov.build.action, - "hash": prov.build.hash, - "verifiedBy": prov.build.verifiedBy - }, - "installation": { - "timestamp": prov.installation.timestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'"), - "action": prov.installation.action, - "hash": prov.installation.hash, - "verifiedBy": prov.installation.verifiedBy - } - } - - result = $jsonObj - -proc fromJson*(jsonStr: string): PackageMetadata = - ## Deserialize metadata from JSON format (Requirement 7.4) - - let jsonObj = parseJson(jsonStr) - - # Parse format type - let formatType = case jsonObj["formatType"].getStr() - of "NPK": FormatType.NPK - of "NIP": FormatType.NIP - of "NEXTER": FormatType.NEXTER - else: FormatType.NPK - - # Parse source info - let source = SourceInfo( - origin: jsonObj["source"]["origin"].getStr(), - maintainer: jsonObj["source"]["maintainer"].getStr(), - upstreamUrl: jsonObj["source"]["upstreamUrl"].getStr(), - sourceHash: jsonObj["source"]["sourceHash"].getStr() - ) - - # Parse build info - var compilerFlags: seq[string] = @[] - for flag in jsonObj["buildInfo"]["compilerFlags"]: - compilerFlags.add(flag.getStr()) - - let buildInfo = BuildInfo( - compilerVersion: jsonObj["buildInfo"]["compilerVersion"].getStr(), - compilerFlags: compilerFlags, - targetArchitecture: jsonObj["buildInfo"]["targetArchitecture"].getStr(), - buildHash: jsonObj["buildInfo"]["buildHash"].getStr(), - buildTimestamp: parse(jsonObj["buildInfo"]["buildTimestamp"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'") - ) - - # Parse dependencies - var dependencies: seq[DependencyInfo] = @[] - if jsonObj.hasKey("dependencies"): - for dep in jsonObj["dependencies"]: - dependencies.add(DependencyInfo( - name: dep["name"].getStr(), - version: dep["version"].getStr(), - buildHash: dep["buildHash"].getStr() - )) - - # Parse provenance if present - var provenance = none(ProvenanceChain) - if jsonObj.hasKey("provenance"): - let prov = jsonObj["provenance"] - provenance = some(ProvenanceChain( - sourceDownload: ProvenanceStep( - timestamp: parse(prov["sourceDownload"]["timestamp"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'"), - action: prov["sourceDownload"]["action"].getStr(), - hash: prov["sourceDownload"]["hash"].getStr(), - verifiedBy: prov["sourceDownload"]["verifiedBy"].getStr() - ), - build: ProvenanceStep( - timestamp: parse(prov["build"]["timestamp"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'"), - action: prov["build"]["action"].getStr(), - hash: prov["build"]["hash"].getStr(), - verifiedBy: prov["build"]["verifiedBy"].getStr() - ), - installation: ProvenanceStep( - timestamp: parse(prov["installation"]["timestamp"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'"), - action: prov["installation"]["action"].getStr(), - hash: prov["installation"]["hash"].getStr(), - verifiedBy: prov["installation"]["verifiedBy"].getStr() - ) - )) - - result = PackageMetadata( - packageName: jsonObj["packageName"].getStr(), - version: jsonObj["version"].getStr(), - formatType: formatType, - source: source, - buildInfo: buildInfo, - provenance: provenance, - dependencies: dependencies, - createdAt: parse(jsonObj["createdAt"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'") - ) - -proc validateMetadata*(metadata: PackageMetadata): bool = - ## Validate metadata completeness and correctness - ## - ## Ensures all required fields are present and hashes use xxh3 format. - - # Check required fields - if metadata.packageName.len == 0: return false - if metadata.version.len == 0: return false - - # Validate source info - if metadata.source.origin.len == 0: return false - if metadata.source.maintainer.len == 0: return false - if metadata.source.upstreamUrl.len == 0: return false - - # Validate hashes use xxh3 format (Requirement 7.5) - if not metadata.source.sourceHash.startsWith("xxh3-"): return false - if not metadata.buildInfo.buildHash.startsWith("xxh3-"): return false - - # Validate dependency hashes - for dep in metadata.dependencies: - if not dep.buildHash.startsWith("xxh3-"): return false - - # Validate provenance hashes if present - if metadata.provenance.isSome: - let prov = metadata.provenance.get() - if not prov.sourceDownload.hash.startsWith("xxh3-"): return false - if not prov.build.hash.startsWith("xxh3-"): return false - if not prov.installation.hash.startsWith("xxh3-"): return false - - return true diff --git a/src/nip/namespace.nim b/src/nip/namespace.nim deleted file mode 100644 index 4f4abea..0000000 --- a/src/nip/namespace.nim +++ /dev/null @@ -1,148 +0,0 @@ -## NIP Namespace Isolation -## -## This module implements the sandboxing and namespace isolation for NIP applications. -## It uses Linux namespaces (User, Mount, PID, Net, IPC) to restrict the application. - -import std/[os, posix, strutils, strformat, logging, options] -import nip/manifest_parser - -# Linux specific constants (if not in std/posix) -const - CLONE_NEWNS* = 0x00020000 - CLONE_NEWUTS* = 0x04000000 - CLONE_NEWIPC* = 0x08000000 - CLONE_NEWUSER* = 0x10000000 - CLONE_NEWPID* = 0x20000000 - CLONE_NEWNET* = 0x40000000 - MS_BIND* = 4096 - MS_REC* = 16384 - MS_PRIVATE* = 262144 - MS_RDONLY* = 1 - -type - SandboxError* = object of CatchableError - - Launcher* = ref object - manifest*: PackageManifest - installDir*: string - casRoot*: string - -proc unshare(flags: cint): cint {.importc: "unshare", header: "".} -proc mount(source, target, filesystemtype: cstring, mountflags: culong, data: cstring): cint {.importc: "mount", header: "".} - -proc newLauncher*(manifest: PackageManifest, installDir, casRoot: string): Launcher = - Launcher(manifest: manifest, installDir: installDir, casRoot: casRoot) - -proc setupUserNamespace(l: Launcher) = - ## Map current user to root inside the namespace - let uid = getuid() - let gid = getgid() - - let uidMap = fmt"0 {uid} 1" - let gidMap = fmt"0 {gid} 1" - - writeFile("/proc/self/uid_map", uidMap) - writeFile("/proc/self/setgroups", "deny") - writeFile("/proc/self/gid_map", gidMap) - -proc setupMountNamespace(l: Launcher) = - ## Setup mount namespace and bind mounts - - # 1. Make all mounts private to avoid propagating changes - if mount("none", "/", "", (MS_REC or MS_PRIVATE).culong, "") != 0: - raise newException(SandboxError, "Failed to make mounts private") - - # 2. Bind mount the application directory - # We might want to mount it to a standard location like /app - # For now, let's just ensure it's accessible. - - # 3. Bind mount CAS (Read-Only) - # This is critical for security and integrity - if mount(l.casRoot.cstring, l.casRoot.cstring, "none", (MS_BIND or MS_REC).culong, "") != 0: - raise newException(SandboxError, "Failed to bind mount CAS") - - if mount("none", l.casRoot.cstring, "none", (MS_BIND or MS_REC or MS_RDONLY).culong, "") != 0: - raise newException(SandboxError, "Failed to remount CAS read-only") - - # 4. Handle /proc (needed for PID namespace) - if mount("proc", "/proc", "proc", 0, "") != 0: - # This might fail if we are not root or fully unshared yet. - # In a user namespace, we can mount proc if we are root inside it. - discard - -proc run*(l: Launcher, args: seq[string]) = - ## Run the application in the sandbox - info(fmt"Launching {l.manifest.name} in sandbox...") - - var flags: cint = 0 - - # Determine flags based on SandboxConfig - if l.manifest.sandbox.isSome: - let sb = l.manifest.sandbox.get() - - # Always use User Namespace for rootless execution - flags = flags or CLONE_NEWUSER - - # Always use Mount Namespace for filesystem isolation - flags = flags or CLONE_NEWNS - - # PID Namespace - if "pid" in sb.namespaces: - flags = flags or CLONE_NEWPID - - # Network Namespace - if "net" in sb.namespaces: - flags = flags or CLONE_NEWNET - - # IPC Namespace - if "ipc" in sb.namespaces: - flags = flags or CLONE_NEWIPC - - else: - # Default strict sandbox - flags = CLONE_NEWUSER or CLONE_NEWNS or CLONE_NEWPID or CLONE_NEWIPC - - # 1. Unshare namespaces - if unshare(flags) != 0: - raise newException(SandboxError, "Failed to unshare namespaces: " & $strerror(errno)) - - # 2. Setup User Mapping (Must be done before other operations that require root) - if (flags and CLONE_NEWUSER) != 0: - l.setupUserNamespace() - - # 3. Fork for PID namespace (PID 1 inside namespace) - if (flags and CLONE_NEWPID) != 0: - let pid = fork() - if pid < 0: - raise newException(SandboxError, "Fork failed") - - if pid > 0: - # Parent: wait for child - var status: cint - discard waitpid(pid, status, 0) - return # Exit parent - - # Child continues here (as PID 1 in new namespace) - # We need to mount /proc here - if mount("proc", "/proc", "proc", 0, "") != 0: - warn("Failed to mount /proc in new PID namespace") - - # 4. Setup Mounts - if (flags and CLONE_NEWNS) != 0: - l.setupMountNamespace() - - # 5. Drop Capabilities (TODO) - # if l.manifest.sandbox.isSome: ... - - # 6. Execute Application - # Find the executable. For now, assume it's in bin/ - let binPath = l.installDir / "bin" / l.manifest.name - - # Construct args - var cargs: seq[cstring] = @[binPath.cstring] - for arg in args: - cargs.add(arg.cstring) - cargs.add(nil) - - if execv(binPath.cstring, cast[cstringArray](addr cargs[0])) != 0: - raise newException(SandboxError, "Failed to exec: " & $strerror(errno)) diff --git a/src/nip/nexter.nim b/src/nip/nexter.nim deleted file mode 100644 index 4836661..0000000 --- a/src/nip/nexter.nim +++ /dev/null @@ -1,347 +0,0 @@ -## NEXTER Archive Handler -## -## **Purpose:** -## Handles .nexter (Nexus Container) archive creation and parsing. -## NEXTER containers are tar.zst archives containing manifest.kdl, environment config, -## CAS chunks, and Ed25519 signatures. -## -## **Design Principles:** -## - Lightweight container isolation -## - Content-addressable storage for deduplication -## - Atomic operations with rollback capability -## - Ed25519 signature verification -## -## **Requirements:** -## - Requirement 5.1: .nexter contains manifest.kdl, environment config, CAS chunks, Ed25519 signature -## - Requirement 8.2: Use zstd --auto for archive compression -## -## **Archive Structure:** -## ``` -## container.nexter (tar.zst) -## ├── manifest.kdl # Container metadata -## ├── environment.kdl # Environment variables -## ├── chunks/ # CAS chunks -## │ ├── xxh3-abc123.zst -## │ ├── xxh3-def456.zst -## │ └── ... -## └── signature.sig # Ed25519 signature -## ``` - -import std/[os, strutils, times, options, sequtils, osproc, logging] -import nip/cas -import nip/xxh -import nip/nexter_manifest - -type - NEXTERContainer* = object - ## Complete NEXTER container with all components - manifest*: NEXTERManifest - environment*: string - chunks*: seq[ChunkData] - signature*: string - archivePath*: string - - ChunkData* = object - ## Chunk data extracted from archive - hash*: string - data*: string - size*: int64 - chunkType*: ChunkType - - NEXTERArchiveError* = object of CatchableError - code*: NEXTERArchiveErrorCode - context*: string - suggestions*: seq[string] - - NEXTERArchiveErrorCode* = enum - ArchiveNotFound, - InvalidArchive, - ManifestMissing, - EnvironmentMissing, - SignatureMissing, - ChunkMissing, - ExtractionFailed, - CompressionFailed, - InvalidFormat - -# ============================================================================ -# Archive Parsing -# ============================================================================ - -proc parseNEXTER*(path: string): NEXTERContainer = - ## Parse .nexter archive and extract all components - ## - ## **Requirements:** - ## - Requirement 5.1: Extract manifest.kdl, environment config, CAS chunks, signature - ## - Requirement 8.2: Handle zstd --auto compressed archives - ## - ## **Process:** - ## 1. Verify archive exists and is readable - ## 2. Extract to temporary directory - ## 3. Parse manifest.kdl - ## 4. Load environment.kdl - ## 5. Load chunks from chunks/ directory - ## 6. Load signature from signature.sig - ## 7. Verify integrity - ## - ## **Raises:** - ## - NEXTERArchiveError if archive is invalid or missing components - - if not fileExists(path): - raise newException(NEXTERArchiveError, "NEXTER archive not found: " & path) - - # Create temporary extraction directory - let tempDir = getTempDir() / "nexter-extract-" & $getTime().toUnix() - createDir(tempDir) - - try: - # Extract archive using tar with zstd decompression - # Using --auto-compress lets tar detect compression automatically - let extractCmd = "tar --auto-compress -xf " & quoteShell(path) & " -C " & - quoteShell(tempDir) - let exitCode = execCmd(extractCmd) - - if exitCode != 0: - raise newException(NEXTERArchiveError, "Failed to extract NEXTER archive") - - # Verify required files exist - let manifestPath = tempDir / "manifest.kdl" - let environmentPath = tempDir / "environment.kdl" - let signaturePath = tempDir / "signature.sig" - let chunksDir = tempDir / "chunks" - - if not fileExists(manifestPath): - raise newException(NEXTERArchiveError, "Invalid archive: manifest.kdl missing") - - if not fileExists(environmentPath): - raise newException(NEXTERArchiveError, "Invalid archive: environment.kdl missing") - - if not fileExists(signaturePath): - raise newException(NEXTERArchiveError, "Invalid archive: signature.sig missing") - - # Parse manifest - let manifestContent = readFile(manifestPath) - let manifest = parseNEXTERManifest(manifestContent) - - # Load environment - let environment = readFile(environmentPath) - - # Load signature - let signature = readFile(signaturePath) - - # Load chunks - var chunks: seq[ChunkData] = @[] - if dirExists(chunksDir): - for file in walkFiles(chunksDir / "*.zst"): - let fileName = file.extractFilename() - let hash = fileName.replace(".zst", "") - let data = readFile(file) - chunks.add(ChunkData( - hash: hash, - data: data, - size: data.len.int64, - chunkType: Binary - )) - - return NEXTERContainer( - manifest: manifest, - environment: environment, - chunks: chunks, - signature: signature, - archivePath: path - ) - - finally: - # Clean up temporary directory - if dirExists(tempDir): - removeDir(tempDir) - -# ============================================================================ -# Archive Creation -# ============================================================================ - -proc createNEXTER*(manifest: NEXTERManifest, environment: string, chunks: seq[ChunkData], - signature: string, outputPath: string) = - ## Create .nexter archive from components - ## - ## **Requirements:** - ## - Requirement 5.1: Create archive with manifest.kdl, environment config, CAS chunks, signature - ## - Requirement 8.2: Use zstd --auto for archive compression - ## - ## **Process:** - ## 1. Validate output path is writable - ## 2. Create temporary directory - ## 3. Write manifest.kdl - ## 4. Write environment.kdl - ## 5. Write chunks to chunks/ directory - ## 6. Write signature.sig - ## 7. Create tar.zst archive - ## 8. Verify archive integrity - ## - ## **Raises:** - ## - OSError if output directory doesn't exist or isn't writable - ## - NEXTERArchiveError if creation fails - - # Validate output path - let outputDir = outputPath.parentDir() - if not dirExists(outputDir): - raise newException(OSError, "Output directory does not exist: " & outputDir) - - let tempDir = getTempDir() / "nexter-create-" & $getTime().toUnix() - createDir(tempDir) - - try: - # Write manifest - let manifestContent = generateNEXTERManifest(manifest) - writeFile(tempDir / "manifest.kdl", manifestContent) - - # Write environment - writeFile(tempDir / "environment.kdl", environment) - - # Write chunks - let chunksDir = tempDir / "chunks" - createDir(chunksDir) - for chunk in chunks: - let chunkPath = chunksDir / (chunk.hash & ".zst") - writeFile(chunkPath, chunk.data) - - # Write signature - writeFile(tempDir / "signature.sig", signature) - - # Create tar.zst archive - let createCmd = "tar --auto-compress -cf " & quoteShell(outputPath) & - " -C " & quoteShell(tempDir) & " ." - let exitCode = execCmd(createCmd) - - if exitCode != 0: - raise newException(NEXTERArchiveError, "Failed to create NEXTER archive") - - info("Created NEXTER archive: " & outputPath) - - finally: - # Clean up temporary directory - if dirExists(tempDir): - removeDir(tempDir) - -# ============================================================================ -# Chunk Extraction to CAS -# ============================================================================ - -proc extractChunksToCAS*(container: NEXTERContainer, casRoot: string): seq[string] = - ## Extract chunks from NEXTER container to CAS - ## - ## **Requirements:** - ## - Requirement 2.1: Store chunks in CAS with xxh3 hashing - ## - Requirement 2.2: Verify integrity using xxh3 hash - ## - ## **Process:** - ## 1. For each chunk in container - ## 2. Decompress chunk - ## 3. Verify xxh3 hash - ## 4. Store in CAS - ## 5. Return list of stored hashes - ## - ## **Returns:** - ## - List of stored chunk hashes - - result = @[] - - for chunk in container.chunks: - try: - # Decompress chunk - let decompressed = chunk.data # TODO: Implement zstd decompression - - # Verify hash - let calculatedHash = "xxh3-" & $calculateXXH3(decompressed) - if calculatedHash != chunk.hash: - warn("Hash mismatch for chunk: " & chunk.hash) - continue - - # Store in CAS - let entry = storeObject(decompressed, casRoot) - result.add(string(entry.hash)) - - except Exception as e: - warn("Failed to extract chunk " & chunk.hash & ": " & e.msg) - -# ============================================================================ -# Archive Verification -# ============================================================================ - -proc verifyNEXTER*(path: string): bool = - ## Verify NEXTER archive integrity - ## - ## **Requirements:** - ## - Requirement 9.2: Verify Ed25519 signature - ## - Requirement 14.1: Verify xxh3 hashes - ## - ## **Checks:** - ## 1. Archive exists and is readable - ## 2. Archive is valid tar.zst - ## 3. All required components present - ## 4. Manifest is valid - ## 5. Signature is present - ## - ## **Returns:** - ## - true if archive is valid, false otherwise - - try: - let container = parseNEXTER(path) - - # Verify manifest - if container.manifest.name.len == 0: - return false - - # Verify signature - if container.signature.len == 0: - return false - - # Verify chunks - if container.chunks.len == 0: - warn("NEXTER archive has no chunks") - - return true - - except Exception as e: - warn("NEXTER verification failed: " & e.msg) - return false - -# ============================================================================ -# Utility Functions -# ============================================================================ - -proc listChunksInArchive*(path: string): seq[string] = - ## List all chunks in a NEXTER archive - ## - ## **Returns:** - ## - List of chunk hashes - - try: - let container = parseNEXTER(path) - return container.chunks.mapIt(it.hash) - except Exception as e: - warn("Failed to list chunks: " & e.msg) - return @[] - -proc getArchiveSize*(path: string): int64 = - ## Get size of NEXTER archive - ## - ## **Returns:** - ## - Size in bytes - - if fileExists(path): - return getFileSize(path) - return 0 - -proc getContainerInfo*(path: string): Option[NEXTERManifest] = - ## Get container information from archive - ## - ## **Returns:** - ## - Container manifest if valid, none otherwise - - try: - let container = parseNEXTER(path) - return some(container.manifest) - except Exception as e: - warn("Failed to get container info: " & e.msg) - return none(NEXTERManifest) diff --git a/src/nip/nexter_installer.nim b/src/nip/nexter_installer.nim deleted file mode 100644 index 6198b40..0000000 --- a/src/nip/nexter_installer.nim +++ /dev/null @@ -1,362 +0,0 @@ -## NEXTER Installation Workflow -## -## **Purpose:** -## Implements atomic installation workflow for .nexter container packages. -## Handles chunk extraction to CAS, manifest creation, reference tracking, -## and rollback on failure. -## -## **Design Principles:** -## - Atomic operations (all-or-nothing) -## - Automatic rollback on failure -## - CAS deduplication -## - Reference tracking for garbage collection -## - Container isolation and lifecycle management -## -## **Requirements:** -## - Requirement 5.3: Extract chunks to CAS and create manifest in ~/.local/share/nexus/nexters/ -## - Requirement 11.1: Container installation SHALL be atomic (all-or-nothing) -## - Requirement 11.2: Installation failures SHALL rollback to previous state - -import std/[os, strutils, times, options] -import nip/[nexter, nexter_manifest, manifest_parser] - -type - ContainerInstallResult* = object - ## Result of NEXTER container installation - success*: bool - containerName*: string - version*: string - installPath*: string - chunksInstalled*: int - error*: string - - ContainerInstallError* = object of CatchableError - code*: ContainerInstallErrorCode - context*: string - suggestions*: seq[string] - - ContainerInstallErrorCode* = enum - ContainerAlreadyInstalled, - InsufficientSpace, - PermissionDenied, - ChunkExtractionFailed, - ManifestCreationFailed, - RollbackFailed, - InvalidContainer, - EnvironmentConfigInvalid - - ContainerInstallTransaction* = object - ## Transaction tracking for atomic container installation - id*: string - containerName*: string - startTime*: DateTime - operations*: seq[ContainerInstallOperation] - completed*: bool - - ContainerInstallOperation* = object - ## Individual operation in container installation transaction - kind*: OperationKind - path*: string - data*: string - timestamp*: DateTime - - OperationKind* = enum - CreateDirectory, - WriteFile, - CreateSymlink, - AddCASChunk, - AddReference - -# ============================================================================ -# Forward Declarations -# ============================================================================ - -proc rollbackContainerInstallation*(transaction: ContainerInstallTransaction, storageRoot: string) - -# ============================================================================ -# Installation Workflow -# ============================================================================ - -proc installNEXTER*(containerPath: string, storageRoot: string = ""): ContainerInstallResult = - ## Install NEXTER container atomically - ## - ## **Requirements:** - ## - Requirement 5.3: Extract chunks to CAS and create manifest - ## - Requirement 11.1: Atomic installation (all-or-nothing) - ## - Requirement 11.2: Rollback on failure - ## - ## **Process:** - ## 1. Parse NEXTER container archive - ## 2. Validate container integrity - ## 3. Check if already installed - ## 4. Create installation transaction - ## 5. Extract chunks to CAS with deduplication - ## 6. Create manifest in ~/.local/share/nexus/nexters/ - ## 7. Create environment config - ## 8. Add references to cas/refs/nexters/ - ## 9. Commit transaction or rollback on failure - ## - ## **Returns:** - ## - ContainerInstallResult with success status and details - ## - ## **Raises:** - ## - ContainerInstallError if installation fails - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let nextersDir = root / "nexters" - let casRoot = root / "cas" - - try: - # Parse container archive - let container = parseNEXTER(containerPath) - - # Check if already installed - let installPath = nextersDir / container.manifest.name - if dirExists(installPath): - return ContainerInstallResult( - success: false, - containerName: container.manifest.name, - version: $container.manifest.version, - error: "Container already installed at " & installPath, - installPath: installPath - ) - - # Create installation transaction - let transactionId = "nexter-" & $getTime().toUnix() - var transaction = ContainerInstallTransaction( - id: transactionId, - containerName: container.manifest.name, - startTime: now(), - operations: @[], - completed: false - ) - - # Create directories - createDir(nextersDir) - createDir(installPath) - transaction.operations.add(ContainerInstallOperation( - kind: CreateDirectory, - path: installPath, - timestamp: now() - )) - - # Extract chunks to CAS - var chunksInstalled = 0 - for chunk in container.chunks: - let chunkPath = casRoot / "chunks" / (chunk.hash & ".zst") - if not fileExists(chunkPath): - createDir(casRoot / "chunks") - writeFile(chunkPath, chunk.data) - transaction.operations.add(ContainerInstallOperation( - kind: AddCASChunk, - path: chunkPath, - data: chunk.hash, - timestamp: now() - )) - chunksInstalled += 1 - - # Create manifest file - let manifestContent = generateNEXTERManifest(container.manifest) - let manifestPath = installPath / "manifest.kdl" - writeFile(manifestPath, manifestContent) - transaction.operations.add(ContainerInstallOperation( - kind: WriteFile, - path: manifestPath, - timestamp: now() - )) - - # Create environment config - let environmentPath = installPath / "environment.kdl" - writeFile(environmentPath, container.environment) - transaction.operations.add(ContainerInstallOperation( - kind: WriteFile, - path: environmentPath, - timestamp: now() - )) - - # Create signature file - let signaturePath = installPath / "signature.sig" - writeFile(signaturePath, container.signature) - transaction.operations.add(ContainerInstallOperation( - kind: WriteFile, - path: signaturePath, - timestamp: now() - )) - - # Add references to CAS - let refsDir = casRoot / "refs" / "nexters" - createDir(refsDir) - let refsPath = refsDir / (container.manifest.name & ".refs") - var refsList: seq[string] = @[] - for chunk in container.chunks: - refsList.add(chunk.hash) - writeFile(refsPath, refsList.join("\n")) - transaction.operations.add(ContainerInstallOperation( - kind: AddReference, - path: refsPath, - timestamp: now() - )) - - # Mark transaction as completed - transaction.completed = true - - return ContainerInstallResult( - success: true, - containerName: container.manifest.name, - version: $container.manifest.version, - installPath: installPath, - chunksInstalled: chunksInstalled, - error: "" - ) - - except Exception as e: - return ContainerInstallResult( - success: false, - containerName: "", - version: "", - error: "Installation failed: " & e.msg, - installPath: "" - ) - -# ============================================================================ -# Rollback -# ============================================================================ - -proc rollbackContainerInstallation*(transaction: ContainerInstallTransaction, storageRoot: string) = - ## Rollback container installation on failure - ## - ## **Requirements:** - ## - Requirement 11.2: Rollback to previous state on failure - ## - ## **Process:** - ## 1. Process operations in reverse order - ## 2. Remove files and directories - ## 3. Don't remove CAS chunks (might be shared) - ## 4. Continue rollback even if individual operations fail - ## - ## **Note:** - ## - CAS chunks are not removed (garbage collection handles orphaned chunks) - ## - References are removed to mark chunks as orphaned - - # Process operations in reverse order - for i in countdown(transaction.operations.len - 1, 0): - let op = transaction.operations[i] - - try: - case op.kind: - of CreateDirectory: - if dirExists(op.path): - removeDir(op.path) - of WriteFile, CreateSymlink: - if fileExists(op.path): - removeFile(op.path) - of AddCASChunk: - # Don't remove CAS chunks - garbage collection handles them - discard - of AddReference: - if fileExists(op.path): - removeFile(op.path) - except: - # Continue rollback even if individual operations fail - discard - -# ============================================================================ -# Query Functions -# ============================================================================ - -proc isContainerInstalled*(containerName: string, storageRoot: string = ""): bool = - ## Check if container is installed - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let installPath = root / "nexters" / containerName - return dirExists(installPath) - -proc getInstalledContainerVersion*(containerName: string, storageRoot: string = ""): Option[string] = - ## Get installed container version - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let manifestPath = root / "nexters" / containerName / "manifest.kdl" - - if not fileExists(manifestPath): - return none[string]() - - try: - let content = readFile(manifestPath) - # Parse manifest to extract version - let manifest = parseNEXTERManifest(content) - return some($manifest.version) - except: - discard - - return none[string]() - -proc listInstalledContainers*(storageRoot: string = ""): seq[string] = - ## List all installed containers - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let nextersDir = root / "nexters" - - result = @[] - if not dirExists(nextersDir): - return - - for entry in walkDir(nextersDir): - if entry.kind == pcDir: - result.add(entry.path.extractFilename()) - -# ============================================================================ -# Verification -# ============================================================================ - -proc verifyContainerInstallation*(containerName: string, storageRoot: string = ""): bool = - ## Verify container installation integrity - ## - ## **Requirements:** - ## - Requirement 5.3: Verify manifest and environment config exist - ## - ## **Checks:** - ## 1. Container directory exists - ## 2. manifest.kdl exists and is readable - ## 3. environment.kdl exists and is readable - ## 4. signature.sig exists and is readable - ## 5. All referenced chunks exist in CAS - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let installPath = root / "nexters" / containerName - let casRoot = root / "cas" - - # Check directory exists - if not dirExists(installPath): - return false - - # Check required files - if not fileExists(installPath / "manifest.kdl"): - return false - if not fileExists(installPath / "environment.kdl"): - return false - if not fileExists(installPath / "signature.sig"): - return false - - # Check CAS chunks referenced - let refsPath = casRoot / "refs" / "nexters" / (containerName & ".refs") - if fileExists(refsPath): - try: - let refs = readFile(refsPath).split('\n') - for refHash in refs: - if refHash.len > 0: - let chunkPath = casRoot / "chunks" / (refHash & ".zst") - if not fileExists(chunkPath): - return false - except: - return false - - return true - -# ============================================================================ -# Formatting -# ============================================================================ - -proc `$`*(installResult: ContainerInstallResult): string = - ## Format installation result as string - if installResult.success: - return "✅ Installed " & installResult.containerName & " v" & installResult.version & - " to " & installResult.installPath & " (" & $installResult.chunksInstalled & " chunks)" - else: - return "❌ Installation failed: " & installResult.error diff --git a/src/nip/nexter_manifest.nim b/src/nip/nexter_manifest.nim deleted file mode 100644 index feb5450..0000000 --- a/src/nip/nexter_manifest.nim +++ /dev/null @@ -1,606 +0,0 @@ -## NEXTER Manifest Schema - Container Format -## -## **Purpose:** -## Defines the NEXTER (Nexus Container) manifest schema for lightweight containers. -## NEXTER containers provide isolated environments for development and deployment. -## -## **Design Principles:** -## - Lightweight container isolation -## - Base image support with CAS deduplication -## - Environment variable configuration -## - Namespace isolation -## - Ed25519 signature support -## -## **Requirements:** -## - Requirement 5.1: manifest.kdl, environment config, CAS chunks, signature -## - Requirement 5.2: container name, base image, packages, environment variables -## - Requirement 6.2: KDL format with chunk references by xxh3 hash -## - Requirement 6.5: exact versions and build hashes for dependencies - -import std/[times, options, strutils, tables, algorithm] -import nip/manifest_parser - -type - # ============================================================================ - # NEXTER-Specific Types - # ============================================================================ - - NEXTERManifest* = object - ## Complete NEXTER manifest for containers - # Core identity - name*: string - version*: SemanticVersion - buildDate*: DateTime - - # Container metadata - metadata*: ContainerInfo - provenance*: ProvenanceInfo - buildConfig*: BuildConfiguration - - # Base configuration - base*: BaseConfig - - # Environment variables - environment*: Table[string, string] - - # CAS chunk references - casChunks*: seq[ChunkReference] - - # Namespace configuration - namespace*: ContainerNamespace - - # Startup configuration - startup*: StartupConfig - - # Integrity - buildHash*: string ## xxh3-128 hash of build configuration - signature*: SignatureInfo - - ContainerInfo* = object - ## Container metadata - description*: string - homepage*: Option[string] - license*: string - author*: Option[string] - maintainer*: Option[string] - tags*: seq[string] - purpose*: Option[string] ## Container purpose (e.g., "development", "production") - - ProvenanceInfo* = object - ## Complete provenance tracking - source*: string ## Source URL or repository - sourceHash*: string ## xxh3-128 hash of source - upstream*: Option[string] ## Upstream project URL - buildTimestamp*: DateTime - builder*: Option[string] ## Who built this container - - BuildConfiguration* = object - ## Build configuration for reproducibility - configureFlags*: seq[string] - compilerFlags*: seq[string] - compilerVersion*: string - targetArchitecture*: string - libc*: string ## musl, glibc - allocator*: string ## jemalloc, tcmalloc, default - buildSystem*: string ## cmake, meson, autotools, etc. - - BaseConfig* = object - ## Base image configuration - baseImage*: Option[string] ## Base image name (e.g., "alpine", "debian") - baseVersion*: Option[string] ## Base image version - packages*: seq[string] ## Additional packages to include - - ChunkReference* = object - ## Reference to a CAS chunk - hash*: string ## xxh3-128 hash - size*: int64 - chunkType*: ChunkType - path*: string ## Relative path in container - - ChunkType* = enum - ## Type of chunk content - Binary, Library, Runtime, Config, Data, Base, Tools - - ContainerNamespace* = object - ## Container namespace isolation configuration - isolationType*: string ## "full", "network", "pid", "ipc", "uts" - capabilities*: seq[string] ## Linux capabilities - mounts*: seq[MountSpec] - devices*: seq[DeviceSpec] - - MountSpec* = object - ## Filesystem mount specification - source*: string - target*: string - mountType*: string ## "bind", "tmpfs", "devtmpfs" - readOnly*: bool - options*: seq[string] - - DeviceSpec* = object - ## Device access specification - path*: string - deviceType*: string ## "c" (character), "b" (block) - major*: int - minor*: int - permissions*: string ## "rwm" - - StartupConfig* = object - ## Container startup configuration - command*: seq[string] ## Startup command - workingDir*: string ## Working directory - user*: Option[string] ## User to run as - entrypoint*: Option[string] ## Entrypoint script - - SignatureInfo* = object - ## Ed25519 signature information - algorithm*: string ## "ed25519" - keyId*: string - signature*: string ## Base64-encoded signature - - # ============================================================================ - # Error Types - # ============================================================================ - - NEXTERError* = object of CatchableError - code*: NEXTERErrorCode - context*: string - - NEXTERErrorCode* = enum - InvalidManifest, - MissingField, - InvalidHash, - InvalidSignature, - InvalidConfiguration - -# ============================================================================ -# KDL Parsing - Minimal implementation to expose gaps via tests -# ============================================================================ - -proc parseNEXTERManifest*(kdl: string): NEXTERManifest = - ## Parse NEXTER manifest from KDL format - ## - ## **Requirements:** - ## - Requirement 5.2: Parse container name, base image, packages, environment variables - ## - Requirement 6.2: Validate chunk references by xxh3 hash - ## - ## **Implementation Note:** - ## This is a simple line-based parser that extracts key values from KDL format. - ## It handles the specific structure generated by generateNEXTERManifest(). - - var name = "unknown" - var version = SemanticVersion(major: 1, minor: 0, patch: 0) - var buildDate = now() - var description = "Unknown" - var license = "Unknown" - var homepage = none[string]() - var author = none[string]() - var maintainer = none[string]() - var purpose = none[string]() - var tags: seq[string] = @[] - var source = "unknown" - var sourceHash = "xxh3-0000000000000000" - var upstream = none[string]() - var buildTimestamp = now() - var builder = none[string]() - var configureFlags: seq[string] = @[] - var compilerFlags: seq[string] = @[] - var compilerVersion = "unknown" - var targetArchitecture = "x86_64" - var libc = "musl" - var allocator = "jemalloc" - var buildSystem = "unknown" - var baseImage = none[string]() - var baseVersion = none[string]() - var basePackages: seq[string] = @[] - var environment = initTable[string, string]() - var casChunks: seq[ChunkReference] = @[] - var isolationType = "full" - var capabilities: seq[string] = @[] - var buildHash = "xxh3-0000000000000000" - var signatureAlgorithm = "ed25519" - var keyId = "unknown" - var signature = "" - var command: seq[string] = @[] - var workingDir = "/" - var user = none[string]() - var entrypoint = none[string]() - - # Parse line by line - let lines = kdl.split('\n') - var inSection = "" - var inSubsection = "" - - for line in lines: - let trimmed = line.strip() - - # Skip empty lines and comments - if trimmed.len == 0 or trimmed.startsWith("#"): - continue - - # Extract container name and version - if trimmed.startsWith("container"): - let parts = trimmed.split('"') - if parts.len >= 2: - name = parts[1] - inSection = "container" - continue - - # Track sections - if trimmed.endsWith("{"): - if trimmed.startsWith("metadata"): - inSection = "metadata" - elif trimmed.startsWith("provenance"): - inSection = "provenance" - elif trimmed.startsWith("build_config"): - inSection = "build_config" - elif trimmed.startsWith("base"): - inSection = "base" - elif trimmed.startsWith("environment"): - inSection = "environment" - elif trimmed.startsWith("cas_chunks"): - inSection = "cas_chunks" - elif trimmed.startsWith("namespace"): - inSection = "namespace" - elif trimmed.startsWith("startup"): - inSection = "startup" - elif trimmed.startsWith("signature"): - inSection = "signature" - continue - - # End of section - if trimmed == "}": - inSection = "" - inSubsection = "" - continue - - # Parse key-value pairs - if trimmed.contains("\""): - let parts = trimmed.split('"') - if parts.len >= 2: - let key = parts[0].strip() - let value = parts[1] - - case inSection: - of "container": - if key == "version": - version = parseSemanticVersion(value) - elif key == "build_date": - buildDate = parse(value, "yyyy-MM-dd'T'HH:mm:ss'Z'") - of "metadata": - if key == "description": - description = value - elif key == "license": - license = value - elif key == "homepage": - homepage = some(value) - elif key == "author": - author = some(value) - elif key == "maintainer": - maintainer = some(value) - elif key == "purpose": - purpose = some(value) - elif key == "tags": - tags = value.split(" ") - of "provenance": - if key == "source": - source = value - elif key == "source_hash": - sourceHash = value - elif key == "upstream": - upstream = some(value) - elif key == "build_timestamp": - buildTimestamp = parse(value, "yyyy-MM-dd'T'HH:mm:ss'Z'") - elif key == "builder": - builder = some(value) - of "build_config": - if key == "configure_flags": - configureFlags = value.split(" ") - elif key == "compiler_flags": - compilerFlags = value.split(" ") - elif key == "compiler_version": - compilerVersion = value - elif key == "target_architecture": - targetArchitecture = value - elif key == "libc": - libc = value - elif key == "allocator": - allocator = value - elif key == "build_system": - buildSystem = value - of "base": - if key == "image": - baseImage = some(value) - elif key == "version": - baseVersion = some(value) - elif key == "packages": - basePackages = value.split(" ") - of "environment": - environment[key] = value - of "namespace": - if key == "isolation": - isolationType = value - elif key == "capabilities": - capabilities = value.split(" ") - of "startup": - if key == "command": - command = value.split(" ") - elif key == "working_dir": - workingDir = value - elif key == "user": - user = some(value) - elif key == "entrypoint": - entrypoint = some(value) - of "signature": - if key == "algorithm": - signatureAlgorithm = value - elif key == "key_id": - keyId = value - elif key == "signature": - signature = value - else: - if key == "build_hash": - buildHash = value - discard - - result = NEXTERManifest( - name: name, - version: version, - buildDate: buildDate, - metadata: ContainerInfo( - description: description, - license: license, - homepage: homepage, - author: author, - maintainer: maintainer, - purpose: purpose, - tags: tags - ), - provenance: ProvenanceInfo( - source: source, - sourceHash: sourceHash, - upstream: upstream, - buildTimestamp: buildTimestamp, - builder: builder - ), - buildConfig: BuildConfiguration( - configureFlags: configureFlags, - compilerFlags: compilerFlags, - compilerVersion: compilerVersion, - targetArchitecture: targetArchitecture, - libc: libc, - allocator: allocator, - buildSystem: buildSystem - ), - base: BaseConfig( - baseImage: baseImage, - baseVersion: baseVersion, - packages: basePackages - ), - environment: environment, - casChunks: casChunks, - namespace: ContainerNamespace( - isolationType: isolationType, - capabilities: capabilities, - mounts: @[], - devices: @[] - ), - startup: StartupConfig( - command: command, - workingDir: workingDir, - user: user, - entrypoint: entrypoint - ), - buildHash: buildHash, - signature: SignatureInfo( - algorithm: signatureAlgorithm, - keyId: keyId, - signature: signature - ) - ) - -# ============================================================================ -# KDL Generation -# ============================================================================ - -proc generateNEXTERManifest*(manifest: NEXTERManifest): string = - ## Generate KDL manifest from NEXTERManifest - ## - ## **Requirements:** - ## - Requirement 5.2: Generate container name, base image, packages, environment variables - ## - Requirement 6.4: Deterministic generation (same input = same output) - ## - ## **Determinism:** Fields are output in a fixed order to ensure same input = same output - - result = "container \"" & manifest.name & "\" {\n" - - # Core identity - result.add(" version \"" & $manifest.version & "\"\n") - result.add(" build_date \"" & manifest.buildDate.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - result.add("\n") - - # Metadata section - result.add(" metadata {\n") - result.add(" description \"" & manifest.metadata.description & "\"\n") - result.add(" license \"" & manifest.metadata.license & "\"\n") - if manifest.metadata.homepage.isSome: - result.add(" homepage \"" & manifest.metadata.homepage.get() & "\"\n") - if manifest.metadata.author.isSome: - result.add(" author \"" & manifest.metadata.author.get() & "\"\n") - if manifest.metadata.maintainer.isSome: - result.add(" maintainer \"" & manifest.metadata.maintainer.get() & "\"\n") - if manifest.metadata.purpose.isSome: - result.add(" purpose \"" & manifest.metadata.purpose.get() & "\"\n") - if manifest.metadata.tags.len > 0: - result.add(" tags \"" & manifest.metadata.tags.join(" ") & "\"\n") - result.add(" }\n\n") - - # Provenance section - result.add(" provenance {\n") - result.add(" source \"" & manifest.provenance.source & "\"\n") - result.add(" source_hash \"" & manifest.provenance.sourceHash & "\"\n") - if manifest.provenance.upstream.isSome: - result.add(" upstream \"" & manifest.provenance.upstream.get() & "\"\n") - result.add(" build_timestamp \"" & manifest.provenance.buildTimestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - if manifest.provenance.builder.isSome: - result.add(" builder \"" & manifest.provenance.builder.get() & "\"\n") - result.add(" }\n\n") - - # Build configuration section - result.add(" build_config {\n") - if manifest.buildConfig.configureFlags.len > 0: - result.add(" configure_flags \"" & manifest.buildConfig.configureFlags.join(" ") & "\"\n") - if manifest.buildConfig.compilerFlags.len > 0: - result.add(" compiler_flags \"" & manifest.buildConfig.compilerFlags.join(" ") & "\"\n") - result.add(" compiler_version \"" & manifest.buildConfig.compilerVersion & "\"\n") - result.add(" target_architecture \"" & manifest.buildConfig.targetArchitecture & "\"\n") - result.add(" libc \"" & manifest.buildConfig.libc & "\"\n") - result.add(" allocator \"" & manifest.buildConfig.allocator & "\"\n") - result.add(" build_system \"" & manifest.buildConfig.buildSystem & "\"\n") - result.add(" }\n\n") - - # Base configuration section - result.add(" base {\n") - if manifest.base.baseImage.isSome: - result.add(" image \"" & manifest.base.baseImage.get() & "\"\n") - if manifest.base.baseVersion.isSome: - result.add(" version \"" & manifest.base.baseVersion.get() & "\"\n") - if manifest.base.packages.len > 0: - result.add(" packages \"" & manifest.base.packages.join(" ") & "\"\n") - result.add(" }\n\n") - - # Environment variables section - if manifest.environment.len > 0: - result.add(" environment {\n") - # Sort keys for determinism - var sortedKeys = newSeq[string]() - for key in manifest.environment.keys: - sortedKeys.add(key) - sortedKeys.sort() - for key in sortedKeys: - result.add(" " & key & " \"" & manifest.environment[key] & "\"\n") - result.add(" }\n\n") - - # CAS chunks section - if manifest.casChunks.len > 0: - result.add(" cas_chunks {\n") - for chunk in manifest.casChunks: - result.add(" chunk \"" & chunk.hash & "\" {\n") - result.add(" size " & $chunk.size & "\n") - result.add(" type \"" & ($chunk.chunkType).toLowerAscii() & "\"\n") - result.add(" path \"" & chunk.path & "\"\n") - result.add(" }\n") - result.add(" }\n\n") - - # Namespace configuration section - result.add(" namespace {\n") - result.add(" isolation \"" & manifest.namespace.isolationType & "\"\n") - if manifest.namespace.capabilities.len > 0: - result.add(" capabilities \"" & manifest.namespace.capabilities.join(" ") & "\"\n") - - # Mounts - if manifest.namespace.mounts.len > 0: - result.add("\n mounts {\n") - for mount in manifest.namespace.mounts: - result.add(" mount {\n") - result.add(" source \"" & mount.source & "\"\n") - result.add(" target \"" & mount.target & "\"\n") - result.add(" type \"" & mount.mountType & "\"\n") - result.add(" read_only " & $mount.readOnly & "\n") - if mount.options.len > 0: - result.add(" options \"" & mount.options.join(",") & "\"\n") - result.add(" }\n") - result.add(" }\n") - - # Devices - if manifest.namespace.devices.len > 0: - result.add("\n devices {\n") - for device in manifest.namespace.devices: - result.add(" device {\n") - result.add(" path \"" & device.path & "\"\n") - result.add(" type \"" & device.deviceType & "\"\n") - result.add(" major " & $device.major & "\n") - result.add(" minor " & $device.minor & "\n") - result.add(" permissions \"" & device.permissions & "\"\n") - result.add(" }\n") - result.add(" }\n") - - result.add(" }\n\n") - - # Startup configuration section - result.add(" startup {\n") - if manifest.startup.command.len > 0: - result.add(" command \"" & manifest.startup.command.join(" ") & "\"\n") - result.add(" working_dir \"" & manifest.startup.workingDir & "\"\n") - if manifest.startup.user.isSome: - result.add(" user \"" & manifest.startup.user.get() & "\"\n") - if manifest.startup.entrypoint.isSome: - result.add(" entrypoint \"" & manifest.startup.entrypoint.get() & "\"\n") - result.add(" }\n\n") - - # Build hash - result.add(" build_hash \"" & manifest.buildHash & "\"\n\n") - - # Signature - result.add(" signature {\n") - result.add(" algorithm \"" & manifest.signature.algorithm & "\"\n") - result.add(" key_id \"" & manifest.signature.keyId & "\"\n") - result.add(" signature \"" & manifest.signature.signature & "\"\n") - result.add(" }\n") - - result.add("}\n") - -# ============================================================================ -# Validation -# ============================================================================ - -proc validateNEXTERManifest*(manifest: NEXTERManifest): seq[string] = - ## Validate NEXTER manifest and return list of issues - ## - ## **Requirements:** - ## - Requirement 6.3: Validate all required fields and hash formats - - result = @[] - - # Validate name - if manifest.name.len == 0: - result.add("Container name cannot be empty") - - # Validate build hash format (xxh3-128) - if manifest.buildHash.len > 0 and not manifest.buildHash.startsWith("xxh3-"): - result.add("Build hash must use xxh3-128 format (xxh3-...)") - - # Validate source hash format - if manifest.provenance.sourceHash.len > 0 and not manifest.provenance.sourceHash.startsWith("xxh3-"): - result.add("Source hash must use xxh3-128 format (xxh3-...)") - - # Validate CAS chunks have xxh3 hashes - for chunk in manifest.casChunks: - if not chunk.hash.startsWith("xxh3-"): - result.add("Chunk hash must use xxh3-128 format (xxh3-...)") - if chunk.size <= 0: - result.add("Chunk size must be positive") - - # Validate startup configuration - if manifest.startup.workingDir.len == 0: - result.add("Startup working_dir cannot be empty") - - # Validate signature - if manifest.signature.algorithm.len > 0 and manifest.signature.algorithm != "ed25519": - result.add("Signature algorithm must be 'ed25519'") - if manifest.signature.keyId.len == 0: - result.add("Signature key_id cannot be empty") - if manifest.signature.signature.len == 0: - result.add("Signature value cannot be empty") - -# ============================================================================ -# Convenience Functions -# ============================================================================ - -proc `$`*(manifest: NEXTERManifest): string = - ## Convert NEXTER manifest to human-readable string - result = "NEXTER Container: " & manifest.name & " v" & $manifest.version & "\n" - result.add("Build Date: " & manifest.buildDate.format("yyyy-MM-dd HH:mm:ss") & "\n") - result.add("License: " & manifest.metadata.license & "\n") - result.add("Build Hash: " & manifest.buildHash & "\n") - result.add("CAS Chunks: " & $manifest.casChunks.len & "\n") - result.add("Isolation: " & manifest.namespace.isolationType & "\n") diff --git a/src/nip/nexter_removal.nim b/src/nip/nexter_removal.nim deleted file mode 100644 index 0c37131..0000000 --- a/src/nip/nexter_removal.nim +++ /dev/null @@ -1,278 +0,0 @@ -## NEXTER Container Removal -## -## **Purpose:** -## Implements atomic removal of NEXTER containers including stopping running -## instances, removing references, cleaning up state, and marking chunks for -## garbage collection. -## -## **Design Principles:** -## - Atomic removal operations -## - Graceful container shutdown -## - Reference cleanup for garbage collection -## - State preservation for recovery -## -## **Requirements:** -## - Requirement 5.3: Remove NEXTER containers -## - Requirement 12.1: Mark chunks for garbage collection - -import std/[os, strutils, times, options, tables] -import nip/[nexter_installer, container_management, nexter_manifest] - -type - RemovalResult* = object - ## Result of NEXTER removal - success*: bool - containerName*: string - removedPath*: string - chunksMarkedForGC*: int - error*: string - - RemovalError* = object of CatchableError - code*: RemovalErrorCode - context*: string - suggestions*: seq[string] - - RemovalErrorCode* = enum - ContainerNotFound, - ContainerStillRunning, - RemovalFailed, - ReferenceCleanupFailed, - StateCleanupFailed - -# ============================================================================ -# Container Removal -# ============================================================================ - -proc removeNEXTER*(containerName: string, storageRoot: string = "", - manager: Option[ContainerManager] = none[ContainerManager]()): RemovalResult = - ## Remove NEXTER container atomically - ## - ## **Requirements:** - ## - Requirement 5.3: Remove container - ## - Requirement 12.1: Mark chunks for garbage collection - ## - ## **Process:** - ## 1. Stop running container (if any) - ## 2. Remove manifest and configuration - ## 3. Remove references from CAS - ## 4. Mark chunks for garbage collection - ## 5. Clean up container state - ## - ## **Returns:** - ## - RemovalResult with success status and details - ## - ## **Raises:** - ## - RemovalError if removal fails - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let containerPath = root / "nexters" / containerName - let casRoot = root / "cas" - let refsPath = casRoot / "refs" / "nexters" / (containerName & ".refs") - - try: - # Check if container exists - if not dirExists(containerPath): - return RemovalResult( - success: false, - containerName: containerName, - removedPath: containerPath, - chunksMarkedForGC: 0, - error: "Container not found at " & containerPath - ) - - # Stop running container if manager provided - if manager.isSome: - var mgr = manager.get() - if isContainerRunning(mgr): - if not stopContainer(mgr, timeout=10): - return RemovalResult( - success: false, - containerName: containerName, - removedPath: containerPath, - chunksMarkedForGC: 0, - error: "Failed to stop running container" - ) - - # Read references before removal - var chunksMarkedForGC = 0 - if fileExists(refsPath): - try: - let refs = readFile(refsPath).split('\n') - chunksMarkedForGC = refs.len - except: - discard - - # Remove manifest and configuration files - try: - let manifestPath = containerPath / "manifest.kdl" - let environmentPath = containerPath / "environment.kdl" - let signaturePath = containerPath / "signature.sig" - - if fileExists(manifestPath): - removeFile(manifestPath) - if fileExists(environmentPath): - removeFile(environmentPath) - if fileExists(signaturePath): - removeFile(signaturePath) - - # Remove container directory - removeDir(containerPath) - - except Exception as e: - return RemovalResult( - success: false, - containerName: containerName, - removedPath: containerPath, - chunksMarkedForGC: 0, - error: "Failed to remove container files: " & e.msg - ) - - # Remove references to mark chunks for garbage collection - try: - if fileExists(refsPath): - removeFile(refsPath) - except Exception as e: - return RemovalResult( - success: false, - containerName: containerName, - removedPath: containerPath, - chunksMarkedForGC: 0, - error: "Failed to remove references: " & e.msg - ) - - return RemovalResult( - success: true, - containerName: containerName, - removedPath: containerPath, - chunksMarkedForGC: chunksMarkedForGC, - error: "" - ) - - except Exception as e: - return RemovalResult( - success: false, - containerName: containerName, - removedPath: containerPath, - chunksMarkedForGC: 0, - error: "Removal failed: " & e.msg - ) - -# ============================================================================ -# Batch Removal -# ============================================================================ - -proc removeAllNEXTER*(storageRoot: string = ""): seq[RemovalResult] = - ## Remove all NEXTER containers - ## - ## **Requirements:** - ## - Requirement 5.3: Remove all containers - ## - ## **Process:** - ## 1. List all installed containers - ## 2. Remove each container - ## 3. Return results for each removal - ## - ## **Returns:** - ## - Sequence of RemovalResult for each container - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let nextersDir = root / "nexters" - - result = @[] - - if not dirExists(nextersDir): - return - - try: - for entry in walkDir(nextersDir): - if entry.kind == pcDir: - let containerName = entry.path.extractFilename() - let removalResult = removeNEXTER(containerName, storageRoot) - result.add(removalResult) - except: - discard - -# ============================================================================ -# Verification -# ============================================================================ - -proc verifyRemoval*(containerName: string, storageRoot: string = ""): bool = - ## Verify container has been removed - ## - ## **Requirements:** - ## - Requirement 5.3: Verify removal - ## - ## **Checks:** - ## 1. Container directory doesn't exist - ## 2. References file doesn't exist - ## 3. No manifest files remain - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let containerPath = root / "nexters" / containerName - let casRoot = root / "cas" - let refsPath = casRoot / "refs" / "nexters" / (containerName & ".refs") - - # Check container directory - if dirExists(containerPath): - return false - - # Check references - if fileExists(refsPath): - return false - - return true - -# ============================================================================ -# Cleanup Utilities -# ============================================================================ - -proc cleanupOrphanedReferences*(storageRoot: string = ""): int = - ## Clean up orphaned reference files - ## - ## **Requirements:** - ## - Requirement 12.1: Clean up orphaned references - ## - ## **Process:** - ## 1. List all reference files - ## 2. Check if corresponding container exists - ## 3. Remove orphaned references - ## - ## **Returns:** - ## - Number of orphaned references cleaned up - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let nextersDir = root / "nexters" - let refsDir = root / "cas" / "refs" / "nexters" - - var cleanedCount = 0 - - if not dirExists(refsDir): - return 0 - - try: - for refFile in walkFiles(refsDir / "*.refs"): - let containerName = refFile.extractFilename().replace(".refs", "") - let containerPath = nextersDir / containerName - - # If container doesn't exist, remove the reference - if not dirExists(containerPath): - try: - removeFile(refFile) - cleanedCount += 1 - except: - discard - except: - discard - - return cleanedCount - -# ============================================================================ -# Formatting -# ============================================================================ - -proc `$`*(removalResult: RemovalResult): string = - ## Format removal result as string - if removalResult.success: - return "✅ Removed " & removalResult.containerName & " (" & $removalResult.chunksMarkedForGC & " chunks marked for GC)" - else: - return "❌ Failed to remove " & removalResult.containerName & ": " & removalResult.error diff --git a/src/nip/nip_installer.nim b/src/nip/nip_installer.nim deleted file mode 100644 index e51fe2a..0000000 --- a/src/nip/nip_installer.nim +++ /dev/null @@ -1,249 +0,0 @@ -## NIP Installer - User Application Installation -## -## This module handles the installation of NIP packages (User Applications) -## into the user's home directory (~/.local/share/nexus/nips). -## It integrates with the desktop environment via XDG standards. - -import std/[os, strutils, strformat, options, logging, sequtils, osproc] -import nip/manifest_parser -import nip/cas -import nip/types - -type - NipInstaller* = ref object - casRoot*: string - installRoot*: string # ~/.local/share/nexus/nips - appsRoot*: string # ~/.local/share/applications - iconsRoot*: string # ~/.local/share/icons - dryRun*: bool - -proc newNipInstaller*(casRoot: string, dryRun: bool = false): NipInstaller = - let home = getHomeDir() - result = NipInstaller( - casRoot: casRoot, - installRoot: home / ".local/share/nexus/nips", - appsRoot: home / ".local/share/applications", - iconsRoot: home / ".local/share/icons", - dryRun: dryRun - ) - -proc log(ni: NipInstaller, msg: string) = - if ni.dryRun: - echo "[DRY-RUN] " & msg - else: - info(msg) - -# ============================================================================ -# File Reconstruction (Shared Logic - could be refactored) -# ============================================================================ - -proc reconstructFiles(ni: NipInstaller, manifest: PackageManifest, installDir: string) = - ## Reconstruct files from CAS - ni.log(fmt"Reconstructing files for {manifest.name} in {installDir}") - - if not ni.dryRun: - createDir(installDir) - - for file in manifest.files: - let destPath = installDir / file.path - let destDir = destPath.parentDir - - if not ni.dryRun: - createDir(destDir) - try: - # Retrieve content from CAS - let content = retrieveObject(Multihash(file.hash), ni.casRoot) - writeFile(destPath, content) - - # Set permissions (basic) - # TODO: Parse permissions string properly - setFilePermissions(destPath, {fpUserRead, fpUserWrite, fpUserExec}) - - # Add CAS reference - let refId = fmt"{manifest.name}:{manifest.version}" - addReference(ni.casRoot, Multihash(file.hash), "nip", refId) - - except Exception as e: - error(fmt"Failed to reconstruct file {file.path}: {e.msg}") - raise - -# ============================================================================ -# Desktop Integration -# ============================================================================ - -proc generateDesktopFile(ni: NipInstaller, manifest: PackageManifest) = - ## Generate .desktop file for the application - if manifest.desktop.isNone: - return - - let dt = manifest.desktop.get() - let desktopFile = ni.appsRoot / (manifest.name & ".desktop") - - ni.log(fmt"Generating desktop entry: {desktopFile}") - - if not ni.dryRun: - createDir(ni.appsRoot) - - var content = "[Desktop Entry]\n" - content.add("Type=Application\n") - content.add(fmt"Name={dt.displayName}\n") - - # Exec command - # We use 'nip run' to launch the app in its sandbox - # TODO: Ensure 'nip' is in PATH or use absolute path - content.add(fmt"Exec=nip run {manifest.name}\n") - - if dt.icon.isSome: - content.add(fmt"Icon={dt.icon.get()}\n") - - if dt.categories.len > 0: - content.add("Categories=" & dt.categories.join(";") & ";\n") - - if dt.keywords.len > 0: - content.add("Keywords=" & dt.keywords.join(";") & ";\n") - - if dt.mimeTypes.len > 0: - content.add("MimeType=" & dt.mimeTypes.join(";") & ";\n") - - content.add(fmt"Terminal={dt.terminal}\n") - content.add(fmt"StartupNotify={dt.startupNotify}\n") - - if dt.startupWMClass.isSome: - content.add(fmt"StartupWMClass={dt.startupWMClass.get()}\n") - - writeFile(desktopFile, content) - -proc installIcons(ni: NipInstaller, manifest: PackageManifest, installDir: string) = - ## Install icons to ~/.local/share/icons - if manifest.desktop.isNone: return - let dt = manifest.desktop.get() - if dt.icon.isNone: return - - let iconName = dt.icon.get() - # Check if icon is a file path in the package - # We assume standard paths like share/icons/hicolor/48x48/apps/icon.png - # Or just a file at the root? - # For MVP, let's look for the file in the installDir - - # Heuristic: If iconName has an extension, it's a file. - if iconName.endsWith(".png") or iconName.endsWith(".svg"): - let srcPath = installDir / iconName - if fileExists(srcPath): - # Determine destination based on size/type? - # For MVP, put in hicolor/48x48/apps/ if png, scalable/apps/ if svg - # Better: Just put in ~/.local/share/icons/hicolor/48x48/apps/ for now - # Or ~/.local/share/icons/ if we don't know size - - let destDir = if iconName.endsWith(".svg"): - ni.iconsRoot / "hicolor/scalable/apps" - else: - ni.iconsRoot / "hicolor/48x48/apps" - - let destPath = destDir / (manifest.name & iconName.extractFilename.splitFile.ext) - - ni.log(fmt"Installing icon to {destPath}") - if not ni.dryRun: - createDir(destDir) - copyFile(srcPath, destPath) - -proc updateDesktopDb(ni: NipInstaller) = - ## Update desktop database - ni.log("Updating desktop database") - if not ni.dryRun: - discard execCmd("update-desktop-database " & ni.appsRoot) - -# ============================================================================ -# Main Installation Procedure -# ============================================================================ - -proc installNip*(ni: NipInstaller, manifest: PackageManifest) = - ## Install a NIP package - info(fmt"Installing NIP: {manifest.name} v{manifest.version}") - - # 1. Determine paths - let installDir = ni.installRoot / manifest.name / $manifest.version / manifest.artifactHash - let currentLink = ni.installRoot / manifest.name / "Current" - - # 2. Reconstruct files - ni.reconstructFiles(manifest, installDir) - - # 2.1 Write Manifest - # We need the manifest at runtime for sandboxing configuration - if not ni.dryRun: - writeFile(installDir / "manifest.kdl", serializeManifestToKDL(manifest)) - - # 3. Update 'Current' symlink - if not ni.dryRun: - if symlinkExists(currentLink) or fileExists(currentLink): - removeFile(currentLink) - createSymlink(installDir, currentLink) - - # 4. Desktop Integration - ni.generateDesktopFile(manifest) - ni.installIcons(manifest, installDir) - ni.updateDesktopDb() - - info(fmt"NIP installation of {manifest.name} complete") - -# ============================================================================ -# Removal Procedure -# ============================================================================ - -proc removeNip*(ni: NipInstaller, manifest: PackageManifest) = - ## Remove a NIP package - info(fmt"Removing NIP: {manifest.name}") - - let installDir = ni.installRoot / manifest.name / $manifest.version / manifest.artifactHash - let currentLink = ni.installRoot / manifest.name / "Current" - let desktopFile = ni.appsRoot / (manifest.name & ".desktop") - - # 1. Remove Desktop Entry - if fileExists(desktopFile): - ni.log("Removing desktop entry") - if not ni.dryRun: - removeFile(desktopFile) - - # 1.5 Remove Icons (Best effort) - # We'd need to know where we put them. - # For now, check standard locations - let iconPng = ni.iconsRoot / "hicolor/48x48/apps" / (manifest.name & ".png") - if fileExists(iconPng): - ni.log("Removing icon (png)") - if not ni.dryRun: removeFile(iconPng) - - let iconSvg = ni.iconsRoot / "hicolor/scalable/apps" / (manifest.name & ".svg") - if fileExists(iconSvg): - ni.log("Removing icon (svg)") - if not ni.dryRun: removeFile(iconSvg) - - # 2. Remove 'Current' link if it points to this version - if symlinkExists(currentLink): - if expandSymlink(currentLink) == installDir: - ni.log("Removing Current symlink") - if not ni.dryRun: - removeFile(currentLink) - - # 3. Remove Installation Directory - if dirExists(installDir): - ni.log("Removing installation directory") - if not ni.dryRun: - removeDir(installDir) - - # Clean up parent dirs - let versionDir = installDir.parentDir - if dirExists(versionDir) and toSeq(walkDir(versionDir)).len == 0: - removeDir(versionDir) - - let packageDir = ni.installRoot / manifest.name - if dirExists(packageDir) and toSeq(walkDir(packageDir)).len == 0: - removeDir(packageDir) - - # 4. Remove CAS References - ni.log("Removing CAS references") - if not ni.dryRun: - let refId = fmt"{manifest.name}:{manifest.version}" - for file in manifest.files: - removeReference(ni.casRoot, Multihash(file.hash), "nip", refId) - - ni.updateDesktopDb() - info(fmt"NIP removal of {manifest.name} complete") diff --git a/src/nip/nip_manifest.nim b/src/nip/nip_manifest.nim deleted file mode 100644 index 1009dae..0000000 --- a/src/nip/nip_manifest.nim +++ /dev/null @@ -1,768 +0,0 @@ -## NIP Manifest Schema - User Application Format -## -## **Purpose:** -## Defines the NIP (Nexus Installation Package) manifest schema for user applications. -## NIP packages are sandboxed desktop applications with namespace isolation. -## -## **Design Principles:** -## - Desktop integration (icons, .desktop files, MIME types) -## - Namespace isolation with permission controls -## - User-level installation (no root required) -## - Sandboxed execution environment -## - Ed25519 signature support -## -## **Requirements:** -## - Requirement 4.1: manifest.kdl, metadata.json, desktop integration files, CAS chunks, signature -## - Requirement 4.2: app name, version, permissions, namespace config, CAS chunk references -## - Requirement 4.3: .desktop file, icons, MIME type associations -## - Requirement 6.2: KDL format with chunk references by xxh3 hash -## - Requirement 6.5: exact versions and build hashes for dependencies - -import std/[times, options, strutils, tables] -import nip/manifest_parser - -type - # ============================================================================ - # NIP-Specific Types - # ============================================================================ - - NIPManifest* = object - ## Complete NIP manifest for user applications - # Core identity - name*: string - version*: SemanticVersion - buildDate*: DateTime - - # Application metadata - metadata*: AppInfo - provenance*: ProvenanceInfo - buildConfig*: BuildConfiguration - - # CAS chunk references - casChunks*: seq[ChunkReference] - - # Desktop integration - desktop*: DesktopMetadata - - # Namespace isolation and permissions - namespace*: NamespaceConfig - - # Integrity - buildHash*: string ## xxh3-128 hash of build configuration - signature*: SignatureInfo - - AppInfo* = object - ## Application metadata - description*: string - homepage*: Option[string] - license*: string - author*: Option[string] - maintainer*: Option[string] - tags*: seq[string] - category*: Option[string] ## Application category (e.g., "Graphics", "Network") - - ProvenanceInfo* = object - ## Complete provenance tracking - source*: string ## Source URL or repository - sourceHash*: string ## xxh3-128 hash of source - upstream*: Option[string] ## Upstream project URL - buildTimestamp*: DateTime - builder*: Option[string] ## Who built this package - - BuildConfiguration* = object - ## Build configuration for reproducibility - configureFlags*: seq[string] - compilerFlags*: seq[string] - compilerVersion*: string - targetArchitecture*: string - libc*: string ## musl, glibc - allocator*: string ## jemalloc, tcmalloc, default - buildSystem*: string ## cmake, meson, autotools, etc. - - ChunkReference* = object - ## Reference to a CAS chunk - hash*: string ## xxh3-128 hash - size*: int64 - chunkType*: ChunkType - path*: string ## Relative path in package - - ChunkType* = enum - ## Type of chunk content - Binary, Library, Runtime, Config, Data - - DesktopMetadata* = object - ## Desktop integration metadata - desktopFile*: DesktopFileSpec - icons*: seq[IconSpec] - mimeTypes*: seq[string] - appId*: string ## Unique application ID (e.g., "org.mozilla.firefox") - - DesktopFileSpec* = object - ## .desktop file specification - name*: string ## Display name - genericName*: Option[string] - comment*: Option[string] - exec*: string ## Executable command - icon*: string ## Icon name - terminal*: bool - categories*: seq[string] - keywords*: seq[string] - - IconSpec* = object - ## Icon specification - size*: int ## Icon size (e.g., 48, 64, 128) - path*: string ## Path to icon file in package - format*: string ## Icon format (png, svg) - - NamespaceConfig* = object - ## Namespace isolation configuration - namespaceType*: string ## "user", "strict", "none" - permissions*: Permissions - mounts*: seq[Mount] - - Permissions* = object - ## Application permissions - network*: bool - gpu*: bool - audio*: bool - camera*: bool - microphone*: bool - filesystem*: seq[FilesystemAccess] - dbus*: DBusAccess - - FilesystemAccess* = object - ## Filesystem access permission - path*: string - mode*: AccessMode - - AccessMode* = enum - ## Filesystem access mode - ReadOnly, ReadWrite, Create - - DBusAccess* = object - ## D-Bus access permissions - session*: seq[string] ## Session bus names - system*: seq[string] ## System bus names - own*: seq[string] ## Bus names to own - - Mount* = object - ## Filesystem mount specification - source*: string - target*: string - mountType*: MountType - readOnly*: bool - - MountType* = enum - ## Mount type - Bind, Tmpfs, Devtmpfs - - SignatureInfo* = object - ## Ed25519 signature information - algorithm*: string ## "ed25519" - keyId*: string - signature*: string ## Base64-encoded signature - - # ============================================================================ - # Error Types - # ============================================================================ - - NIPError* = object of CatchableError - code*: NIPErrorCode - context*: string - - NIPErrorCode* = enum - InvalidManifest, - MissingField, - InvalidHash, - InvalidSignature, - InvalidPermissions - -# ============================================================================ -# KDL Parsing - Minimal implementation to expose gaps via tests -# ============================================================================ - -proc parseNIPManifest*(kdl: string): NIPManifest = - ## Parse NIP manifest from KDL format - ## - ## **Requirements:** - ## - Requirement 4.2: Parse app name, version, permissions, namespace config, CAS chunks - ## - Requirement 4.3: Parse .desktop file, icons, MIME type associations - ## - Requirement 6.2: Validate chunk references by xxh3 hash - ## - Requirement 6.5: Parse exact versions and build hashes for dependencies - - # Simple line-based parser for the KDL format we generate - # This works because we control the generation format - - var lines = kdl.splitLines() - var name = "" - var version = SemanticVersion(major: 0, minor: 0, patch: 0) - var buildDate = now() - var buildHash = "" - - var metadata = AppInfo(description: "", license: "", tags: @[]) - var provenance = ProvenanceInfo(source: "", sourceHash: "", buildTimestamp: now()) - var buildConfig = BuildConfiguration( - configureFlags: @[], compilerFlags: @[], - compilerVersion: "", targetArchitecture: "", - libc: "", allocator: "", buildSystem: "" - ) - var casChunks: seq[ChunkReference] = @[] - var desktop = DesktopMetadata( - desktopFile: DesktopFileSpec(name: "", exec: "", icon: "", terminal: false, categories: @[], keywords: @[]), - icons: @[], mimeTypes: @[], appId: "" - ) - var namespace = NamespaceConfig( - namespaceType: "user", - permissions: Permissions( - network: false, gpu: false, audio: false, camera: false, microphone: false, - filesystem: @[], dbus: DBusAccess(session: @[], system: @[], own: @[]) - ), - mounts: @[] - ) - var signature = SignatureInfo(algorithm: "", keyId: "", signature: "") - - # Helper to extract quoted string - proc extractQuoted(line: string): string = - let start = line.find("\"") - if start >= 0: - let endIdx = line.find("\"", start + 1) - if endIdx > start: - return line[start+1..= 3: - version = SemanticVersion( - major: parseInt(parts[0]), - minor: parseInt(parts[1]), - patch: parseInt(parts[2]) - ) - - elif line.startsWith("build_date \""): - let dateStr = extractQuoted(line) - try: - buildDate = parse(dateStr, "yyyy-MM-dd'T'HH:mm:ss'Z'") - except: - buildDate = now() - - elif line.startsWith("build_hash \""): - buildHash = extractQuoted(line) - - # Track sections - elif line == "metadata {": - currentSection = "metadata" - elif line == "provenance {": - currentSection = "provenance" - elif line == "build_config {": - currentSection = "build_config" - elif line == "cas_chunks {": - currentSection = "cas_chunks" - elif line == "desktop {": - currentSection = "desktop" - elif line == "desktop_file {": - currentSection = "desktop_file" - elif line == "icons {": - currentSection = "icons" - elif line == "namespace {": - currentSection = "namespace" - elif line == "permissions {": - currentSection = "permissions" - elif line == "filesystem {": - currentSection = "filesystem" - elif line == "dbus {": - currentSection = "dbus" - elif line == "mounts {": - currentSection = "mounts" - elif line == "signature {": - currentSection = "signature" - - # Parse section content - elif currentSection == "metadata": - if line.startsWith("description \""): - metadata.description = extractQuoted(line) - elif line.startsWith("license \""): - metadata.license = extractQuoted(line) - elif line.startsWith("homepage \""): - metadata.homepage = some(extractQuoted(line)) - elif line.startsWith("author \""): - metadata.author = some(extractQuoted(line)) - elif line.startsWith("maintainer \""): - metadata.maintainer = some(extractQuoted(line)) - elif line.startsWith("category \""): - metadata.category = some(extractQuoted(line)) - elif line.startsWith("tags \""): - let tagsStr = extractQuoted(line) - metadata.tags = tagsStr.split() - - elif currentSection == "provenance": - if line.startsWith("source \""): - provenance.source = extractQuoted(line) - elif line.startsWith("source_hash \""): - provenance.sourceHash = extractQuoted(line) - elif line.startsWith("upstream \""): - provenance.upstream = some(extractQuoted(line)) - elif line.startsWith("build_timestamp \""): - let dateStr = extractQuoted(line) - try: - provenance.buildTimestamp = parse(dateStr, "yyyy-MM-dd'T'HH:mm:ss'Z'") - except: - provenance.buildTimestamp = now() - elif line.startsWith("builder \""): - provenance.builder = some(extractQuoted(line)) - - elif currentSection == "build_config": - if line.startsWith("configure_flags \""): - let flagsStr = extractQuoted(line) - buildConfig.configureFlags = flagsStr.split() - elif line.startsWith("compiler_flags \""): - let flagsStr = extractQuoted(line) - buildConfig.compilerFlags = flagsStr.split() - elif line.startsWith("compiler_version \""): - buildConfig.compilerVersion = extractQuoted(line) - elif line.startsWith("target_architecture \""): - buildConfig.targetArchitecture = extractQuoted(line) - elif line.startsWith("libc \""): - buildConfig.libc = extractQuoted(line) - elif line.startsWith("allocator \""): - buildConfig.allocator = extractQuoted(line) - elif line.startsWith("build_system \""): - buildConfig.buildSystem = extractQuoted(line) - - elif currentSection == "cas_chunks": - if line.startsWith("chunk \""): - currentChunk = ChunkReference(hash: extractQuoted(line), size: 0, chunkType: Binary, path: "") - elif line.startsWith("size "): - currentChunk.size = extractInt(line).int64 - elif line.startsWith("type \""): - let typeStr = extractQuoted(line) - case typeStr: - of "binary": currentChunk.chunkType = Binary - of "library": currentChunk.chunkType = Library - of "runtime": currentChunk.chunkType = Runtime - of "config": currentChunk.chunkType = Config - of "data": currentChunk.chunkType = Data - else: currentChunk.chunkType = Binary - elif line.startsWith("path \""): - currentChunk.path = extractQuoted(line) - elif line == "}": - if currentChunk.hash.len > 0: - casChunks.add(currentChunk) - currentChunk = ChunkReference(hash: "", size: 0, chunkType: Binary, path: "") - skipSectionReset = true # Don't reset section, we're still in cas_chunks - - elif currentSection == "desktop": - if line.startsWith("app_id \""): - desktop.appId = extractQuoted(line) - elif line.startsWith("mime_types \""): - let mimeStr = extractQuoted(line) - desktop.mimeTypes = mimeStr.split(";") - - elif currentSection == "desktop_file": - if line.startsWith("name \""): - desktop.desktopFile.name = extractQuoted(line) - elif line.startsWith("generic_name \""): - desktop.desktopFile.genericName = some(extractQuoted(line)) - elif line.startsWith("comment \""): - desktop.desktopFile.comment = some(extractQuoted(line)) - elif line.startsWith("exec \""): - desktop.desktopFile.exec = extractQuoted(line) - elif line.startsWith("icon \""): - desktop.desktopFile.icon = extractQuoted(line) - elif line.startsWith("terminal "): - desktop.desktopFile.terminal = extractBool(line) - elif line.startsWith("categories \""): - let catStr = extractQuoted(line) - desktop.desktopFile.categories = catStr.split(";") - elif line.startsWith("keywords \""): - let kwStr = extractQuoted(line) - desktop.desktopFile.keywords = kwStr.split(";") - - elif currentSection == "icons": - if line.startsWith("icon {"): - currentIcon = IconSpec(size: 0, path: "", format: "") - elif line.startsWith("size "): - currentIcon.size = extractInt(line) - elif line.startsWith("path \""): - currentIcon.path = extractQuoted(line) - elif line.startsWith("format \""): - currentIcon.format = extractQuoted(line) - elif line == "}" and currentIcon.path.len > 0: - # This closes an individual icon block - desktop.icons.add(currentIcon) - currentIcon = IconSpec(size: 0, path: "", format: "") - skipSectionReset = true # Don't reset section, we're still in icons - - elif currentSection == "namespace": - if line.startsWith("type \""): - namespace.namespaceType = extractQuoted(line) - - elif currentSection == "permissions": - if line.startsWith("network "): - namespace.permissions.network = extractBool(line) - elif line.startsWith("gpu "): - namespace.permissions.gpu = extractBool(line) - elif line.startsWith("audio "): - namespace.permissions.audio = extractBool(line) - elif line.startsWith("camera "): - namespace.permissions.camera = extractBool(line) - elif line.startsWith("microphone "): - namespace.permissions.microphone = extractBool(line) - - elif currentSection == "filesystem": - if line.startsWith("access \""): - let parts = line.split("\"") - if parts.len >= 4: - currentFsAccess = FilesystemAccess(path: parts[1], mode: ReadOnly) - let modeStr = parts[3].toLowerAscii() - case modeStr: - of "readonly": currentFsAccess.mode = ReadOnly - of "readwrite": currentFsAccess.mode = ReadWrite - of "create": currentFsAccess.mode = Create - else: currentFsAccess.mode = ReadOnly - namespace.permissions.filesystem.add(currentFsAccess) - - elif currentSection == "dbus": - if line.startsWith("session \""): - let sessStr = extractQuoted(line) - namespace.permissions.dbus.session = sessStr.split() - elif line.startsWith("system \""): - let sysStr = extractQuoted(line) - namespace.permissions.dbus.system = sysStr.split() - elif line.startsWith("own \""): - let ownStr = extractQuoted(line) - namespace.permissions.dbus.own = ownStr.split() - - elif currentSection == "mounts": - if line.startsWith("mount {"): - currentMount = Mount(source: "", target: "", mountType: Bind, readOnly: false) - elif line.startsWith("source \""): - currentMount.source = extractQuoted(line) - elif line.startsWith("target \""): - currentMount.target = extractQuoted(line) - elif line.startsWith("type \""): - let typeStr = extractQuoted(line) - case typeStr: - of "bind": currentMount.mountType = Bind - of "tmpfs": currentMount.mountType = Tmpfs - of "devtmpfs": currentMount.mountType = Devtmpfs - else: currentMount.mountType = Bind - elif line.startsWith("read_only "): - currentMount.readOnly = extractBool(line) - elif line == "}": - if currentMount.source.len > 0: - namespace.mounts.add(currentMount) - currentMount = Mount(source: "", target: "", mountType: Bind, readOnly: false) - skipSectionReset = true # Don't reset section, we're still in mounts - - elif currentSection == "signature": - if line.startsWith("algorithm \""): - signature.algorithm = extractQuoted(line) - elif line.startsWith("key_id \""): - signature.keyId = extractQuoted(line) - elif line.startsWith("signature \""): - signature.signature = extractQuoted(line) - - # Reset section on closing brace (unless we just processed a nested block) - if line == "}" and currentSection != "" and not skipSectionReset: - if currentSection in ["metadata", "provenance", "build_config", "desktop", "namespace", "signature"]: - currentSection = "" - elif currentSection == "desktop_file": - currentSection = "desktop" - elif currentSection == "icons": - currentSection = "desktop" - elif currentSection == "permissions": - currentSection = "namespace" - elif currentSection == "filesystem": - currentSection = "permissions" - elif currentSection == "dbus": - currentSection = "permissions" - elif currentSection == "mounts": - currentSection = "namespace" - elif currentSection == "cas_chunks": - currentSection = "" - - # Reset the skip flag for next iteration - skipSectionReset = false - - i += 1 - - result = NIPManifest( - name: name, - version: version, - buildDate: buildDate, - metadata: metadata, - provenance: provenance, - buildConfig: buildConfig, - casChunks: casChunks, - desktop: desktop, - namespace: namespace, - buildHash: buildHash, - signature: signature - ) - -# ============================================================================ -# KDL Generation -# ============================================================================ - -proc generateNIPManifest*(manifest: NIPManifest): string = - ## Generate KDL manifest from NIPManifest - ## - ## **Requirements:** - ## - Requirement 4.2: Generate app name, version, permissions, namespace config, CAS chunks - ## - Requirement 4.3: Generate .desktop file, icons, MIME type associations - ## - Requirement 6.4: Deterministic generation (same input = same output) - ## - ## **Determinism:** Fields are output in a fixed order to ensure same input = same output - - result = "app \"" & manifest.name & "\" {\n" - - # Core identity - result.add(" version \"" & $manifest.version & "\"\n") - result.add(" build_date \"" & manifest.buildDate.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - result.add("\n") - - # Metadata section - result.add(" metadata {\n") - result.add(" description \"" & manifest.metadata.description & "\"\n") - result.add(" license \"" & manifest.metadata.license & "\"\n") - if manifest.metadata.homepage.isSome: - result.add(" homepage \"" & manifest.metadata.homepage.get() & "\"\n") - if manifest.metadata.author.isSome: - result.add(" author \"" & manifest.metadata.author.get() & "\"\n") - if manifest.metadata.maintainer.isSome: - result.add(" maintainer \"" & manifest.metadata.maintainer.get() & "\"\n") - if manifest.metadata.category.isSome: - result.add(" category \"" & manifest.metadata.category.get() & "\"\n") - if manifest.metadata.tags.len > 0: - result.add(" tags \"" & manifest.metadata.tags.join(" ") & "\"\n") - result.add(" }\n\n") - - # Provenance section - result.add(" provenance {\n") - result.add(" source \"" & manifest.provenance.source & "\"\n") - result.add(" source_hash \"" & manifest.provenance.sourceHash & "\"\n") - if manifest.provenance.upstream.isSome: - result.add(" upstream \"" & manifest.provenance.upstream.get() & "\"\n") - result.add(" build_timestamp \"" & manifest.provenance.buildTimestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - if manifest.provenance.builder.isSome: - result.add(" builder \"" & manifest.provenance.builder.get() & "\"\n") - result.add(" }\n\n") - - # Build configuration section - result.add(" build_config {\n") - if manifest.buildConfig.configureFlags.len > 0: - result.add(" configure_flags \"" & manifest.buildConfig.configureFlags.join(" ") & "\"\n") - if manifest.buildConfig.compilerFlags.len > 0: - result.add(" compiler_flags \"" & manifest.buildConfig.compilerFlags.join(" ") & "\"\n") - result.add(" compiler_version \"" & manifest.buildConfig.compilerVersion & "\"\n") - result.add(" target_architecture \"" & manifest.buildConfig.targetArchitecture & "\"\n") - result.add(" libc \"" & manifest.buildConfig.libc & "\"\n") - result.add(" allocator \"" & manifest.buildConfig.allocator & "\"\n") - result.add(" build_system \"" & manifest.buildConfig.buildSystem & "\"\n") - result.add(" }\n\n") - - # CAS chunks section - if manifest.casChunks.len > 0: - result.add(" cas_chunks {\n") - for chunk in manifest.casChunks: - result.add(" chunk \"" & chunk.hash & "\" {\n") - result.add(" size " & $chunk.size & "\n") - result.add(" type \"" & ($chunk.chunkType).toLowerAscii() & "\"\n") - result.add(" path \"" & chunk.path & "\"\n") - result.add(" }\n") - result.add(" }\n\n") - - # Desktop integration section - result.add(" desktop {\n") - result.add(" app_id \"" & manifest.desktop.appId & "\"\n\n") - - # Desktop file - result.add(" desktop_file {\n") - result.add(" name \"" & manifest.desktop.desktopFile.name & "\"\n") - if manifest.desktop.desktopFile.genericName.isSome: - result.add(" generic_name \"" & manifest.desktop.desktopFile.genericName.get() & "\"\n") - if manifest.desktop.desktopFile.comment.isSome: - result.add(" comment \"" & manifest.desktop.desktopFile.comment.get() & "\"\n") - result.add(" exec \"" & manifest.desktop.desktopFile.exec & "\"\n") - result.add(" icon \"" & manifest.desktop.desktopFile.icon & "\"\n") - result.add(" terminal " & $manifest.desktop.desktopFile.terminal & "\n") - if manifest.desktop.desktopFile.categories.len > 0: - result.add(" categories \"" & manifest.desktop.desktopFile.categories.join(";") & "\"\n") - if manifest.desktop.desktopFile.keywords.len > 0: - result.add(" keywords \"" & manifest.desktop.desktopFile.keywords.join(";") & "\"\n") - result.add(" }\n\n") - - # Icons - if manifest.desktop.icons.len > 0: - result.add(" icons {\n") - for icon in manifest.desktop.icons: - result.add(" icon {\n") - result.add(" size " & $icon.size & "\n") - result.add(" path \"" & icon.path & "\"\n") - result.add(" format \"" & icon.format & "\"\n") - result.add(" }\n") - result.add(" }\n\n") - - # MIME types - if manifest.desktop.mimeTypes.len > 0: - result.add(" mime_types \"" & manifest.desktop.mimeTypes.join(";") & "\"\n") - - result.add(" }\n\n") - - # Namespace configuration section - result.add(" namespace {\n") - result.add(" type \"" & manifest.namespace.namespaceType & "\"\n\n") - - # Permissions - result.add(" permissions {\n") - result.add(" network " & $manifest.namespace.permissions.network & "\n") - result.add(" gpu " & $manifest.namespace.permissions.gpu & "\n") - result.add(" audio " & $manifest.namespace.permissions.audio & "\n") - result.add(" camera " & $manifest.namespace.permissions.camera & "\n") - result.add(" microphone " & $manifest.namespace.permissions.microphone & "\n") - - # Filesystem access - if manifest.namespace.permissions.filesystem.len > 0: - result.add("\n filesystem {\n") - for fs in manifest.namespace.permissions.filesystem: - result.add(" access \"" & fs.path & "\" \"" & ($fs.mode).toLowerAscii() & "\"\n") - result.add(" }\n") - - # D-Bus access - if manifest.namespace.permissions.dbus.session.len > 0 or - manifest.namespace.permissions.dbus.system.len > 0 or - manifest.namespace.permissions.dbus.own.len > 0: - result.add("\n dbus {\n") - if manifest.namespace.permissions.dbus.session.len > 0: - result.add(" session \"" & manifest.namespace.permissions.dbus.session.join(" ") & "\"\n") - if manifest.namespace.permissions.dbus.system.len > 0: - result.add(" system \"" & manifest.namespace.permissions.dbus.system.join(" ") & "\"\n") - if manifest.namespace.permissions.dbus.own.len > 0: - result.add(" own \"" & manifest.namespace.permissions.dbus.own.join(" ") & "\"\n") - result.add(" }\n") - - result.add(" }\n") - - # Mounts - if manifest.namespace.mounts.len > 0: - result.add("\n mounts {\n") - for mount in manifest.namespace.mounts: - result.add(" mount {\n") - result.add(" source \"" & mount.source & "\"\n") - result.add(" target \"" & mount.target & "\"\n") - result.add(" type \"" & ($mount.mountType).toLowerAscii() & "\"\n") - result.add(" read_only " & $mount.readOnly & "\n") - result.add(" }\n") - result.add(" }\n") - - result.add(" }\n\n") - - # Build hash - result.add(" build_hash \"" & manifest.buildHash & "\"\n\n") - - # Signature - result.add(" signature {\n") - result.add(" algorithm \"" & manifest.signature.algorithm & "\"\n") - result.add(" key_id \"" & manifest.signature.keyId & "\"\n") - result.add(" signature \"" & manifest.signature.signature & "\"\n") - result.add(" }\n") - - result.add("}\n") - -# ============================================================================ -# Validation -# ============================================================================ - -proc validateNIPManifest*(manifest: NIPManifest): seq[string] = - ## Validate NIP manifest and return list of issues - ## - ## **Requirements:** - ## - Requirement 6.3: Validate all required fields and hash formats - ## - Requirement 4.2: Validate permissions and namespace config - - result = @[] - - # Validate name - if manifest.name.len == 0: - result.add("Application name cannot be empty") - - # Validate build hash format (xxh3-128) - if manifest.buildHash.len > 0 and not manifest.buildHash.startsWith("xxh3-"): - result.add("Build hash must use xxh3-128 format (xxh3-...)") - - # Validate source hash format - if manifest.provenance.sourceHash.len > 0 and not manifest.provenance.sourceHash.startsWith("xxh3-"): - result.add("Source hash must use xxh3-128 format (xxh3-...)") - - # Validate CAS chunks have xxh3 hashes - for chunk in manifest.casChunks: - if not chunk.hash.startsWith("xxh3-"): - result.add("Chunk hash must use xxh3-128 format (xxh3-...)") - if chunk.size <= 0: - result.add("Chunk size must be positive") - - # Validate desktop integration - if manifest.desktop.appId.len == 0: - result.add("Desktop app_id cannot be empty") - if manifest.desktop.desktopFile.name.len == 0: - result.add("Desktop file name cannot be empty") - if manifest.desktop.desktopFile.exec.len == 0: - result.add("Desktop file exec command cannot be empty") - - # Validate namespace type - if manifest.namespace.namespaceType notin ["user", "strict", "none"]: - result.add("Namespace type must be 'user', 'strict', or 'none'") - - # Validate signature - if manifest.signature.algorithm.len > 0 and manifest.signature.algorithm != "ed25519": - result.add("Signature algorithm must be 'ed25519'") - if manifest.signature.keyId.len == 0: - result.add("Signature key_id cannot be empty") - if manifest.signature.signature.len == 0: - result.add("Signature value cannot be empty") - -# ============================================================================ -# Convenience Functions -# ============================================================================ - -proc `$`*(manifest: NIPManifest): string = - ## Convert NIP manifest to human-readable string - result = "NIP Application: " & manifest.name & " v" & $manifest.version & "\n" - result.add("Build Date: " & manifest.buildDate.format("yyyy-MM-dd HH:mm:ss") & "\n") - result.add("License: " & manifest.metadata.license & "\n") - result.add("App ID: " & manifest.desktop.appId & "\n") - result.add("Build Hash: " & manifest.buildHash & "\n") - result.add("CAS Chunks: " & $manifest.casChunks.len & "\n") - result.add("Namespace: " & manifest.namespace.namespaceType & "\n") diff --git a/src/nip/nip_manifest.nim.backup b/src/nip/nip_manifest.nim.backup deleted file mode 100644 index a0d474a..0000000 --- a/src/nip/nip_manifest.nim.backup +++ /dev/null @@ -1,761 +0,0 @@ -## NIP Manifest Schema - User Application Format -## -## **Purpose:** -## Defines the NIP (Nexus Installation Package) manifest schema for user applications. -## NIP packages are sandboxed desktop applications with namespace isolation. -## -## **Design Principles:** -## - Desktop integration (icons, .desktop files, MIME types) -## - Namespace isolation with permission controls -## - User-level installation (no root required) -## - Sandboxed execution environment -## - Ed25519 signature support -## -## **Requirements:** -## - Requirement 4.1: manifest.kdl, metadata.json, desktop integration files, CAS chunks, signature -## - Requirement 4.2: app name, version, permissions, namespace config, CAS chunk references -## - Requirement 4.3: .desktop file, icons, MIME type associations -## - Requirement 6.2: KDL format with chunk references by xxh3 hash -## - Requirement 6.5: exact versions and build hashes for dependencies - -import std/[times, options, strutils, tables] -import nip/manifest_parser - -type - # ============================================================================ - # NIP-Specific Types - # ============================================================================ - - NIPManifest* = object - ## Complete NIP manifest for user applications - # Core identity - name*: string - version*: SemanticVersion - buildDate*: DateTime - - # Application metadata - metadata*: AppInfo - provenance*: ProvenanceInfo - buildConfig*: BuildConfiguration - - # CAS chunk references - casChunks*: seq[ChunkReference] - - # Desktop integration - desktop*: DesktopMetadata - - # Namespace isolation and permissions - namespace*: NamespaceConfig - - # Integrity - buildHash*: string ## xxh3-128 hash of build configuration - signature*: SignatureInfo - - AppInfo* = object - ## Application metadata - description*: string - homepage*: Option[string] - license*: string - author*: Option[string] - maintainer*: Option[string] - tags*: seq[string] - category*: Option[string] ## Application category (e.g., "Graphics", "Network") - - ProvenanceInfo* = object - ## Complete provenance tracking - source*: string ## Source URL or repository - sourceHash*: string ## xxh3-128 hash of source - upstream*: Option[string] ## Upstream project URL - buildTimestamp*: DateTime - builder*: Option[string] ## Who built this package - - BuildConfiguration* = object - ## Build configuration for reproducibility - configureFlags*: seq[string] - compilerFlags*: seq[string] - compilerVersion*: string - targetArchitecture*: string - libc*: string ## musl, glibc - allocator*: string ## jemalloc, tcmalloc, default - buildSystem*: string ## cmake, meson, autotools, etc. - - ChunkReference* = object - ## Reference to a CAS chunk - hash*: string ## xxh3-128 hash - size*: int64 - chunkType*: ChunkType - path*: string ## Relative path in package - - ChunkType* = enum - ## Type of chunk content - Binary, Library, Runtime, Config, Data - - DesktopMetadata* = object - ## Desktop integration metadata - desktopFile*: DesktopFileSpec - icons*: seq[IconSpec] - mimeTypes*: seq[string] - appId*: string ## Unique application ID (e.g., "org.mozilla.firefox") - - DesktopFileSpec* = object - ## .desktop file specification - name*: string ## Display name - genericName*: Option[string] - comment*: Option[string] - exec*: string ## Executable command - icon*: string ## Icon name - terminal*: bool - categories*: seq[string] - keywords*: seq[string] - - IconSpec* = object - ## Icon specification - size*: int ## Icon size (e.g., 48, 64, 128) - path*: string ## Path to icon file in package - format*: string ## Icon format (png, svg) - - NamespaceConfig* = object - ## Namespace isolation configuration - namespaceType*: string ## "user", "strict", "none" - permissions*: Permissions - mounts*: seq[Mount] - - Permissions* = object - ## Application permissions - network*: bool - gpu*: bool - audio*: bool - camera*: bool - microphone*: bool - filesystem*: seq[FilesystemAccess] - dbus*: DBusAccess - - FilesystemAccess* = object - ## Filesystem access permission - path*: string - mode*: AccessMode - - AccessMode* = enum - ## Filesystem access mode - ReadOnly, ReadWrite, Create - - DBusAccess* = object - ## D-Bus access permissions - session*: seq[string] ## Session bus names - system*: seq[string] ## System bus names - own*: seq[string] ## Bus names to own - - Mount* = object - ## Filesystem mount specification - source*: string - target*: string - mountType*: MountType - readOnly*: bool - - MountType* = enum - ## Mount type - Bind, Tmpfs, Devtmpfs - - SignatureInfo* = object - ## Ed25519 signature information - algorithm*: string ## "ed25519" - keyId*: string - signature*: string ## Base64-encoded signature - - # ============================================================================ - # Error Types - # ============================================================================ - - NIPError* = object of CatchableError - code*: NIPErrorCode - context*: string - - NIPErrorCode* = enum - InvalidManifest, - MissingField, - InvalidHash, - InvalidSignature, - InvalidPermissions - -# ============================================================================ -# KDL Parsing - Minimal implementation to expose gaps via tests -# ============================================================================ - -proc parseNIPManifest*(kdl: string): NIPManifest = - ## Parse NIP manifest from KDL format - ## - ## **Requirements:** - ## - Requirement 4.2: Parse app name, version, permissions, namespace config, CAS chunks - ## - Requirement 4.3: Parse .desktop file, icons, MIME type associations - ## - Requirement 6.2: Validate chunk references by xxh3 hash - ## - Requirement 6.5: Parse exact versions and build hashes for dependencies - - # Simple line-based parser for the KDL format we generate - # This works because we control the generation format - - var lines = kdl.splitLines() - var name = "" - var version = SemanticVersion(major: 0, minor: 0, patch: 0) - var buildDate = now() - var buildHash = "" - - var metadata = AppInfo(description: "", license: "", tags: @[]) - var provenance = ProvenanceInfo(source: "", sourceHash: "", buildTimestamp: now()) - var buildConfig = BuildConfiguration( - configureFlags: @[], compilerFlags: @[], - compilerVersion: "", targetArchitecture: "", - libc: "", allocator: "", buildSystem: "" - ) - var casChunks: seq[ChunkReference] = @[] - var desktop = DesktopMetadata( - desktopFile: DesktopFileSpec(name: "", exec: "", icon: "", terminal: false, categories: @[], keywords: @[]), - icons: @[], mimeTypes: @[], appId: "" - ) - var namespace = NamespaceConfig( - namespaceType: "user", - permissions: Permissions( - network: false, gpu: false, audio: false, camera: false, microphone: false, - filesystem: @[], dbus: DBusAccess(session: @[], system: @[], own: @[]) - ), - mounts: @[] - ) - var signature = SignatureInfo(algorithm: "", keyId: "", signature: "") - - # Helper to extract quoted string - proc extractQuoted(line: string): string = - let start = line.find("\"") - if start >= 0: - let endIdx = line.find("\"", start + 1) - if endIdx > start: - return line[start+1..= 3: - version = SemanticVersion( - major: parseInt(parts[0]), - minor: parseInt(parts[1]), - patch: parseInt(parts[2]) - ) - - elif line.startsWith("build_date \""): - let dateStr = extractQuoted(line) - try: - buildDate = parse(dateStr, "yyyy-MM-dd'T'HH:mm:ss'Z'") - except: - buildDate = now() - - elif line.startsWith("build_hash \""): - buildHash = extractQuoted(line) - - # Track sections - elif line == "metadata {": - currentSection = "metadata" - elif line == "provenance {": - currentSection = "provenance" - elif line == "build_config {": - currentSection = "build_config" - elif line == "cas_chunks {": - currentSection = "cas_chunks" - elif line == "desktop {": - currentSection = "desktop" - elif line == "desktop_file {": - currentSection = "desktop_file" - elif line == "icons {": - currentSection = "icons" - elif line == "namespace {": - currentSection = "namespace" - elif line == "permissions {": - currentSection = "permissions" - elif line == "filesystem {": - currentSection = "filesystem" - elif line == "dbus {": - currentSection = "dbus" - elif line == "mounts {": - currentSection = "mounts" - elif line == "signature {": - currentSection = "signature" - - # Parse section content - elif currentSection == "metadata": - if line.startsWith("description \""): - metadata.description = extractQuoted(line) - elif line.startsWith("license \""): - metadata.license = extractQuoted(line) - elif line.startsWith("homepage \""): - metadata.homepage = some(extractQuoted(line)) - elif line.startsWith("author \""): - metadata.author = some(extractQuoted(line)) - elif line.startsWith("maintainer \""): - metadata.maintainer = some(extractQuoted(line)) - elif line.startsWith("category \""): - metadata.category = some(extractQuoted(line)) - elif line.startsWith("tags \""): - let tagsStr = extractQuoted(line) - metadata.tags = tagsStr.split() - - elif currentSection == "provenance": - if line.startsWith("source \""): - provenance.source = extractQuoted(line) - elif line.startsWith("source_hash \""): - provenance.sourceHash = extractQuoted(line) - elif line.startsWith("upstream \""): - provenance.upstream = some(extractQuoted(line)) - elif line.startsWith("build_timestamp \""): - let dateStr = extractQuoted(line) - try: - provenance.buildTimestamp = parse(dateStr, "yyyy-MM-dd'T'HH:mm:ss'Z'") - except: - provenance.buildTimestamp = now() - elif line.startsWith("builder \""): - provenance.builder = some(extractQuoted(line)) - - elif currentSection == "build_config": - if line.startsWith("configure_flags \""): - let flagsStr = extractQuoted(line) - buildConfig.configureFlags = flagsStr.split() - elif line.startsWith("compiler_flags \""): - let flagsStr = extractQuoted(line) - buildConfig.compilerFlags = flagsStr.split() - elif line.startsWith("compiler_version \""): - buildConfig.compilerVersion = extractQuoted(line) - elif line.startsWith("target_architecture \""): - buildConfig.targetArchitecture = extractQuoted(line) - elif line.startsWith("libc \""): - buildConfig.libc = extractQuoted(line) - elif line.startsWith("allocator \""): - buildConfig.allocator = extractQuoted(line) - elif line.startsWith("build_system \""): - buildConfig.buildSystem = extractQuoted(line) - - elif currentSection == "cas_chunks": - if line.startsWith("chunk \""): - currentChunk = ChunkReference(hash: extractQuoted(line), size: 0, chunkType: Binary, path: "") - elif line.startsWith("size "): - currentChunk.size = extractInt(line).int64 - elif line.startsWith("type \""): - let typeStr = extractQuoted(line) - case typeStr: - of "binary": currentChunk.chunkType = Binary - of "library": currentChunk.chunkType = Library - of "runtime": currentChunk.chunkType = Runtime - of "config": currentChunk.chunkType = Config - of "data": currentChunk.chunkType = Data - else: currentChunk.chunkType = Binary - elif line.startsWith("path \""): - currentChunk.path = extractQuoted(line) - elif line == "}": - if currentChunk.hash.len > 0: - casChunks.add(currentChunk) - currentChunk = ChunkReference(hash: "", size: 0, chunkType: Binary, path: "") - - elif currentSection == "desktop": - if line.startsWith("app_id \""): - desktop.appId = extractQuoted(line) - elif line.startsWith("mime_types \""): - let mimeStr = extractQuoted(line) - desktop.mimeTypes = mimeStr.split(";") - - elif currentSection == "desktop_file": - if line.startsWith("name \""): - desktop.desktopFile.name = extractQuoted(line) - elif line.startsWith("generic_name \""): - desktop.desktopFile.genericName = some(extractQuoted(line)) - elif line.startsWith("comment \""): - desktop.desktopFile.comment = some(extractQuoted(line)) - elif line.startsWith("exec \""): - desktop.desktopFile.exec = extractQuoted(line) - elif line.startsWith("icon \""): - desktop.desktopFile.icon = extractQuoted(line) - elif line.startsWith("terminal "): - desktop.desktopFile.terminal = extractBool(line) - elif line.startsWith("categories \""): - let catStr = extractQuoted(line) - desktop.desktopFile.categories = catStr.split(";") - elif line.startsWith("keywords \""): - let kwStr = extractQuoted(line) - desktop.desktopFile.keywords = kwStr.split(";") - - elif currentSection == "icons": - if line.startsWith("icon {"): - currentIcon = IconSpec(size: 0, path: "", format: "") - elif line.startsWith("size "): - currentIcon.size = extractInt(line) - elif line.startsWith("path \""): - currentIcon.path = extractQuoted(line) - elif line.startsWith("format \""): - currentIcon.format = extractQuoted(line) - elif line == "}": - if currentIcon.path.len > 0: - desktop.icons.add(currentIcon) - currentIcon = IconSpec(size: 0, path: "", format: "") - - elif currentSection == "namespace": - if line.startsWith("type \""): - namespace.namespaceType = extractQuoted(line) - - elif currentSection == "permissions": - if line.startsWith("network "): - namespace.permissions.network = extractBool(line) - elif line.startsWith("gpu "): - namespace.permissions.gpu = extractBool(line) - elif line.startsWith("audio "): - namespace.permissions.audio = extractBool(line) - elif line.startsWith("camera "): - namespace.permissions.camera = extractBool(line) - elif line.startsWith("microphone "): - namespace.permissions.microphone = extractBool(line) - - elif currentSection == "filesystem": - if line.startsWith("access \""): - let parts = line.split("\"") - if parts.len >= 4: - currentFsAccess = FilesystemAccess(path: parts[1], mode: ReadOnly) - let modeStr = parts[3].toLowerAscii() - case modeStr: - of "readonly": currentFsAccess.mode = ReadOnly - of "readwrite": currentFsAccess.mode = ReadWrite - of "create": currentFsAccess.mode = Create - else: currentFsAccess.mode = ReadOnly - namespace.permissions.filesystem.add(currentFsAccess) - - elif currentSection == "dbus": - if line.startsWith("session \""): - let sessStr = extractQuoted(line) - namespace.permissions.dbus.session = sessStr.split() - elif line.startsWith("system \""): - let sysStr = extractQuoted(line) - namespace.permissions.dbus.system = sysStr.split() - elif line.startsWith("own \""): - let ownStr = extractQuoted(line) - namespace.permissions.dbus.own = ownStr.split() - - elif currentSection == "mounts": - if line.startsWith("mount {"): - currentMount = Mount(source: "", target: "", mountType: Bind, readOnly: false) - elif line.startsWith("source \""): - currentMount.source = extractQuoted(line) - elif line.startsWith("target \""): - currentMount.target = extractQuoted(line) - elif line.startsWith("type \""): - let typeStr = extractQuoted(line) - case typeStr: - of "bind": currentMount.mountType = Bind - of "tmpfs": currentMount.mountType = Tmpfs - of "devtmpfs": currentMount.mountType = Devtmpfs - else: currentMount.mountType = Bind - elif line.startsWith("read_only "): - currentMount.readOnly = extractBool(line) - elif line == "}": - if currentMount.source.len > 0: - namespace.mounts.add(currentMount) - currentMount = Mount(source: "", target: "", mountType: Bind, readOnly: false) - - elif currentSection == "signature": - if line.startsWith("algorithm \""): - signature.algorithm = extractQuoted(line) - elif line.startsWith("key_id \""): - signature.keyId = extractQuoted(line) - elif line.startsWith("signature \""): - signature.signature = extractQuoted(line) - - # Reset section on closing brace - if line == "}" and currentSection != "": - if currentSection in ["metadata", "provenance", "build_config", "desktop", "namespace", "signature"]: - currentSection = "" - elif currentSection == "desktop_file": - currentSection = "desktop" - elif currentSection == "icons": - currentSection = "desktop" - elif currentSection == "permissions": - currentSection = "namespace" - elif currentSection == "filesystem": - currentSection = "permissions" - elif currentSection == "dbus": - currentSection = "permissions" - elif currentSection == "mounts": - currentSection = "namespace" - elif currentSection == "cas_chunks": - currentSection = "" - - i += 1 - - result = NIPManifest( - name: name, - version: version, - buildDate: buildDate, - metadata: metadata, - provenance: provenance, - buildConfig: buildConfig, - casChunks: casChunks, - desktop: desktop, - namespace: namespace, - buildHash: buildHash, - signature: signature - ) - -# ============================================================================ -# KDL Generation -# ============================================================================ - -proc generateNIPManifest*(manifest: NIPManifest): string = - ## Generate KDL manifest from NIPManifest - ## - ## **Requirements:** - ## - Requirement 4.2: Generate app name, version, permissions, namespace config, CAS chunks - ## - Requirement 4.3: Generate .desktop file, icons, MIME type associations - ## - Requirement 6.4: Deterministic generation (same input = same output) - ## - ## **Determinism:** Fields are output in a fixed order to ensure same input = same output - - result = "app \"" & manifest.name & "\" {\n" - - # Core identity - result.add(" version \"" & $manifest.version & "\"\n") - result.add(" build_date \"" & manifest.buildDate.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - result.add("\n") - - # Metadata section - result.add(" metadata {\n") - result.add(" description \"" & manifest.metadata.description & "\"\n") - result.add(" license \"" & manifest.metadata.license & "\"\n") - if manifest.metadata.homepage.isSome: - result.add(" homepage \"" & manifest.metadata.homepage.get() & "\"\n") - if manifest.metadata.author.isSome: - result.add(" author \"" & manifest.metadata.author.get() & "\"\n") - if manifest.metadata.maintainer.isSome: - result.add(" maintainer \"" & manifest.metadata.maintainer.get() & "\"\n") - if manifest.metadata.category.isSome: - result.add(" category \"" & manifest.metadata.category.get() & "\"\n") - if manifest.metadata.tags.len > 0: - result.add(" tags \"" & manifest.metadata.tags.join(" ") & "\"\n") - result.add(" }\n\n") - - # Provenance section - result.add(" provenance {\n") - result.add(" source \"" & manifest.provenance.source & "\"\n") - result.add(" source_hash \"" & manifest.provenance.sourceHash & "\"\n") - if manifest.provenance.upstream.isSome: - result.add(" upstream \"" & manifest.provenance.upstream.get() & "\"\n") - result.add(" build_timestamp \"" & manifest.provenance.buildTimestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - if manifest.provenance.builder.isSome: - result.add(" builder \"" & manifest.provenance.builder.get() & "\"\n") - result.add(" }\n\n") - - # Build configuration section - result.add(" build_config {\n") - if manifest.buildConfig.configureFlags.len > 0: - result.add(" configure_flags \"" & manifest.buildConfig.configureFlags.join(" ") & "\"\n") - if manifest.buildConfig.compilerFlags.len > 0: - result.add(" compiler_flags \"" & manifest.buildConfig.compilerFlags.join(" ") & "\"\n") - result.add(" compiler_version \"" & manifest.buildConfig.compilerVersion & "\"\n") - result.add(" target_architecture \"" & manifest.buildConfig.targetArchitecture & "\"\n") - result.add(" libc \"" & manifest.buildConfig.libc & "\"\n") - result.add(" allocator \"" & manifest.buildConfig.allocator & "\"\n") - result.add(" build_system \"" & manifest.buildConfig.buildSystem & "\"\n") - result.add(" }\n\n") - - # CAS chunks section - if manifest.casChunks.len > 0: - result.add(" cas_chunks {\n") - for chunk in manifest.casChunks: - result.add(" chunk \"" & chunk.hash & "\" {\n") - result.add(" size " & $chunk.size & "\n") - result.add(" type \"" & ($chunk.chunkType).toLowerAscii() & "\"\n") - result.add(" path \"" & chunk.path & "\"\n") - result.add(" }\n") - result.add(" }\n\n") - - # Desktop integration section - result.add(" desktop {\n") - result.add(" app_id \"" & manifest.desktop.appId & "\"\n\n") - - # Desktop file - result.add(" desktop_file {\n") - result.add(" name \"" & manifest.desktop.desktopFile.name & "\"\n") - if manifest.desktop.desktopFile.genericName.isSome: - result.add(" generic_name \"" & manifest.desktop.desktopFile.genericName.get() & "\"\n") - if manifest.desktop.desktopFile.comment.isSome: - result.add(" comment \"" & manifest.desktop.desktopFile.comment.get() & "\"\n") - result.add(" exec \"" & manifest.desktop.desktopFile.exec & "\"\n") - result.add(" icon \"" & manifest.desktop.desktopFile.icon & "\"\n") - result.add(" terminal " & $manifest.desktop.desktopFile.terminal & "\n") - if manifest.desktop.desktopFile.categories.len > 0: - result.add(" categories \"" & manifest.desktop.desktopFile.categories.join(";") & "\"\n") - if manifest.desktop.desktopFile.keywords.len > 0: - result.add(" keywords \"" & manifest.desktop.desktopFile.keywords.join(";") & "\"\n") - result.add(" }\n\n") - - # Icons - if manifest.desktop.icons.len > 0: - result.add(" icons {\n") - for icon in manifest.desktop.icons: - result.add(" icon {\n") - result.add(" size " & $icon.size & "\n") - result.add(" path \"" & icon.path & "\"\n") - result.add(" format \"" & icon.format & "\"\n") - result.add(" }\n") - result.add(" }\n\n") - - # MIME types - if manifest.desktop.mimeTypes.len > 0: - result.add(" mime_types \"" & manifest.desktop.mimeTypes.join(";") & "\"\n") - - result.add(" }\n\n") - - # Namespace configuration section - result.add(" namespace {\n") - result.add(" type \"" & manifest.namespace.namespaceType & "\"\n\n") - - # Permissions - result.add(" permissions {\n") - result.add(" network " & $manifest.namespace.permissions.network & "\n") - result.add(" gpu " & $manifest.namespace.permissions.gpu & "\n") - result.add(" audio " & $manifest.namespace.permissions.audio & "\n") - result.add(" camera " & $manifest.namespace.permissions.camera & "\n") - result.add(" microphone " & $manifest.namespace.permissions.microphone & "\n") - - # Filesystem access - if manifest.namespace.permissions.filesystem.len > 0: - result.add("\n filesystem {\n") - for fs in manifest.namespace.permissions.filesystem: - result.add(" access \"" & fs.path & "\" \"" & ($fs.mode).toLowerAscii() & "\"\n") - result.add(" }\n") - - # D-Bus access - if manifest.namespace.permissions.dbus.session.len > 0 or - manifest.namespace.permissions.dbus.system.len > 0 or - manifest.namespace.permissions.dbus.own.len > 0: - result.add("\n dbus {\n") - if manifest.namespace.permissions.dbus.session.len > 0: - result.add(" session \"" & manifest.namespace.permissions.dbus.session.join(" ") & "\"\n") - if manifest.namespace.permissions.dbus.system.len > 0: - result.add(" system \"" & manifest.namespace.permissions.dbus.system.join(" ") & "\"\n") - if manifest.namespace.permissions.dbus.own.len > 0: - result.add(" own \"" & manifest.namespace.permissions.dbus.own.join(" ") & "\"\n") - result.add(" }\n") - - result.add(" }\n") - - # Mounts - if manifest.namespace.mounts.len > 0: - result.add("\n mounts {\n") - for mount in manifest.namespace.mounts: - result.add(" mount {\n") - result.add(" source \"" & mount.source & "\"\n") - result.add(" target \"" & mount.target & "\"\n") - result.add(" type \"" & ($mount.mountType).toLowerAscii() & "\"\n") - result.add(" read_only " & $mount.readOnly & "\n") - result.add(" }\n") - result.add(" }\n") - - result.add(" }\n\n") - - # Build hash - result.add(" build_hash \"" & manifest.buildHash & "\"\n\n") - - # Signature - result.add(" signature {\n") - result.add(" algorithm \"" & manifest.signature.algorithm & "\"\n") - result.add(" key_id \"" & manifest.signature.keyId & "\"\n") - result.add(" signature \"" & manifest.signature.signature & "\"\n") - result.add(" }\n") - - result.add("}\n") - -# ============================================================================ -# Validation -# ============================================================================ - -proc validateNIPManifest*(manifest: NIPManifest): seq[string] = - ## Validate NIP manifest and return list of issues - ## - ## **Requirements:** - ## - Requirement 6.3: Validate all required fields and hash formats - ## - Requirement 4.2: Validate permissions and namespace config - - result = @[] - - # Validate name - if manifest.name.len == 0: - result.add("Application name cannot be empty") - - # Validate build hash format (xxh3-128) - if manifest.buildHash.len > 0 and not manifest.buildHash.startsWith("xxh3-"): - result.add("Build hash must use xxh3-128 format (xxh3-...)") - - # Validate source hash format - if manifest.provenance.sourceHash.len > 0 and not manifest.provenance.sourceHash.startsWith("xxh3-"): - result.add("Source hash must use xxh3-128 format (xxh3-...)") - - # Validate CAS chunks have xxh3 hashes - for chunk in manifest.casChunks: - if not chunk.hash.startsWith("xxh3-"): - result.add("Chunk hash must use xxh3-128 format (xxh3-...)") - if chunk.size <= 0: - result.add("Chunk size must be positive") - - # Validate desktop integration - if manifest.desktop.appId.len == 0: - result.add("Desktop app_id cannot be empty") - if manifest.desktop.desktopFile.name.len == 0: - result.add("Desktop file name cannot be empty") - if manifest.desktop.desktopFile.exec.len == 0: - result.add("Desktop file exec command cannot be empty") - - # Validate namespace type - if manifest.namespace.namespaceType notin ["user", "strict", "none"]: - result.add("Namespace type must be 'user', 'strict', or 'none'") - - # Validate signature - if manifest.signature.algorithm.len > 0 and manifest.signature.algorithm != "ed25519": - result.add("Signature algorithm must be 'ed25519'") - if manifest.signature.keyId.len == 0: - result.add("Signature key_id cannot be empty") - if manifest.signature.signature.len == 0: - result.add("Signature value cannot be empty") - -# ============================================================================ -# Convenience Functions -# ============================================================================ - -proc `$`*(manifest: NIPManifest): string = - ## Convert NIP manifest to human-readable string - result = "NIP Application: " & manifest.name & " v" & $manifest.version & "\n" - result.add("Build Date: " & manifest.buildDate.format("yyyy-MM-dd HH:mm:ss") & "\n") - result.add("License: " & manifest.metadata.license & "\n") - result.add("App ID: " & manifest.desktop.appId & "\n") - result.add("Build Hash: " & manifest.buildHash & "\n") - result.add("CAS Chunks: " & $manifest.casChunks.len & "\n") - result.add("Namespace: " & manifest.namespace.namespaceType & "\n") diff --git a/src/nip/npk.nim b/src/nip/npk.nim deleted file mode 100644 index db5e154..0000000 --- a/src/nip/npk.nim +++ /dev/null @@ -1,367 +0,0 @@ -## NPK Archive Handler -## -## **Purpose:** -## Handles .npk (Nexus Package Kit) archive creation and parsing. -## NPK packages are tar.zst archives containing manifest.kdl, metadata.json, -## CAS chunks, and Ed25519 signatures. -## -## **Design Principles:** -## - System packages installed to /Programs/App/Version/ -## - Content-addressable storage for deduplication -## - Atomic operations with rollback capability -## - Ed25519 signature verification -## -## **Requirements:** -## - Requirement 3.1: .npk contains manifest.kdl, metadata.json, CAS chunks, Ed25519 signature -## - Requirement 8.2: Use zstd --auto for archive compression -## -## **Archive Structure:** -## ``` -## package.npk (tar.zst) -## ├── manifest.kdl # Package metadata -## ├── metadata.json # Additional metadata -## ├── chunks/ # CAS chunks -## │ ├── xxh3-abc123.zst -## │ ├── xxh3-def456.zst -## │ └── ... -## └── signature.sig # Ed25519 signature -## ``` - -import std/[os, strutils, times, json, options, sequtils] -import nip/cas -import nip/xxh -import nip/npk_manifest -import nip/manifest_parser - -type - NPKPackage* = object - ## Complete NPK package with all components - manifest*: NPKManifest - metadata*: JsonNode - chunks*: seq[ChunkData] - signature*: string - archivePath*: string - - ChunkData* = object - ## Chunk data extracted from archive - hash*: string - data*: string - size*: int64 - chunkType*: ChunkType - - NPKError* = object of CatchableError - code*: NPKErrorCode - context*: string - suggestions*: seq[string] - - NPKErrorCode* = enum - ArchiveNotFound, - InvalidArchive, - ManifestMissing, - SignatureMissing, - ChunkMissing, - ExtractionFailed, - CompressionFailed, - InvalidFormat - -# ============================================================================ -# Archive Parsing -# ============================================================================ - -proc parseNPK*(path: string): NPKPackage = - ## Parse .npk archive and extract all components - ## - ## **Requirements:** - ## - Requirement 3.1: Extract manifest.kdl, metadata.json, CAS chunks, signature - ## - Requirement 8.2: Handle zstd --auto compressed archives - ## - ## **Process:** - ## 1. Verify archive exists and is readable - ## 2. Extract to temporary directory - ## 3. Parse manifest.kdl - ## 4. Parse metadata.json - ## 5. Load chunks from chunks/ directory - ## 6. Load signature from signature.sig - ## 7. Verify integrity - ## - ## **Raises:** - ## - NPKError if archive is invalid or missing components - - if not fileExists(path): - raise newException(NPKError, "NPK archive not found: " & path) - - # Create temporary extraction directory - let tempDir = getTempDir() / "npk-extract-" & $getTime().toUnix() - createDir(tempDir) - - try: - # Extract archive using tar with zstd decompression - # Using --auto-compress lets tar detect compression automatically - let extractCmd = "tar --auto-compress -xf " & quoteShell(path) & " -C " & - quoteShell(tempDir) - let extractResult = execShellCmd(extractCmd) - - if extractResult != 0: - raise newException(NPKError, "Failed to extract NPK archive: " & path) - - # Parse manifest.kdl - let manifestPath = tempDir / "manifest.kdl" - if not fileExists(manifestPath): - raise newException(NPKError, "manifest.kdl not found in archive") - - let manifestKdl = readFile(manifestPath) - let manifest = parseNPKManifest(manifestKdl) - - # Parse metadata.json - let metadataPath = tempDir / "metadata.json" - var metadata = newJObject() - if fileExists(metadataPath): - let metadataStr = readFile(metadataPath) - metadata = parseJson(metadataStr) - - # Load chunks from chunks/ directory - var chunks: seq[ChunkData] = @[] - let chunksDir = tempDir / "chunks" - if dirExists(chunksDir): - for chunkFile in walkFiles(chunksDir / "*.zst"): - let chunkName = extractFilename(chunkFile) - let chunkHash = chunkName.replace(".zst", "") - - # Read compressed chunk data - let chunkData = readFile(chunkFile) - - chunks.add(ChunkData( - hash: chunkHash, - data: chunkData, - size: chunkData.len.int64, - chunkType: Binary # Will be determined from manifest - )) - - # Load signature - let signaturePath = tempDir / "signature.sig" - var signature = "" - if fileExists(signaturePath): - signature = readFile(signaturePath) - - result = NPKPackage( - manifest: manifest, - metadata: metadata, - chunks: chunks, - signature: signature, - archivePath: path - ) - - finally: - # Clean up temporary directory - if dirExists(tempDir): - removeDir(tempDir) - -# ============================================================================ -# Archive Creation -# ============================================================================ - -proc createNPK*(manifest: NPKManifest, chunks: seq[ChunkData], - metadata: JsonNode, signature: string, - outputPath: string): NPKPackage = - ## Create .npk archive from components - ## - ## **Requirements:** - ## - Requirement 3.1: Package manifest.kdl, metadata.json, CAS chunks, signature - ## - Requirement 8.2: Use zstd --auto for archive compression - ## - ## **Process:** - ## 1. Create temporary staging directory - ## 2. Write manifest.kdl - ## 3. Write metadata.json - ## 4. Write chunks to chunks/ directory - ## 5. Write signature.sig - ## 6. Create tar.zst archive with --auto-compress - ## 7. Verify archive integrity - ## - ## **Returns:** - ## - NPKPackage with all components - ## - ## **Raises:** - ## - NPKError if creation fails - - # Create temporary staging directory - let tempDir = getTempDir() / "npk-create-" & $getTime().toUnix() - createDir(tempDir) - - try: - # Write manifest.kdl - let manifestKdl = generateNPKManifest(manifest) - writeFile(tempDir / "manifest.kdl", manifestKdl) - - # Write metadata.json - writeFile(tempDir / "metadata.json", $metadata) - - # Write chunks to chunks/ directory - let chunksDir = tempDir / "chunks" - createDir(chunksDir) - - for chunk in chunks: - let chunkPath = chunksDir / (chunk.hash & ".zst") - writeFile(chunkPath, chunk.data) - - # Write signature - writeFile(tempDir / "signature.sig", signature) - - # Create tar.zst archive - # Using --auto-compress lets tar choose optimal compression - let createCmd = "tar --auto-compress -cf " & quoteShell(outputPath) & - " -C " & quoteShell(tempDir) & " ." - let createResult = execShellCmd(createCmd) - - if createResult != 0: - raise newException(NPKError, "Failed to create NPK archive: " & outputPath) - - result = NPKPackage( - manifest: manifest, - metadata: metadata, - chunks: chunks, - signature: signature, - archivePath: outputPath - ) - - finally: - # Clean up temporary directory - if dirExists(tempDir): - removeDir(tempDir) - -# ============================================================================ -# Chunk Extraction -# ============================================================================ - -proc extractChunks*(pkg: NPKPackage, casRoot: string): seq[string] = - ## Extract chunks from NPK package to CAS - ## - ## **Requirements:** - ## - Requirement 3.1: Extract CAS chunks from archive - ## - Requirement 2.1: Store chunks with xxh3-128 hashing - ## - ## **Process:** - ## 1. For each chunk in package - ## 2. Decompress chunk data (if compressed) - ## 3. Calculate xxh3-128 hash - ## 4. Verify hash matches manifest - ## 5. Store in CAS with deduplication - ## 6. Return list of stored chunk hashes - ## - ## **Returns:** - ## - List of chunk hashes stored in CAS - ## - ## **Raises:** - ## - NPKError if chunk extraction or verification fails - - result = @[] - - for chunk in pkg.chunks: - # Decompress chunk data - # TODO: Implement zstd decompression when library available - let decompressedData = chunk.data - - # Calculate xxh3-128 hash - let calculatedHash = $calculateXxh3(decompressedData) - - # Verify hash matches manifest - let manifestChunk = pkg.manifest.casChunks.filterIt(it.hash == chunk.hash) - if manifestChunk.len == 0: - raise newException(NPKError, "Chunk not found in manifest: " & chunk.hash) - - if calculatedHash != chunk.hash: - raise newException(NPKError, - "Chunk hash mismatch: expected " & chunk.hash & ", got " & calculatedHash) - - # Store in CAS (will deduplicate automatically) - let casObject = storeObject(decompressedData, casRoot, compress = true) - - result.add(string(casObject.hash)) - -# ============================================================================ -# Verification -# ============================================================================ - -proc verifyNPK*(pkg: NPKPackage): bool = - ## Verify NPK package integrity - ## - ## **Requirements:** - ## - Requirement 3.4: Verify Ed25519 signature - ## - Requirement 2.2: Verify chunk integrity using xxh3 hash - ## - ## **Checks:** - ## 1. Manifest is valid - ## 2. All chunks referenced in manifest are present - ## 3. Chunk hashes match manifest - ## 4. Signature is valid (if present) - ## - ## **Returns:** - ## - true if package is valid, false otherwise - - # Validate manifest - let issues = validateNPKManifest(pkg.manifest) - if issues.len > 0: - return false - - # Verify all chunks are present - for manifestChunk in pkg.manifest.casChunks: - let found = pkg.chunks.anyIt(it.hash == manifestChunk.hash) - if not found: - return false - - # Verify chunk hashes - for chunk in pkg.chunks: - # TODO: Implement hash verification when xxh3 library available - discard - - # Verify signature - # TODO: Implement Ed25519 signature verification - if pkg.signature.len == 0: - return false - - result = true - -# ============================================================================ -# Utility Functions -# ============================================================================ - -proc listChunks*(pkg: NPKPackage): seq[string] = - ## List all chunk hashes in package - result = pkg.chunks.mapIt(it.hash) - -proc getChunk*(pkg: NPKPackage, hash: string): Option[ChunkData] = - ## Get chunk data by hash - for chunk in pkg.chunks: - if chunk.hash == hash: - return some(chunk) - return none(ChunkData) - -proc packageSize*(pkg: NPKPackage): int64 = - ## Calculate total package size (sum of all chunks) - result = 0 - for chunk in pkg.chunks: - result += chunk.size - -proc `$`*(pkg: NPKPackage): string = - ## Convert NPK package to human-readable string - result = "NPK Package: " & pkg.manifest.name & " v" & manifest_parser.`$`( - pkg.manifest.version) & "\n" - result.add("Archive: " & pkg.archivePath & "\n") - result.add("Chunks: " & $pkg.chunks.len & "\n") - result.add("Total Size: " & $(packageSize(pkg) div 1024) & " KB\n") - result.add("Signature: " & (if pkg.signature.len > - 0: "Present" else: "Missing") & "\n") - -# ============================================================================ -# Error Formatting -# ============================================================================ - -proc formatNPKError*(error: NPKError): string = - ## Format NPK error with context and suggestions - result = "❌ [" & $error.code & "] " & error.msg & "\n" - if error.context.len > 0: - result.add("🔍 Context: " & error.context & "\n") - if error.suggestions.len > 0: - result.add("💡 Suggestions:\n") - for suggestion in error.suggestions: - result.add(" • " & suggestion & "\n") diff --git a/src/nip/npk_installer.nim b/src/nip/npk_installer.nim deleted file mode 100644 index 262d2fc..0000000 --- a/src/nip/npk_installer.nim +++ /dev/null @@ -1,380 +0,0 @@ -## NPK Installation Workflow -## -## **Purpose:** -## Implements atomic installation workflow for .npk system packages. -## Handles chunk extraction to CAS, manifest creation, reference tracking, -## and rollback on failure. -## -## **Design Principles:** -## - Atomic operations (all-or-nothing) -## - Automatic rollback on failure -## - CAS deduplication -## - Reference tracking for garbage collection -## -## **Requirements:** -## - Requirement 3.5: Extract chunks to CAS and create manifest in ~/.local/share/nexus/npks/ -## - Requirement 11.1: Package installation SHALL be atomic (all-or-nothing) -## - Requirement 11.2: Installation failures SHALL rollback to previous state - -import std/[os, strutils, times, json, options] -import nip/[npk, npk_manifest, cas, unified_storage, manifest_parser] - -type - InstallResult* = object - ## Result of NPK installation - success*: bool - packageName*: string - version*: string - installPath*: string - chunksInstalled*: int - error*: string - - InstallError* = object of CatchableError - code*: InstallErrorCode - context*: string - suggestions*: seq[string] - - InstallErrorCode* = enum - PackageAlreadyInstalled, - InsufficientSpace, - PermissionDenied, - ChunkExtractionFailed, - ManifestCreationFailed, - RollbackFailed, - InvalidPackage - - InstallTransaction* = object - ## Transaction tracking for atomic installation - id*: string - packageName*: string - startTime*: times.Time - operations*: seq[InstallOperation] - completed*: bool - - InstallOperation* = object - ## Individual operation in installation transaction - kind*: OperationKind - path*: string - data*: string - timestamp*: times.Time - - OperationKind* = enum - CreateDirectory, - WriteFile, - CreateSymlink, - AddCASChunk, - AddReference - -# ============================================================================ -# Forward Declarations -# ============================================================================ - -proc rollbackInstallation*(transaction: InstallTransaction, storageRoot: string) - -# ============================================================================ -# Installation Workflow -# ============================================================================ - -proc installNPK*(pkgPath: string, storageRoot: string = ""): InstallResult = - ## Install NPK package atomically - ## - ## **Requirements:** - ## - Requirement 3.5: Extract chunks to CAS and create manifest - ## - Requirement 11.1: Atomic installation (all-or-nothing) - ## - Requirement 11.2: Rollback on failure - ## - ## **Process:** - ## 1. Parse NPK package - ## 2. Validate package integrity - ## 3. Check if already installed - ## 4. Create installation transaction - ## 5. Extract chunks to CAS with deduplication - ## 6. Create manifest in ~/.local/share/nexus/npks/ - ## 7. Add references to cas/refs/npks/ - ## 8. Commit transaction or rollback on failure - ## - ## **Returns:** - ## - InstallResult with success status and details - ## - ## **Raises:** - ## - InstallError if installation fails - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - - # Initialize result - result = InstallResult( - success: false, - packageName: "", - version: "", - installPath: "", - chunksInstalled: 0, - error: "" - ) - - # Create installation transaction - var transaction = InstallTransaction( - id: "install-" & $getTime().toUnix(), - packageName: "", - startTime: getTime(), - operations: @[], - completed: false - ) - - try: - # Step 1: Parse NPK package - let pkg = parseNPK(pkgPath) - transaction.packageName = pkg.manifest.name - result.packageName = pkg.manifest.name - result.version = manifest_parser.`$`(pkg.manifest.version) - - # Step 2: Validate package integrity - if not verifyNPK(pkg): - raise newException(InstallError, "Package verification failed") - - # Step 3: Check if already installed - let npksDir = root / "npks" - let manifestPath = npksDir / (pkg.manifest.name & ".kdl") - if fileExists(manifestPath): - result.error = "Package already installed" - raise newException(InstallError, "Package already installed: " & pkg.manifest.name) - - # Step 4: Create necessary directories - createDir(npksDir) - transaction.operations.add(InstallOperation( - kind: CreateDirectory, - path: npksDir, - data: "", - timestamp: getTime() - )) - - let casDir = root / "cas" - createDir(casDir) - createDir(casDir / "chunks") - createDir(casDir / "refs") - createDir(casDir / "refs" / "npks") - - # Step 5: Extract chunks to CAS with deduplication - let casRoot = casDir - var installedChunks: seq[string] = @[] - - for chunk in pkg.chunks: - # Store chunk in CAS (will deduplicate automatically) - let casObject = storeObject(chunk.data, casRoot / "chunks", compress = true) - installedChunks.add(string(casObject.hash)) - - transaction.operations.add(InstallOperation( - kind: AddCASChunk, - path: casRoot / "chunks" / string(casObject.hash), - data: string(casObject.hash), - timestamp: getTime() - )) - - result.chunksInstalled = installedChunks.len - - # Step 6: Create manifest in ~/.local/share/nexus/npks/ - let manifestKdl = generateNPKManifest(pkg.manifest) - writeFile(manifestPath, manifestKdl) - - transaction.operations.add(InstallOperation( - kind: WriteFile, - path: manifestPath, - data: manifestKdl, - timestamp: getTime() - )) - - result.installPath = manifestPath - - # Step 7: Add references to cas/refs/npks/ - let refsPath = casDir / "refs" / "npks" / (pkg.manifest.name & ".refs") - var refsContent = "# NPK Package References\n" - refsContent.add("package: " & pkg.manifest.name & "\n") - refsContent.add("version: " & result.version & "\n") - refsContent.add("installed: " & $getTime() & "\n") - refsContent.add("chunks:\n") - for chunkHash in installedChunks: - refsContent.add(" - " & chunkHash & "\n") - - writeFile(refsPath, refsContent) - - transaction.operations.add(InstallOperation( - kind: AddReference, - path: refsPath, - data: refsContent, - timestamp: getTime() - )) - - # Step 8: Commit transaction - transaction.completed = true - result.success = true - - except CatchableError as e: - # Rollback on failure - result.error = e.msg - result.success = false - - # Attempt rollback - try: - rollbackInstallation(transaction, root) - except: - # Rollback failed - log error but don't throw - result.error.add(" (Rollback also failed)") - -# ============================================================================ -# Rollback -# ============================================================================ - -proc rollbackInstallation*(transaction: InstallTransaction, storageRoot: string) = - ## Rollback installation transaction - ## - ## **Requirement 11.2:** Rollback to previous state on failure - ## - ## **Process:** - ## 1. Remove created files in reverse order - ## 2. Remove created directories if empty - ## 3. Remove CAS references - ## 4. Log rollback operations - ## - ## **Raises:** - ## - InstallError if rollback fails - - # Process operations in reverse order - for i in countdown(transaction.operations.len - 1, 0): - let op = transaction.operations[i] - - try: - case op.kind: - of WriteFile: - if fileExists(op.path): - removeFile(op.path) - - of CreateDirectory: - if dirExists(op.path): - # Only remove if empty - try: - removeDir(op.path) - except: - discard # Directory not empty, leave it - - of AddReference: - if fileExists(op.path): - removeFile(op.path) - - of AddCASChunk: - # Don't remove CAS chunks - they might be shared - # Garbage collection will clean them up later - discard - - of CreateSymlink: - if symlinkExists(op.path): - removeFile(op.path) - - except: - # Log error but continue rollback - discard - -# ============================================================================ -# Query Functions -# ============================================================================ - -proc isInstalled*(packageName: string, storageRoot: string = ""): bool = - ## Check if NPK package is installed - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let manifestPath = root / "npks" / (packageName & ".kdl") - result = fileExists(manifestPath) - -proc getInstalledVersion*(packageName: string, storageRoot: string = ""): Option[string] = - ## Get installed version of NPK package - if not isInstalled(packageName, storageRoot): - return none(string) - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let manifestPath = root / "npks" / (packageName & ".kdl") - - try: - let manifestKdl = readFile(manifestPath) - let manifest = parseNPKManifest(manifestKdl) - return some(manifest_parser.`$`(manifest.version)) - except: - return none(string) - -proc listInstalledPackages*(storageRoot: string = ""): seq[string] = - ## List all installed NPK packages - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - let npksDir = root / "npks" - - result = @[] - - if not dirExists(npksDir): - return result - - for file in walkFiles(npksDir / "*.kdl"): - let packageName = extractFilename(file).replace(".kdl", "") - result.add(packageName) - -# ============================================================================ -# Verification -# ============================================================================ - -proc verifyInstallation*(packageName: string, storageRoot: string = ""): bool = - ## Verify NPK package installation integrity - ## - ## **Checks:** - ## 1. Manifest exists - ## 2. All referenced chunks exist in CAS - ## 3. References file exists - ## - ## **Returns:** - ## - true if installation is valid, false otherwise - - let root = if storageRoot.len > 0: storageRoot else: getHomeDir() / ".local/share/nexus" - - # Check manifest exists - let manifestPath = root / "npks" / (packageName & ".kdl") - if not fileExists(manifestPath): - return false - - # Parse manifest - let manifestKdl = readFile(manifestPath) - let manifest = parseNPKManifest(manifestKdl) - - # Check all chunks exist in CAS - # Note: Chunks are stored with their calculated hash, which may differ from manifest hash - # For now, we just verify that the CAS directory exists and has some chunks - let casDir = root / "cas" / "chunks" - if not dirExists(casDir): - return false - - # Check references file exists - let refsPath = root / "cas" / "refs" / "npks" / (packageName & ".refs") - if not fileExists(refsPath): - return false - - result = true - -# ============================================================================ -# Error Formatting -# ============================================================================ - -proc formatInstallError*(error: InstallError): string = - ## Format installation error with context and suggestions - result = "❌ [" & $error.code & "] " & error.msg & "\n" - if error.context.len > 0: - result.add("🔍 Context: " & error.context & "\n") - if error.suggestions.len > 0: - result.add("💡 Suggestions:\n") - for suggestion in error.suggestions: - result.add(" • " & suggestion & "\n") - -# ============================================================================ -# Utility Functions -# ============================================================================ - -proc `$`*(installResult: InstallResult): string = - ## Convert install result to human-readable string - if installResult.success: - result = "✅ Successfully installed " & installResult.packageName & " v" & installResult.version & "\n" - result.add("📦 Chunks installed: " & $installResult.chunksInstalled & "\n") - result.add("📍 Manifest: " & installResult.installPath & "\n") - else: - result = "❌ Failed to install " & installResult.packageName & "\n" - result.add("⚠️ Error: " & installResult.error & "\n") diff --git a/src/nip/npk_manifest.nim b/src/nip/npk_manifest.nim deleted file mode 100644 index 9cb3f4f..0000000 --- a/src/nip/npk_manifest.nim +++ /dev/null @@ -1,663 +0,0 @@ -## NPK Manifest Schema - System Package Format -## -## **Purpose:** -## Defines the NPK (Nexus Package Kit) manifest schema for system packages. -## NPK packages are installed system-wide and managed by nexus. -## -## **Design Principles:** -## - Complete metadata for system packages -## - Build configuration tracking for reproducibility -## - Dependency resolution with build hashes -## - System integration (services, users, paths) -## - Ed25519 signature support -## -## **Requirements:** -## - Requirement 3.1: manifest.kdl, metadata.json, CAS chunks, Ed25519 signature -## - Requirement 3.2: package name, version, dependencies, build config, CAS chunk references -## - Requirement 6.2: KDL format with chunk references by xxh3 hash -## - Requirement 6.5: exact versions and build hashes for dependencies - -import std/[times, options, strutils] -import nip/manifest_parser -import nimpak/kdl_parser - -type - # ============================================================================ - # NPK-Specific Types - # ============================================================================ - - NPKManifest* = object - ## Complete NPK manifest for system packages - # Core identity (from base PackageManifest) - name*: string - version*: SemanticVersion - buildDate*: DateTime - - # Package metadata - metadata*: PackageInfo - provenance*: ProvenanceInfo - buildConfig*: BuildConfiguration - - # Dependencies with build hashes - dependencies*: seq[Dependency] - - # CAS chunk references - casChunks*: seq[ChunkReference] - - # Installation paths (GoboLinux-style) - install*: InstallPaths - - # System integration - system*: SystemIntegration - - # Integrity - buildHash*: string ## xxh3-128 hash of build configuration - signature*: SignatureInfo - - PackageInfo* = object - ## Package metadata - description*: string - homepage*: Option[string] - license*: string - author*: Option[string] - maintainer*: Option[string] - tags*: seq[string] - - ProvenanceInfo* = object - ## Complete provenance tracking - source*: string ## Source URL or repository - sourceHash*: string ## xxh3-128 hash of source - upstream*: Option[string] ## Upstream project URL - buildTimestamp*: DateTime - builder*: Option[string] ## Who built this package - - BuildConfiguration* = object - ## Build configuration for reproducibility - configureFlags*: seq[string] - compilerFlags*: seq[string] - compilerVersion*: string - targetArchitecture*: string - libc*: string ## musl, glibc - allocator*: string ## jemalloc, tcmalloc, default - buildSystem*: string ## cmake, meson, autotools, etc. - - Dependency* = object - ## Package dependency with build hash - name*: string - version*: string - buildHash*: string ## xxh3-128 hash of dependency's build config - optional*: bool - - ChunkReference* = object - ## Reference to a CAS chunk - hash*: string ## xxh3-128 hash - size*: int64 - chunkType*: ChunkType - path*: string ## Relative path in package - - ChunkType* = enum - ## Type of chunk content - Binary, Library, Runtime, Config, Data - - InstallPaths* = object - ## GoboLinux-style installation paths - programsPath*: string ## /Programs/App/Version/ - binPath*: string ## /Programs/App/Version/bin/ - libPath*: string ## /Programs/App/Version/lib/ - sharePath*: string ## /Programs/App/Version/share/ - etcPath*: string ## /Programs/App/Version/etc/ - - SystemIntegration* = object - ## System-level integration - services*: seq[ServiceSpec] - users*: seq[UserSpec] - groups*: seq[GroupSpec] - systemPaths*: seq[string] ## Paths to link into /System/Index/ - - ServiceSpec* = object - ## System service specification - name*: string - serviceType*: string ## systemd, dinit, etc. - enable*: bool - startOnBoot*: bool - - UserSpec* = object - ## System user specification - name*: string - uid*: Option[int] - system*: bool - shell*: string - home*: Option[string] - - GroupSpec* = object - ## System group specification - name*: string - gid*: Option[int] - system*: bool - - SignatureInfo* = object - ## Ed25519 signature information - algorithm*: string ## "ed25519" - keyId*: string - signature*: string ## Base64-encoded signature - - # ============================================================================ - # Error Types - # ============================================================================ - - NPKError* = object of CatchableError - code*: NPKErrorCode - context*: string - - NPKErrorCode* = enum - InvalidManifest, - MissingField, - InvalidHash, - InvalidSignature, - DependencyError - -# ============================================================================ -# KDL Parsing - Minimal implementation to expose gaps via tests -# ============================================================================ - -proc parseNPKManifest*(kdl: string): NPKManifest = - ## Parse NPK manifest from KDL format - ## - ## **Requirements:** - ## - Requirement 3.2: Parse package name, version, dependencies, build config, CAS chunks - ## - Requirement 6.2: Validate chunk references by xxh3 hash - ## - Requirement 6.5: Parse exact versions and build hashes for dependencies - - # Simple line-based parser for the KDL format we generate - # This works because we control the generation format - - var lines = kdl.splitLines() - var name = "" - var version = SemanticVersion(major: 0, minor: 0, patch: 0) - var buildDate = now() - var buildHash = "" - - var metadata = PackageInfo(description: "", license: "", tags: @[]) - var provenance = ProvenanceInfo(source: "", sourceHash: "", buildTimestamp: now()) - var buildConfig = BuildConfiguration( - configureFlags: @[], compilerFlags: @[], - compilerVersion: "", targetArchitecture: "", - libc: "", allocator: "", buildSystem: "" - ) - var dependencies: seq[Dependency] = @[] - var casChunks: seq[ChunkReference] = @[] - var install = InstallPaths( - programsPath: "", binPath: "", libPath: "", sharePath: "", etcPath: "" - ) - var system = SystemIntegration( - services: @[], users: @[], groups: @[], systemPaths: @[] - ) - var signature = SignatureInfo(algorithm: "", keyId: "", signature: "") - - # Helper to extract quoted string - proc extractQuoted(line: string): string = - let start = line.find("\"") - if start >= 0: - let endIdx = line.find("\"", start + 1) - if endIdx > start: - return line[start+1..= 0: - coreVersion = vstr[0..= 0: - coreVersion = vstr[0..= 3: - version = SemanticVersion( - major: parseInt(parts[0]), - minor: parseInt(parts[1]), - patch: parseInt(parts[2]), - prerelease: "", - build: "" - ) - - # Parse prerelease if present - if dashIdx >= 0: - let endIdx = if plusIdx >= 0: plusIdx else: vstr.len - version.prerelease = vstr[dashIdx+1..= 0: - version.build = vstr[plusIdx+1..^1] - - elif line.startsWith("build_date \""): - let dateStr = extractQuoted(line) - try: - buildDate = parse(dateStr, "yyyy-MM-dd'T'HH:mm:ss'Z'") - except: - buildDate = now() - - elif line.startsWith("build_hash \""): - buildHash = extractQuoted(line) - - # Track sections - elif line == "metadata {": - currentSection = "metadata" - elif line == "provenance {": - currentSection = "provenance" - elif line == "build_config {": - currentSection = "build_config" - elif line == "dependencies {": - currentSection = "dependencies" - elif line == "cas_chunks {": - currentSection = "cas_chunks" - elif line == "install {": - currentSection = "install" - elif line == "system {": - currentSection = "system" - elif line == "signature {": - currentSection = "signature" - - # Parse section content - elif currentSection == "metadata": - if line.startsWith("description \""): - metadata.description = extractQuoted(line) - elif line.startsWith("license \""): - metadata.license = extractQuoted(line) - elif line.startsWith("homepage \""): - metadata.homepage = some(extractQuoted(line)) - elif line.startsWith("author \""): - metadata.author = some(extractQuoted(line)) - elif line.startsWith("maintainer \""): - metadata.maintainer = some(extractQuoted(line)) - elif line.startsWith("tags \""): - let tagsStr = extractQuoted(line) - metadata.tags = tagsStr.split() - - elif currentSection == "provenance": - if line.startsWith("source \""): - provenance.source = extractQuoted(line) - elif line.startsWith("source_hash \""): - provenance.sourceHash = extractQuoted(line) - elif line.startsWith("upstream \""): - provenance.upstream = some(extractQuoted(line)) - elif line.startsWith("build_timestamp \""): - let dateStr = extractQuoted(line) - try: - provenance.buildTimestamp = parse(dateStr, "yyyy-MM-dd'T'HH:mm:ss'Z'") - except: - provenance.buildTimestamp = now() - elif line.startsWith("builder \""): - provenance.builder = some(extractQuoted(line)) - - elif currentSection == "build_config": - if line.startsWith("configure_flags \""): - let flagsStr = extractQuoted(line) - buildConfig.configureFlags = flagsStr.split() - elif line.startsWith("compiler_flags \""): - let flagsStr = extractQuoted(line) - buildConfig.compilerFlags = flagsStr.split() - elif line.startsWith("compiler_version \""): - buildConfig.compilerVersion = extractQuoted(line) - elif line.startsWith("target_architecture \""): - buildConfig.targetArchitecture = extractQuoted(line) - elif line.startsWith("libc \""): - buildConfig.libc = extractQuoted(line) - elif line.startsWith("allocator \""): - buildConfig.allocator = extractQuoted(line) - elif line.startsWith("build_system \""): - buildConfig.buildSystem = extractQuoted(line) - - elif currentSection == "dependencies": - if line.startsWith("dependency \""): - currentDep = Dependency(name: extractQuoted(line), version: "", buildHash: "", optional: false) - elif line.startsWith("version \"") and currentDep.name.len > 0: - currentDep.version = extractQuoted(line) - elif line.startsWith("build_hash \"") and currentDep.name.len > 0: - currentDep.buildHash = extractQuoted(line) - elif line.startsWith("optional ") and currentDep.name.len > 0: - currentDep.optional = extractBool(line) - elif line == "}": - if currentDep.name.len > 0: - dependencies.add(currentDep) - currentDep = Dependency(name: "", version: "", buildHash: "", optional: false) - skipSectionReset = true # Don't reset section, we're still in dependencies - - elif currentSection == "cas_chunks": - if line.startsWith("chunk \""): - currentChunk = ChunkReference(hash: extractQuoted(line), size: 0, chunkType: Binary, path: "") - elif line.startsWith("size "): - currentChunk.size = extractInt(line).int64 - elif line.startsWith("type \""): - let typeStr = extractQuoted(line) - case typeStr: - of "binary": currentChunk.chunkType = Binary - of "library": currentChunk.chunkType = Library - of "runtime": currentChunk.chunkType = Runtime - of "config": currentChunk.chunkType = Config - of "data": currentChunk.chunkType = Data - else: currentChunk.chunkType = Binary - elif line.startsWith("path \""): - currentChunk.path = extractQuoted(line) - elif line == "}": - if currentChunk.hash.len > 0: - casChunks.add(currentChunk) - currentChunk = ChunkReference(hash: "", size: 0, chunkType: Binary, path: "") - skipSectionReset = true # Don't reset section, we're still in cas_chunks - - elif currentSection == "install": - if line.startsWith("programs_path \""): - install.programsPath = extractQuoted(line) - elif line.startsWith("bin_path \""): - install.binPath = extractQuoted(line) - elif line.startsWith("lib_path \""): - install.libPath = extractQuoted(line) - elif line.startsWith("share_path \""): - install.sharePath = extractQuoted(line) - elif line.startsWith("etc_path \""): - install.etcPath = extractQuoted(line) - - elif currentSection == "system": - if line.startsWith("service \""): - currentService = ServiceSpec(name: extractQuoted(line), serviceType: "", enable: false, startOnBoot: false) - elif line.startsWith("type \""): - currentService.serviceType = extractQuoted(line) - elif line.startsWith("enable "): - currentService.enable = extractBool(line) - elif line.startsWith("start_on_boot "): - currentService.startOnBoot = extractBool(line) - elif line.startsWith("user \""): - currentUser = UserSpec(name: extractQuoted(line), uid: none(int), system: false, shell: "", home: none(string)) - elif line.startsWith("uid "): - currentUser.uid = some(extractInt(line)) - elif line.startsWith("system "): - if currentUser.name.len > 0: - currentUser.system = extractBool(line) - elif currentGroup.name.len > 0: - currentGroup.system = extractBool(line) - elif line.startsWith("shell \""): - currentUser.shell = extractQuoted(line) - elif line.startsWith("home \""): - currentUser.home = some(extractQuoted(line)) - elif line.startsWith("group \""): - currentGroup = GroupSpec(name: extractQuoted(line), gid: none(int), system: false) - elif line.startsWith("gid "): - currentGroup.gid = some(extractInt(line)) - elif line == "}": - if currentService.name.len > 0: - system.services.add(currentService) - currentService = ServiceSpec(name: "", serviceType: "", enable: false, startOnBoot: false) - skipSectionReset = true - elif currentUser.name.len > 0: - system.users.add(currentUser) - currentUser = UserSpec(name: "", uid: none(int), system: false, shell: "", home: none(string)) - skipSectionReset = true - elif currentGroup.name.len > 0: - system.groups.add(currentGroup) - currentGroup = GroupSpec(name: "", gid: none(int), system: false) - skipSectionReset = true - - elif currentSection == "signature": - if line.startsWith("algorithm \""): - signature.algorithm = extractQuoted(line) - elif line.startsWith("key_id \""): - signature.keyId = extractQuoted(line) - elif line.startsWith("signature \""): - signature.signature = extractQuoted(line) - - # Reset section on closing brace (unless we just processed a nested block) - if line == "}" and currentSection != "" and not skipSectionReset: - if currentSection in ["metadata", "provenance", "build_config", "dependencies", "cas_chunks", "install", "system", "signature"]: - currentSection = "" - - # Reset the skip flag for next iteration - skipSectionReset = false - - i += 1 - - result = NPKManifest( - name: name, - version: version, - buildDate: buildDate, - metadata: metadata, - provenance: provenance, - buildConfig: buildConfig, - dependencies: dependencies, - casChunks: casChunks, - install: install, - system: system, - buildHash: buildHash, - signature: signature - ) - -# ============================================================================ -# KDL Generation - Minimal implementation to expose gaps via tests -# ============================================================================ - -proc generateNPKManifest*(manifest: NPKManifest): string = - ## Generate KDL manifest from NPKManifest - ## - ## **Requirements:** - ## - Requirement 3.2: Generate package name, version, dependencies, build config, CAS chunks - ## - Requirement 6.4: Deterministic generation (same input = same output) - ## - ## **Determinism:** Fields are output in a fixed order to ensure same input = same output - - result = "package \"" & manifest.name & "\" {\n" - - # Core identity - result.add(" version \"" & $manifest.version & "\"\n") - result.add(" build_date \"" & manifest.buildDate.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - result.add("\n") - - # Metadata section - result.add(" metadata {\n") - result.add(" description \"" & manifest.metadata.description & "\"\n") - result.add(" license \"" & manifest.metadata.license & "\"\n") - if manifest.metadata.homepage.isSome: - result.add(" homepage \"" & manifest.metadata.homepage.get() & "\"\n") - if manifest.metadata.author.isSome: - result.add(" author \"" & manifest.metadata.author.get() & "\"\n") - if manifest.metadata.maintainer.isSome: - result.add(" maintainer \"" & manifest.metadata.maintainer.get() & "\"\n") - if manifest.metadata.tags.len > 0: - result.add(" tags \"" & manifest.metadata.tags.join(" ") & "\"\n") - result.add(" }\n\n") - - # Provenance section - result.add(" provenance {\n") - result.add(" source \"" & manifest.provenance.source & "\"\n") - result.add(" source_hash \"" & manifest.provenance.sourceHash & "\"\n") - if manifest.provenance.upstream.isSome: - result.add(" upstream \"" & manifest.provenance.upstream.get() & "\"\n") - result.add(" build_timestamp \"" & manifest.provenance.buildTimestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'") & "\"\n") - if manifest.provenance.builder.isSome: - result.add(" builder \"" & manifest.provenance.builder.get() & "\"\n") - result.add(" }\n\n") - - # Build configuration section - result.add(" build_config {\n") - if manifest.buildConfig.configureFlags.len > 0: - result.add(" configure_flags \"" & manifest.buildConfig.configureFlags.join(" ") & "\"\n") - if manifest.buildConfig.compilerFlags.len > 0: - result.add(" compiler_flags \"" & manifest.buildConfig.compilerFlags.join(" ") & "\"\n") - result.add(" compiler_version \"" & manifest.buildConfig.compilerVersion & "\"\n") - result.add(" target_architecture \"" & manifest.buildConfig.targetArchitecture & "\"\n") - result.add(" libc \"" & manifest.buildConfig.libc & "\"\n") - result.add(" allocator \"" & manifest.buildConfig.allocator & "\"\n") - result.add(" build_system \"" & manifest.buildConfig.buildSystem & "\"\n") - result.add(" }\n\n") - - # Dependencies section - if manifest.dependencies.len > 0: - result.add(" dependencies {\n") - for dep in manifest.dependencies: - result.add(" dependency \"" & dep.name & "\" {\n") - result.add(" version \"" & dep.version & "\"\n") - result.add(" build_hash \"" & dep.buildHash & "\"\n") - if dep.optional: - result.add(" optional true\n") - result.add(" }\n") - result.add(" }\n\n") - - # CAS chunks section - if manifest.casChunks.len > 0: - result.add(" cas_chunks {\n") - for chunk in manifest.casChunks: - result.add(" chunk \"" & chunk.hash & "\" {\n") - result.add(" size " & $chunk.size & "\n") - result.add(" type \"" & ($chunk.chunkType).toLowerAscii() & "\"\n") - result.add(" path \"" & chunk.path & "\"\n") - result.add(" }\n") - result.add(" }\n\n") - - # Install paths section - result.add(" install {\n") - result.add(" programs_path \"" & manifest.install.programsPath & "\"\n") - result.add(" bin_path \"" & manifest.install.binPath & "\"\n") - result.add(" lib_path \"" & manifest.install.libPath & "\"\n") - result.add(" share_path \"" & manifest.install.sharePath & "\"\n") - result.add(" etc_path \"" & manifest.install.etcPath & "\"\n") - result.add(" }\n\n") - - # System integration section - if manifest.system.services.len > 0 or manifest.system.users.len > 0 or manifest.system.groups.len > 0: - result.add(" system {\n") - - # Services - for service in manifest.system.services: - result.add(" service \"" & service.name & "\" {\n") - result.add(" type \"" & service.serviceType & "\"\n") - result.add(" enable " & $service.enable & "\n") - result.add(" start_on_boot " & $service.startOnBoot & "\n") - result.add(" }\n") - - # Users - for user in manifest.system.users: - result.add(" user \"" & user.name & "\" {\n") - if user.uid.isSome: - result.add(" uid " & $user.uid.get() & "\n") - result.add(" system " & $user.system & "\n") - result.add(" shell \"" & user.shell & "\"\n") - if user.home.isSome: - result.add(" home \"" & user.home.get() & "\"\n") - result.add(" }\n") - - # Groups - for group in manifest.system.groups: - result.add(" group \"" & group.name & "\" {\n") - if group.gid.isSome: - result.add(" gid " & $group.gid.get() & "\n") - result.add(" system " & $group.system & "\n") - result.add(" }\n") - - result.add(" }\n\n") - - # Build hash - result.add(" build_hash \"" & manifest.buildHash & "\"\n\n") - - # Signature - result.add(" signature {\n") - result.add(" algorithm \"" & manifest.signature.algorithm & "\"\n") - result.add(" key_id \"" & manifest.signature.keyId & "\"\n") - result.add(" signature \"" & manifest.signature.signature & "\"\n") - result.add(" }\n") - - result.add("}\n") - -# ============================================================================ -# Validation -# ============================================================================ - -proc validateNPKManifest*(manifest: NPKManifest): seq[string] = - ## Validate NPK manifest and return list of issues - ## - ## **Requirements:** - ## - Requirement 6.3: Validate all required fields and hash formats - ## - Requirement 6.5: Validate exact versions and build hashes for dependencies - - result = @[] - - # Validate name - if manifest.name.len == 0: - result.add("Package name cannot be empty") - - # Validate build hash format (xxh3-128) - if manifest.buildHash.len > 0 and not manifest.buildHash.startsWith("xxh3-"): - result.add("Build hash must use xxh3-128 format (xxh3-...)") - - # Validate source hash format - if manifest.provenance.sourceHash.len > 0 and not manifest.provenance.sourceHash.startsWith("xxh3-"): - result.add("Source hash must use xxh3-128 format (xxh3-...)") - - # Validate dependencies have build hashes - for dep in manifest.dependencies: - if dep.buildHash.len == 0: - result.add("Dependency '" & dep.name & "' missing build hash") - elif not dep.buildHash.startsWith("xxh3-"): - result.add("Dependency '" & dep.name & "' build hash must use xxh3-128 format") - - # Validate CAS chunks have xxh3 hashes - for chunk in manifest.casChunks: - if not chunk.hash.startsWith("xxh3-"): - result.add("Chunk hash must use xxh3-128 format (xxh3-...)") - if chunk.size <= 0: - result.add("Chunk size must be positive") - - # Validate signature - if manifest.signature.algorithm.len > 0 and manifest.signature.algorithm != "ed25519": - result.add("Signature algorithm must be 'ed25519'") - if manifest.signature.keyId.len == 0: - result.add("Signature key_id cannot be empty") - if manifest.signature.signature.len == 0: - result.add("Signature value cannot be empty") - -# ============================================================================ -# Convenience Functions -# ============================================================================ - -proc `$`*(manifest: NPKManifest): string = - ## Convert NPK manifest to human-readable string - result = "NPK Package: " & manifest.name & " v" & $manifest.version & "\n" - result.add("Build Date: " & manifest.buildDate.format("yyyy-MM-dd HH:mm:ss") & "\n") - result.add("License: " & manifest.metadata.license & "\n") - result.add("Build Hash: " & manifest.buildHash & "\n") - result.add("Dependencies: " & $manifest.dependencies.len & "\n") - result.add("CAS Chunks: " & $manifest.casChunks.len & "\n") diff --git a/src/nip/package_metadata.nim b/src/nip/package_metadata.nim deleted file mode 100644 index a27599d..0000000 --- a/src/nip/package_metadata.nim +++ /dev/null @@ -1,356 +0,0 @@ -## Package Metadata (metadata.json) - Provenance Tracking -## -## **Purpose:** -## Defines the metadata.json format for complete provenance tracking across all package formats. -## This provides the audit trail from source to installation. -## -## **Design Principles:** -## - Complete provenance chain (source → build → installation) -## - Format-agnostic (works for NPK, NIP, NEXTER) -## - JSON format for machine readability -## - Cryptographic integrity (xxh3 for builds, Ed25519 for signatures) -## -## **Requirements:** -## - Requirement 7.1: source origin, maintainer, upstream URL, build timestamp -## - Requirement 7.2: compiler version, flags, target architecture, build hash -## - Requirement 7.3: complete chain from source to installation -## - Requirement 7.4: full audit trail -## - Requirement 7.5: xxh3 for build hashes, Ed25519 for signatures - -import std/[json, times, options, tables, strutils] -import nip/manifest_parser - -type - # ============================================================================ - # Package Metadata Types - # ============================================================================ - - PackageMetadata* = object - ## Complete package metadata for provenance tracking - # Format identification - formatType*: string ## "npk", "nip", or "nexter" - formatVersion*: string ## Metadata format version - - # Package identity - name*: string - version*: string - description*: string - license*: string - - # Source provenance - source*: SourceProvenance - - # Build provenance - build*: BuildProvenance - - # Installation provenance - installation*: InstallationProvenance - - # Integrity hashes - hashes*: IntegrityHashes - - # Signatures - signatures*: seq[SignatureRecord] - - # Additional metadata - tags*: seq[string] - metadata*: Table[string, string] ## Extensible metadata - - SourceProvenance* = object - ## Source code provenance - origin*: string ## Source URL or repository - sourceHash*: string ## xxh3-128 hash of source - upstream*: Option[string] ## Upstream project URL - upstreamVersion*: Option[string] ## Upstream version - fetchedAt*: DateTime ## When source was fetched - fetchMethod*: string ## "http", "git", "local", etc. - - BuildProvenance* = object - ## Build process provenance - buildTimestamp*: DateTime - builder*: string ## Who/what built this package - buildHost*: string ## Hostname where built - buildEnvironment*: BuildEnvironment - buildDuration*: Option[int] ## Build time in seconds - - BuildEnvironment* = object - ## Build environment details - compilerVersion*: string - compilerFlags*: seq[string] - configureFlags*: seq[string] - targetArchitecture*: string - libc*: string - allocator*: string - buildSystem*: string - environmentVars*: Table[string, string] ## Relevant env vars - - InstallationProvenance* = object - ## Installation provenance - installedAt*: DateTime - installedBy*: string ## User who installed - installPath*: string ## Installation path - installMethod*: string ## "nip install", "nip graft", etc. - installHost*: string ## Hostname where installed - - IntegrityHashes* = object - ## Cryptographic hashes for integrity - sourceHash*: string ## xxh3-128 of source - buildHash*: string ## xxh3-128 of build configuration - artifactHash*: string ## xxh3-128 of final artifact - manifestHash*: string ## xxh3-128 of manifest.kdl - - SignatureRecord* = object - ## Signature information - algorithm*: string ## "ed25519" - keyId*: string - signature*: string ## Base64-encoded signature - signedBy*: string ## Signer identity - signedAt*: DateTime - -# ============================================================================ -# JSON Generation -# ============================================================================ - -proc toJson*(metadata: PackageMetadata): JsonNode = - ## Convert PackageMetadata to JSON - ## - ## **Requirements:** - ## - Requirement 7.1: Include source origin, maintainer, upstream URL, build timestamp - ## - Requirement 7.2: Include compiler version, flags, target architecture, build hash - ## - Requirement 7.3: Record complete chain from source to installation - - result = %* { - "format_type": metadata.formatType, - "format_version": metadata.formatVersion, - "name": metadata.name, - "version": metadata.version, - "description": metadata.description, - "license": metadata.license, - "tags": metadata.tags, - - "source_provenance": { - "origin": metadata.source.origin, - "source_hash": metadata.source.sourceHash, - "fetched_at": metadata.source.fetchedAt.format("yyyy-MM-dd'T'HH:mm:ss'Z'"), - "fetch_method": metadata.source.fetchMethod - }, - - "build_provenance": { - "build_timestamp": metadata.build.buildTimestamp.format("yyyy-MM-dd'T'HH:mm:ss'Z'"), - "builder": metadata.build.builder, - "build_host": metadata.build.buildHost, - "build_environment": { - "compiler_version": metadata.build.buildEnvironment.compilerVersion, - "compiler_flags": metadata.build.buildEnvironment.compilerFlags, - "configure_flags": metadata.build.buildEnvironment.configureFlags, - "target_architecture": metadata.build.buildEnvironment.targetArchitecture, - "libc": metadata.build.buildEnvironment.libc, - "allocator": metadata.build.buildEnvironment.allocator, - "build_system": metadata.build.buildEnvironment.buildSystem - } - }, - - "installation_provenance": { - "installed_at": metadata.installation.installedAt.format("yyyy-MM-dd'T'HH:mm:ss'Z'"), - "installed_by": metadata.installation.installedBy, - "install_path": metadata.installation.installPath, - "install_method": metadata.installation.installMethod, - "install_host": metadata.installation.installHost - }, - - "integrity_hashes": { - "source_hash": metadata.hashes.sourceHash, - "build_hash": metadata.hashes.buildHash, - "artifact_hash": metadata.hashes.artifactHash, - "manifest_hash": metadata.hashes.manifestHash - }, - - "signatures": newJArray() - } - - # Add optional fields - if metadata.source.upstream.isSome: - result["source_provenance"]["upstream"] = %metadata.source.upstream.get() - if metadata.source.upstreamVersion.isSome: - result["source_provenance"]["upstream_version"] = %metadata.source.upstreamVersion.get() - if metadata.build.buildDuration.isSome: - result["build_provenance"]["build_duration_seconds"] = %metadata.build.buildDuration.get() - - # Add environment variables if present - if metadata.build.buildEnvironment.environmentVars.len > 0: - result["build_provenance"]["build_environment"]["environment_vars"] = newJObject() - for key, val in metadata.build.buildEnvironment.environmentVars: - result["build_provenance"]["build_environment"]["environment_vars"][key] = %val - - # Add signatures - for sig in metadata.signatures: - result["signatures"].add(%* { - "algorithm": sig.algorithm, - "key_id": sig.keyId, - "signature": sig.signature, - "signed_by": sig.signedBy, - "signed_at": sig.signedAt.format("yyyy-MM-dd'T'HH:mm:ss'Z'") - }) - - # Add extensible metadata - if metadata.metadata.len > 0: - result["metadata"] = newJObject() - for key, val in metadata.metadata: - result["metadata"][key] = %val - -proc generateMetadataJson*(metadata: PackageMetadata): string = - ## Generate JSON string from PackageMetadata - ## - ## **Requirements:** - ## - Requirement 7.4: Provide full audit trail - ## - Requirement 7.5: Use xxh3 for build hashes, Ed25519 for signatures - - let jsonNode = metadata.toJson() - result = jsonNode.pretty(indent = 2) - -# ============================================================================ -# JSON Parsing -# ============================================================================ - -proc parseMetadataJson*(jsonStr: string): PackageMetadata = - ## Parse metadata.json from JSON string - ## - ## **Requirements:** - ## - Requirement 7.3: Parse complete chain from source to installation - - let json = parseJson(jsonStr) - - # Parse source provenance - let sourceProv = json["source_provenance"] - var source = SourceProvenance( - origin: sourceProv["origin"].getStr(), - sourceHash: sourceProv["source_hash"].getStr(), - fetchedAt: parse(sourceProv["fetched_at"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'"), - fetchMethod: sourceProv["fetch_method"].getStr() - ) - if sourceProv.hasKey("upstream"): - source.upstream = some(sourceProv["upstream"].getStr()) - if sourceProv.hasKey("upstream_version"): - source.upstreamVersion = some(sourceProv["upstream_version"].getStr()) - - # Parse build provenance - let buildProv = json["build_provenance"] - let buildEnv = buildProv["build_environment"] - - var envVars = initTable[string, string]() - if buildEnv.hasKey("environment_vars"): - for key, val in buildEnv["environment_vars"]: - envVars[key] = val.getStr() - - var build = BuildProvenance( - buildTimestamp: parse(buildProv["build_timestamp"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'"), - builder: buildProv["builder"].getStr(), - buildHost: buildProv["build_host"].getStr(), - buildEnvironment: BuildEnvironment( - compilerVersion: buildEnv["compiler_version"].getStr(), - compilerFlags: buildEnv["compiler_flags"].to(seq[string]), - configureFlags: buildEnv["configure_flags"].to(seq[string]), - targetArchitecture: buildEnv["target_architecture"].getStr(), - libc: buildEnv["libc"].getStr(), - allocator: buildEnv["allocator"].getStr(), - buildSystem: buildEnv["build_system"].getStr(), - environmentVars: envVars - ) - ) - if buildProv.hasKey("build_duration_seconds"): - build.buildDuration = some(buildProv["build_duration_seconds"].getInt()) - - # Parse installation provenance - let installProv = json["installation_provenance"] - let installation = InstallationProvenance( - installedAt: parse(installProv["installed_at"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'"), - installedBy: installProv["installed_by"].getStr(), - installPath: installProv["install_path"].getStr(), - installMethod: installProv["install_method"].getStr(), - installHost: installProv["install_host"].getStr() - ) - - # Parse integrity hashes - let hashesJson = json["integrity_hashes"] - let hashes = IntegrityHashes( - sourceHash: hashesJson["source_hash"].getStr(), - buildHash: hashesJson["build_hash"].getStr(), - artifactHash: hashesJson["artifact_hash"].getStr(), - manifestHash: hashesJson["manifest_hash"].getStr() - ) - - # Parse signatures - var signatures: seq[SignatureRecord] = @[] - for sigJson in json["signatures"]: - signatures.add(SignatureRecord( - algorithm: sigJson["algorithm"].getStr(), - keyId: sigJson["key_id"].getStr(), - signature: sigJson["signature"].getStr(), - signedBy: sigJson["signed_by"].getStr(), - signedAt: parse(sigJson["signed_at"].getStr(), "yyyy-MM-dd'T'HH:mm:ss'Z'") - )) - - # Parse extensible metadata - var metadataTable = initTable[string, string]() - if json.hasKey("metadata"): - for key, val in json["metadata"]: - metadataTable[key] = val.getStr() - - result = PackageMetadata( - formatType: json["format_type"].getStr(), - formatVersion: json["format_version"].getStr(), - name: json["name"].getStr(), - version: json["version"].getStr(), - description: json["description"].getStr(), - license: json["license"].getStr(), - tags: json["tags"].to(seq[string]), - source: source, - build: build, - installation: installation, - hashes: hashes, - signatures: signatures, - metadata: metadataTable - ) - -# ============================================================================ -# Validation -# ============================================================================ - -proc validateMetadata*(metadata: PackageMetadata): seq[string] = - ## Validate package metadata and return list of issues - ## - ## **Requirements:** - ## - Requirement 7.5: Validate xxh3 for build hashes, Ed25519 for signatures - - result = @[] - - # Validate format type - if metadata.formatType notin ["npk", "nip", "nexter"]: - result.add("Format type must be 'npk', 'nip', or 'nexter'") - - # Validate hashes use xxh3 - if not metadata.hashes.sourceHash.startsWith("xxh3-"): - result.add("Source hash must use xxh3-128 format") - if not metadata.hashes.buildHash.startsWith("xxh3-"): - result.add("Build hash must use xxh3-128 format") - if not metadata.hashes.artifactHash.startsWith("xxh3-"): - result.add("Artifact hash must use xxh3-128 format") - if not metadata.hashes.manifestHash.startsWith("xxh3-"): - result.add("Manifest hash must use xxh3-128 format") - - # Validate signatures use Ed25519 - for sig in metadata.signatures: - if sig.algorithm != "ed25519": - result.add("Signature algorithm must be 'ed25519'") - -# ============================================================================ -# Convenience Functions -# ============================================================================ - -proc `$`*(metadata: PackageMetadata): string = - ## Convert metadata to human-readable string - result = "Package: " & metadata.name & " v" & metadata.version & "\n" - result.add("Format: " & metadata.formatType & "\n") - result.add("Source: " & metadata.source.origin & "\n") - result.add("Built by: " & metadata.build.builder & " on " & metadata.build.buildHost & "\n") - result.add("Installed: " & metadata.installation.installPath & "\n") diff --git a/src/nip/platform.nim b/src/nip/platform.nim deleted file mode 100644 index 6511b7f..0000000 --- a/src/nip/platform.nim +++ /dev/null @@ -1,573 +0,0 @@ -## Platform Detection and Isolation Strategy Selection -## -## This module provides runtime detection of OS capabilities and selection of -## appropriate isolation strategies for multi-platform support. -## -## Core Philosophy: -## - Detect, don't assume -## - Graceful degradation when advanced features unavailable -## - Platform-native solutions for each OS -## - No false security - be honest about what each strategy provides - -import std/[os, strutils, sequtils, options] -import std/[osproc, strformat] -when defined(posix): - import posix - -type - ## Operating system types - OSType* = enum - Linux = "linux" - OpenBSD = "openbsd" - DragonflyBSD = "dragonflybsd" # The Proxmox Killer - NetBSD = "netbsd" - macOS = "macos" - Embedded = "embedded" - - ## Isolation strategy options - IsolationStrategy* = enum - LinuxNamespace = "linux-namespace" ## unshare -r -m (Linux 4.19+) - OpenBSDUnveil = "openbsd-unveil" ## unveil + pledge (OpenBSD 6.4+) - DragonflyJail = "dragonfly-jail" ## jail + nullfs (DragonflyBSD 5.x+) - Our Hammer - POSIXFallback = "posix-fallback" ## chmod + Merkle verification (Legacy/Embedded) - - ## Installation mode - InstallMode* = enum - UserMode = "user" ## --user (Linux only with namespaces) - SystemMode = "system" ## --system (root required) - - ## Platform capabilities detected at runtime - PlatformCapabilities* = object - osType*: OSType - hasUserNamespaces*: bool ## Linux user namespace support - hasJails*: bool ## DragonflyBSD jail support (variant 2) - hasUnveil*: bool ## OpenBSD unveil support - isRoot*: bool ## Running as root - kernelVersion*: string ## Kernel version string - isEmbedded*: bool ## Embedded/IoT device detected - memoryTotal*: int64 ## Total system memory in bytes - cpuCount*: int ## Number of CPU cores - - ## Constraints for embedded devices - EmbeddedConstraints* = object - maxConcurrentDownloads*: int - maxConcurrentBuilds*: int - maxCacheSize*: int64 - enableCompression*: bool - enableDeduplication*: bool - enableParallelization*: bool - - ## Platform detection error - PlatformError* = object of CatchableError - -# ============================================================================ -# OS Type Detection -# ============================================================================ - -proc detectOSType*(): OSType = - ## Detect operating system type at compile time and runtime. - ## Note: DragonflyBSD is explicitly unsupported. - when defined(linux): - return Linux - elif defined(dragonfly): # Correct detect for DragonflyBSD - return DragonflyBSD - elif defined(openbsd): - return OpenBSD - elif defined(netbsd): - return NetBSD - elif defined(macosx): - return macOS - else: - # If we are on bare metal or custom firmware - return Embedded - -proc getOSTypeString*(osType: OSType): string = - ## Get human-readable OS type name - case osType: - of Linux: "Linux (NexBox)" - of DragonflyBSD: "DragonflyBSD (DragonBox)" - of OpenBSD: "OpenBSD (OpenBox)" - of NetBSD: "NetBSD" - of macOS: "macOS" - of Embedded: "Embedded/IoT" - -# ============================================================================ -# Root Check -# ============================================================================ - -proc isRoot*(): bool = - ## Check if running as root - when defined(posix): - return getuid() == 0 - else: - return false - -# ============================================================================ -# Kernel Version Detection -# ============================================================================ - -proc getKernelVersion*(): string = - ## Get kernel version string - try: - when defined(linux) or defined(openbsd) or defined(netbsd) or defined(dragonfly): - let output = execProcess("uname -r").strip() - return output - elif defined(macosx): - let output = execProcess("uname -r").strip() - return output - else: - return "unknown-embedded" - except: - return "unknown" - -# ============================================================================ -# Strategy Selection Logic -# ============================================================================ - -proc recommendIsolationStrategy*(caps: PlatformCapabilities): IsolationStrategy = - ## Determine the best isolation strategy for the current platform - case caps.osType: - of Linux: - if caps.hasUserNamespaces: return LinuxNamespace - else: return POSIXFallback - of OpenBSD: - if caps.hasUnveil: return OpenBSDUnveil - else: return POSIXFallback - of DragonflyBSD: - # Dragonfly doesn't have unveil, but Jails are extremely mature - # and light enough for our purposes when combined with nullfs. - if caps.hasJails: return DragonflyJail - else: return POSIXFallback - of NetBSD, macOS, Embedded: - return POSIXFallback - - -proc parseKernelVersion*(versionStr: string): tuple[major: int, minor: int, patch: int] = - ## Parse kernel version string into components - let parts = versionStr.split('.') - var major, minor, patch = 0 - - if parts.len > 0: - try: - major = parseInt(parts[0]) - except: - discard - - if parts.len > 1: - try: - minor = parseInt(parts[1]) - except: - discard - - if parts.len > 2: - try: - # Extract just the numeric part (e.g., "0-generic" -> "0") - let patchStr = parts[2].split('-')[0] - patch = parseInt(patchStr) - except: - discard - - return (major, minor, patch) - -# ============================================================================ -# Capability Detection -# ============================================================================ - -proc checkUserNamespaceSupport*(): bool = - ## Check if Linux user namespaces are available - ## Requires Linux 4.19+ with CONFIG_USER_NS enabled - when defined(linux): - try: - # Check if /proc/sys/user/max_user_namespaces exists and is > 0 - let maxNsPath = "/proc/sys/user/max_user_namespaces" - if fileExists(maxNsPath): - let content = readFile(maxNsPath).strip() - try: - let maxNs = parseInt(content) - return maxNs > 0 - except: - return false - return false - except: - return false - else: - return false - -proc checkJailSupport*(): bool = - ## Check if DragonflyBSD jails are available - when defined(DragonflyBSD): - try: - # Check if jail command exists - let result = execProcess("which jail").strip() - return result.len > 0 - except: - return false - else: - return false - -proc checkUnveilSupport*(): bool = - ## Check if OpenBSD unveil is available - ## Requires OpenBSD 6.4+ - when defined(openbsd): - try: - # Check kernel version - let versionStr = getKernelVersion() - let (major, minor, _) = parseKernelVersion(versionStr) - # OpenBSD 6.4+ has unveil - return major > 6 or (major == 6 and minor >= 4) - except: - return false - else: - return false - -# ============================================================================ -# System Information Detection -# ============================================================================ - -proc getMemoryTotal*(): int64 = - ## Get total system memory in bytes - try: - when defined(linux): - let output = execProcess("grep MemTotal /proc/meminfo").strip() - let parts = output.split() - if parts.len >= 2: - try: - let kb = parseInt(parts[1]) - return kb * 1024 # Convert KB to bytes - except: - return 0 - elif defined(DragonflyBSD): - let output = execProcess("sysctl -n hw.physmem").strip() - try: - return parseInt(output) - except: - return 0 - elif defined(openbsd): - let output = execProcess("sysctl -n hw.physmem").strip() - try: - return parseInt(output) - except: - return 0 - return 0 - except: - return 0 - -proc getCPUCount*(): int = - ## Get number of CPU cores - try: - when defined(linux): - let output = execProcess("nproc").strip() - try: - return parseInt(output) - except: - discard - elif defined(DragonflyBSD) or defined(openbsd): - let output = execProcess("sysctl -n hw.ncpu").strip() - try: - return parseInt(output) - except: - discard - return 1 - except: - return 1 - -# ============================================================================ -# Embedded Device Detection -# ============================================================================ - -proc detectEmbeddedDevice*(): bool = - ## Detect if running on embedded/IoT device - ## Uses multiple indicators for robust detection - try: - var indicators: seq[bool] = @[] - - # Check for OpenWrt - indicators.add(fileExists("/etc/openwrt_release")) - - # Check for device tree (ARM devices) - indicators.add(fileExists("/proc/device-tree")) - - # Check memory (< 512MB suggests embedded) - let memTotal = getMemoryTotal() - indicators.add(memTotal > 0 and memTotal < 512 * 1024 * 1024) - - # Check CPU count (<= 2 cores suggests embedded) - let cpuCount = getCPUCount() - indicators.add(cpuCount <= 2) - - # Check for Raspberry Pi - indicators.add(fileExists("/proc/device-tree/model")) - - # Need at least 2 indicators to be confident - let trueCount = indicators.countIt(it) - return trueCount >= 2 - except: - return false - -# ============================================================================ -# Main Platform Detection -# ============================================================================ - -proc detectPlatform*(): PlatformCapabilities = - ## Detect OS and capabilities at runtime - ## - ## This is the main entry point for platform detection. It queries the - ## system for OS type, kernel version, and available isolation capabilities. - - let osType = detectOSType() - let isRootUser = isRoot() - let kernelVersion = getKernelVersion() - let isEmbedded = detectEmbeddedDevice() - let memoryTotal = getMemoryTotal() - let cpuCount = getCPUCount() - - case osType: - of Linux: - let hasUserNS = checkUserNamespaceSupport() - return PlatformCapabilities( - osType: Linux, - hasUserNamespaces: hasUserNS, - hasJails: false, - hasUnveil: false, - isRoot: isRootUser, - kernelVersion: kernelVersion, - isEmbedded: isEmbedded, - memoryTotal: memoryTotal, - cpuCount: cpuCount - ) - - of DragonflyBSD: - let hasJails = checkJailSupport() - return PlatformCapabilities( - osType: DragonflyBSD, - hasUserNamespaces: false, - hasJails: hasJails, - hasUnveil: false, - isRoot: isRootUser, - kernelVersion: kernelVersion, - isEmbedded: isEmbedded, - memoryTotal: memoryTotal, - cpuCount: cpuCount - ) - - of OpenBSD: - let hasUnveil = checkUnveilSupport() - return PlatformCapabilities( - osType: OpenBSD, - hasUserNamespaces: false, - hasJails: false, - hasUnveil: hasUnveil, - isRoot: isRootUser, - kernelVersion: kernelVersion, - isEmbedded: isEmbedded, - memoryTotal: memoryTotal, - cpuCount: cpuCount - ) - - else: - return PlatformCapabilities( - osType: osType, - hasUserNamespaces: false, - hasJails: false, - hasUnveil: false, - isRoot: isRootUser, - kernelVersion: kernelVersion, - isEmbedded: isEmbedded, - memoryTotal: memoryTotal, - cpuCount: cpuCount - ) - -# ============================================================================ -# Isolation Strategy Selection -# ============================================================================ - -proc selectStrategy*(caps: PlatformCapabilities): IsolationStrategy = - ## Select best isolation strategy based on platform capabilities - ## - ## This implements the strategy selection algorithm: - ## 1. Check for platform-specific advanced isolation - ## 2. Fall back to POSIX fallback if not available - ## 3. Ensure graceful degradation - - case caps.osType: - of Linux: - if caps.hasUserNamespaces: - return LinuxNamespace # Preferred: kernel-enforced isolation - else: - return POSIXFallback # Fallback: chmod + Merkle verification - - of DragonflyBSD: - if caps.hasJails and caps.isRoot: - return DragonflyJail # Preferred: elegant BSD solution - else: - return POSIXFallback # Fallback: chmod + root - - of OpenBSD: - if caps.hasUnveil and caps.isRoot: - return OpenBSDUnveil # Preferred: capability-based security - else: - return POSIXFallback # Fallback: chmod + root - - else: - return POSIXFallback # Default: POSIX fallback for all others - -proc selectMode*(strategy: IsolationStrategy, userRequest: Option[ - InstallMode]): InstallMode = - ## Select installation mode based on strategy and user request - ## - ## Modes: - ## - UserMode: User-level installation (Linux with namespaces only) - ## - SystemMode: System-wide installation (requires root) - - # User explicitly requested a mode - if userRequest.isSome: - let requested = userRequest.get() - - case requested: - of UserMode: - if strategy == LinuxNamespace: - return UserMode # OK: Linux with namespaces - else: - echo "❌ User mode not available on this platform" - echo " Strategy: " & $strategy - echo " Falling back to system mode (requires root)" - return SystemMode - - of SystemMode: - return SystemMode # Always possible if root - - # Auto-select based on strategy - case strategy: - of LinuxNamespace: - return UserMode # Linux: prefer user mode - - of DragonflyJail, OpenBSDUnveil: - return SystemMode # BSD: requires root - - of POSIXFallback: - if isRoot(): - return SystemMode # Root: use system mode - else: - return UserMode # Non-root: use user mode (with warnings) - -# ============================================================================ -# Strategy Information -# ============================================================================ - -proc getStrategyDescription*(strategy: IsolationStrategy): string = - ## Get human-readable description of isolation strategy - case strategy: - of LinuxNamespace: - return "Linux user namespaces (kernel-enforced read-only)" - of DragonflyJail: - return "DragonflyBSD jails with nullfs (elegant BSD solution)" - of OpenBSDUnveil: - return "OpenBSD unveil + pledge (capability-based security)" - of POSIXFallback: - return "POSIX fallback (chmod + Merkle verification)" - -proc getSecurityLevel*(strategy: IsolationStrategy): int = - ## Get security level (1-5 stars) - ## This is informational only - all strategies provide security through Merkle verification - case strategy: - of LinuxNamespace: - return 5 # Kernel-enforced - of DragonflyJail: - return 5 # Kernel-enforced, mature - of OpenBSDUnveil: - return 4 # Capability-based, but reset on exec - of POSIXFallback: - return 1 # UX convenience only (Merkle is primary security) - -proc getStrategyInfo*(strategy: IsolationStrategy): string = - ## Get detailed information about isolation strategy - let desc = getStrategyDescription(strategy) - let level = getSecurityLevel(strategy) - let stars = "⭐".repeat(level) - - case strategy: - of LinuxNamespace: - return fmt"{desc}\n{stars}\nKernel-enforced read-only mount prevents any writes" - of DragonflyJail: - return fmt"{desc}\n{stars}\nProcess confined to jail, cannot escape" - of OpenBSDUnveil: - return fmt"{desc}\n{stars}\nPath-based access control with capability restrictions" - of POSIXFallback: - return fmt"{desc}\n{stars}\nPrimary security: Merkle verification detects tampering" - -# ============================================================================ -# Embedded Device Constraints -# ============================================================================ - -proc getEmbeddedConstraints*(): EmbeddedConstraints = - ## Get constraints for embedded devices - ## - ## Embedded devices have limited resources, so we adjust: - ## - Reduce concurrent operations - ## - Limit cache size - ## - Disable parallelization on single-core devices - - let memoryTotal = getMemoryTotal() - let cpuCount = getCPUCount() - - return EmbeddedConstraints( - maxConcurrentDownloads: if memoryTotal < 256 * 1024 * 1024: 1 else: 2, - maxConcurrentBuilds: 1, - maxCacheSize: if memoryTotal < 256 * 1024 * 1024: 50 * 1024 * - 1024 else: 100 * 1024 * 1024, - enableCompression: true, - enableDeduplication: true, - enableParallelization: cpuCount > 2 - ) - -proc formatBytes*(bytes: int64): string = - ## Format bytes as human-readable string - if bytes < 1024: - return fmt"{bytes}B" - elif bytes < 1024 * 1024: - return fmt"{bytes div 1024}KB" - elif bytes < 1024 * 1024 * 1024: - return fmt"{bytes div (1024 * 1024)}MB" - else: - return fmt"{bytes div (1024 * 1024 * 1024)}GB" - -proc printEmbeddedConstraints*(constraints: EmbeddedConstraints) = - ## Print embedded device constraints - echo "📱 Embedded device detected" - echo " Max concurrent downloads: " & $constraints.maxConcurrentDownloads - echo " Max concurrent builds: " & $constraints.maxConcurrentBuilds - echo " Max cache size: " & formatBytes(constraints.maxCacheSize) - echo " Compression enabled: " & $constraints.enableCompression - echo " Deduplication enabled: " & $constraints.enableDeduplication - echo " Parallelization enabled: " & $constraints.enableParallelization - -# ============================================================================ -# Platform Summary -# ============================================================================ - -proc printPlatformInfo*(caps: PlatformCapabilities) = - ## Print platform information for debugging - echo "🖥️ Platform Information" - echo " OS: " & getOSTypeString(caps.osType) - echo " Kernel: " & caps.kernelVersion - echo " Root: " & $caps.isRoot - echo " Memory: " & formatBytes(caps.memoryTotal) - echo " CPUs: " & $caps.cpuCount - echo " Embedded: " & $caps.isEmbedded - - echo "" - echo "🔒 Isolation Capabilities" - echo " User Namespaces: " & $caps.hasUserNamespaces - echo " Jails: " & $caps.hasJails - echo " Unveil: " & $caps.hasUnveil - - let strategy = selectStrategy(caps) - echo "" - echo "📋 Selected Strategy" - echo " " & getStrategyDescription(strategy) - echo " Security Level: " & "⭐".repeat(getSecurityLevel(strategy)) - - if caps.isEmbedded: - echo "" - let constraints = getEmbeddedConstraints() - printEmbeddedConstraints(constraints) diff --git a/src/nip/remote.nim b/src/nip/remote.nim deleted file mode 100644 index e69de29..0000000 diff --git a/src/nip/resolver/build_synthesis.nim b/src/nip/resolver/build_synthesis.nim deleted file mode 100644 index 3bb300a..0000000 --- a/src/nip/resolver/build_synthesis.nim +++ /dev/null @@ -1,337 +0,0 @@ -## Build Synthesis Module -## -## This module implements deterministic build synthesis for the NIP dependency resolver. -## It takes unified variant profiles and synthesizes reproducible builds with deterministic -## hashing for content-addressable storage. -## -## Philosophy: -## - Build synthesis is the bridge between variant unification and CAS storage -## - Every build has a deterministic hash based on its configuration -## - Same variant profile + source = same build hash (reproducibility guarantee) -## - Build hashes are xxh3-128 for performance (non-cryptographic) - -import std/[tables, strutils, times, options, sequtils, algorithm, os] -import ../xxhash # For xxh3-128 hashing -import ./variant_types - -type - # Build configuration for synthesis - BuildConfig* = object - packageName*: string - packageVersion*: string - variantProfile*: VariantProfile - sourceHash*: string # Hash of source code - compilerVersion*: string # Compiler version used - compilerFlags*: seq[string] # Compiler flags - configureFlags*: seq[string] # Configure flags - targetArchitecture*: string # Target architecture - libc*: string # libc type (musl, glibc) - allocator*: string # Memory allocator (jemalloc, tcmalloc) - timestamp*: times.Time # Build timestamp - - # Result of build synthesis - BuildSynthesisResult* = object - buildHash*: string # xxh3-128 hash of build - casID*: string # CAS identifier (same as buildHash) - buildConfig*: BuildConfig - timestamp*: times.Time - - # Build synthesis error - BuildSynthesisError* = object of CatchableError - reason*: string - -# Constructor for BuildConfig -proc newBuildConfig*( - packageName: string, - packageVersion: string, - variantProfile: VariantProfile, - sourceHash: string = "", - compilerVersion: string = "gcc-13.2.0", - compilerFlags: seq[string] = @["-O2", "-march=native"], - configureFlags: seq[string] = @[], - targetArchitecture: string = "x86_64", - libc: string = "musl", - allocator: string = "jemalloc" -): BuildConfig = - result.packageName = packageName - result.packageVersion = packageVersion - result.variantProfile = variantProfile - result.sourceHash = sourceHash - result.compilerVersion = compilerVersion - result.compilerFlags = compilerFlags - result.configureFlags = configureFlags - result.targetArchitecture = targetArchitecture - result.libc = libc - result.allocator = allocator - result.timestamp = getTime() - -# Calculate canonical representation for build hash -proc toCanonical*(config: BuildConfig): string = - ## Convert build config to canonical string for deterministic hashing - ## Format: package|version|variant_hash|source_hash|compiler|flags|arch|libc|allocator - ## - ## This ensures: - ## - Same configuration always produces same hash - ## - Different configurations produce different hashes - ## - Hash is deterministic across builds and machines - - var parts: seq[string] = @[] - - # Package identification - parts.add(config.packageName) - parts.add(config.packageVersion) - - # Variant profile (already canonical) - parts.add(config.variantProfile.toCanonical()) - - # Source integrity - parts.add(config.sourceHash) - - # Compiler configuration (sorted for determinism) - parts.add(config.compilerVersion) - parts.add(config.compilerFlags.sorted().join(",")) - - # Build configuration (sorted for determinism) - parts.add(config.configureFlags.sorted().join(",")) - - # Target environment - parts.add(config.targetArchitecture) - parts.add(config.libc) - parts.add(config.allocator) - - # Join with | separator - result = parts.join("|") - -# Calculate build hash using xxh3-128 -proc calculateBuildHash*(config: BuildConfig): string = - ## Calculate deterministic xxh3-128 hash for build configuration - ## - ## This hash serves as: - ## - Unique identifier for the build - ## - CAS identifier for storage - ## - Reproducibility guarantee (same config = same hash) - - let canonical = config.toCanonical() - let hashValue = calculateXXH3(canonical) - result = $hashValue # XXH3Hash already includes "xxh3-" prefix - -# Synthesize a build from variant profile -proc synthesizeBuild*( - packageName: string, - packageVersion: string, - variantProfile: VariantProfile, - sourceHash: string = "", - compilerVersion: string = "gcc-13.2.0", - compilerFlags: seq[string] = @["-O2", "-march=native"], - configureFlags: seq[string] = @[], - targetArchitecture: string = "x86_64", - libc: string = "musl", - allocator: string = "jemalloc" -): BuildSynthesisResult = - ## Synthesize a build from a unified variant profile - ## - ## This function: - ## 1. Creates a build configuration from the variant profile - ## 2. Calculates a deterministic build hash - ## 3. Returns the build hash as CAS identifier - ## - ## The build hash is deterministic: same inputs always produce same hash - - # Create build configuration - let config = newBuildConfig( - packageName = packageName, - packageVersion = packageVersion, - variantProfile = variantProfile, - sourceHash = sourceHash, - compilerVersion = compilerVersion, - compilerFlags = compilerFlags, - configureFlags = configureFlags, - targetArchitecture = targetArchitecture, - libc = libc, - allocator = allocator - ) - - # Calculate deterministic build hash - let buildHash = calculateBuildHash(config) - - # Return synthesis result - result = BuildSynthesisResult( - buildHash: buildHash, - casID: buildHash, # CAS ID is the build hash - buildConfig: config, - timestamp: getTime() - ) - -# Store synthesized build in CAS -proc storeBuildInCAS*( - buildResult: BuildSynthesisResult, - casRoot: string, - buildMetadata: string = "" -): string = - ## Store synthesized build in CAS and return CAS ID - ## - ## This function: - ## 1. Serializes the build configuration - ## 2. Stores it in the CAS using the build hash as identifier - ## 3. Returns the CAS ID for later retrieval - ## - ## The CAS ensures: - ## - Content-addressed storage (hash = identifier) - ## - Deduplication (same build = same hash = single storage) - ## - Integrity verification (hash matches content) - - # Serialize build configuration - var serialized = "" - serialized.add("package: " & buildResult.buildConfig.packageName & "\n") - serialized.add("version: " & buildResult.buildConfig.packageVersion & "\n") - serialized.add("variant: " & buildResult.buildConfig.variantProfile.toCanonical() & "\n") - serialized.add("source_hash: " & buildResult.buildConfig.sourceHash & "\n") - serialized.add("compiler: " & buildResult.buildConfig.compilerVersion & "\n") - serialized.add("compiler_flags: " & buildResult.buildConfig.compilerFlags.sorted().join(",") & "\n") - serialized.add("configure_flags: " & buildResult.buildConfig.configureFlags.sorted().join(",") & "\n") - serialized.add("target_arch: " & buildResult.buildConfig.targetArchitecture & "\n") - serialized.add("libc: " & buildResult.buildConfig.libc & "\n") - serialized.add("allocator: " & buildResult.buildConfig.allocator & "\n") - serialized.add("build_hash: " & buildResult.buildHash & "\n") - - if buildMetadata != "": - serialized.add("metadata: " & buildMetadata & "\n") - - # Create directory structure for CAS storage - let shardPath = buildResult.casID[0..3] # Use first 4 chars for sharding - let fullShardPath = casRoot / shardPath - createDir(fullShardPath) - - # Store the serialized build - let objectPath = fullShardPath / buildResult.casID - writeFile(objectPath, serialized) - - # Return CAS ID (which is the build hash) - result = buildResult.casID - -# Retrieve build from CAS -proc retrieveBuildFromCAS*( - casID: string, - casRoot: string -): BuildSynthesisResult = - ## Retrieve a synthesized build from CAS by its ID - ## - ## This function: - ## 1. Retrieves the build metadata from CAS - ## 2. Verifies the hash matches the CAS ID - ## 3. Reconstructs the build configuration - - # Construct path to retrieve from CAS - let shardPath = casID[0..3] # Use first 4 chars for sharding - let objectPath = casRoot / shardPath / casID - - if not fileExists(objectPath): - raise newException(BuildSynthesisError, "Build not found in CAS: " & casID) - - # Retrieve from CAS - let data = readFile(objectPath) - - # Parse the serialized data (simplified parsing) - var config = BuildConfig() - var buildHash = "" - var variantCanonical = "" - var compilerFlagsStr = "" - var configureFlagsStr = "" - - for line in data.split("\n"): - if line.startsWith("package: "): - config.packageName = line[9..^1] - elif line.startsWith("version: "): - config.packageVersion = line[9..^1] - elif line.startsWith("variant: "): - variantCanonical = line[9..^1] - elif line.startsWith("source_hash: "): - config.sourceHash = line[13..^1] - elif line.startsWith("compiler: "): - config.compilerVersion = line[10..^1] - elif line.startsWith("compiler_flags: "): - compilerFlagsStr = line[16..^1] - elif line.startsWith("configure_flags: "): - configureFlagsStr = line[17..^1] - elif line.startsWith("target_arch: "): - config.targetArchitecture = line[13..^1] - elif line.startsWith("libc: "): - config.libc = line[6..^1] - elif line.startsWith("allocator: "): - config.allocator = line[11..^1] - elif line.startsWith("build_hash: "): - buildHash = line[12..^1] - - # Reconstruct compiler and configure flags - if compilerFlagsStr != "": - config.compilerFlags = compilerFlagsStr.split(",") - if configureFlagsStr != "": - config.configureFlags = configureFlagsStr.split(",") - - # Reconstruct variant profile from canonical representation - var reconstructedProfile = newVariantProfile() - # Parse the canonical representation: domain1:flag1,flag2|domain2:flag3 - for domainPart in variantCanonical.split("|"): - if domainPart.contains(":"): - let parts = domainPart.split(":") - let domainName = parts[0] - let flagsStr = parts[1] - for flag in flagsStr.split(","): - if flag != "": - reconstructedProfile.addFlag(domainName, flag) - reconstructedProfile.calculateHash() - config.variantProfile = reconstructedProfile - - # Verify hash matches - let calculatedHash = calculateBuildHash(config) - if calculatedHash != buildHash: - raise newException(BuildSynthesisError, - "Build hash mismatch: expected " & buildHash & ", got " & calculatedHash) - - result = BuildSynthesisResult( - buildHash: buildHash, - casID: casID, - buildConfig: config, - timestamp: getTime() - ) - -# Verify build hash matches configuration -proc verifyBuildHash*( - buildHash: string, - config: BuildConfig -): bool = - ## Verify that a build hash matches its configuration - ## - ## This ensures: - ## - Build integrity (hash matches configuration) - ## - Reproducibility (same config = same hash) - ## - No tampering (hash mismatch = configuration changed) - - let calculatedHash = calculateBuildHash(config) - result = buildHash == calculatedHash - -# Check if two builds are identical -proc isBuildIdentical*( - build1: BuildSynthesisResult, - build2: BuildSynthesisResult -): bool = - ## Check if two builds are identical - ## - ## Two builds are identical if: - ## - They have the same build hash - ## - Their configurations produce the same hash - - result = build1.buildHash == build2.buildHash and - build1.buildHash == calculateBuildHash(build2.buildConfig) - -# String representation for display -proc `$`*(bsr: BuildSynthesisResult): string = - ## Human-readable string representation - - result = "BuildSynthesisResult(\n" & - " package: " & bsr.buildConfig.packageName & "\n" & - " version: " & bsr.buildConfig.packageVersion & "\n" & - " variant: " & bsr.buildConfig.variantProfile.toCanonical() & "\n" & - " build_hash: " & bsr.buildHash & "\n" & - " cas_id: " & bsr.casID & "\n" & - ")" diff --git a/src/nip/resolver/cas_integration.nim b/src/nip/resolver/cas_integration.nim deleted file mode 100644 index 6bc21a3..0000000 --- a/src/nip/resolver/cas_integration.nim +++ /dev/null @@ -1,316 +0,0 @@ -## CAS Integration for Build Synthesis -## -## This module integrates the build synthesis system with the existing -## Content-Addressable Storage (CAS) system. It provides functions to: -## - Store synthesized builds in the CAS -## - Retrieve builds from the CAS -## - Track references to builds -## - Manage build artifacts and metadata - -import std/[tables, strutils, times, options, os, algorithm] -import ../cas -import ../types -import ./build_synthesis -import ./variant_types - -# Result type for error handling -type - Result*[T, E] = object - case isOk*: bool - of true: - value*: T - of false: - error*: E - -template ok*[T](value: T): untyped = - Result[T, string](isOk: true, value: value) - -template err*[T](error: string): untyped = - Result[T, string](isOk: false, error: error) - -proc get*[T](res: Result[T, string]): T = - if res.isOk: - return res.value - raise newException(ValueError, "Cannot get value from error result") - -type - # Reference tracking for builds - BuildReference* = object - buildHash*: string - casHash*: Multihash - packageName*: string - packageVersion*: string - timestamp*: times.Time - refCount*: int - - # Build artifact metadata - BuildArtifact* = object - buildHash*: string - casHash*: Multihash - size*: int64 - compressed*: bool - timestamp*: times.Time - variantProfile*: VariantProfile - - # CAS integration manager - CASIntegrationManager* = object - casRoot*: string - references*: Table[string, BuildReference] # buildHash -> reference - artifacts*: Table[string, BuildArtifact] # buildHash -> artifact - -# Constructor for CASIntegrationManager -proc newCASIntegrationManager*(casRoot: string): CASIntegrationManager = - result.casRoot = casRoot - result.references = initTable[string, BuildReference]() - result.artifacts = initTable[string, BuildArtifact]() - -# Store a synthesized build in the CAS -proc storeBuildInCAS*( - manager: var CASIntegrationManager, - buildResult: BuildSynthesisResult -): Result[Multihash, string] = - ## Store a synthesized build in the CAS and track the reference - ## - ## This function: - ## 1. Serializes the build configuration - ## 2. Stores it in the CAS using BLAKE2b-512 - ## 3. Tracks the reference for garbage collection - ## 4. Returns the CAS hash for retrieval - - try: - # Serialize build configuration - var serialized = "" - serialized.add("package: " & buildResult.buildConfig.packageName & "\n") - serialized.add("version: " & buildResult.buildConfig.packageVersion & "\n") - serialized.add("variant: " & buildResult.buildConfig.variantProfile.toCanonical() & "\n") - serialized.add("source_hash: " & buildResult.buildConfig.sourceHash & "\n") - serialized.add("compiler: " & buildResult.buildConfig.compilerVersion & "\n") - serialized.add("compiler_flags: " & buildResult.buildConfig.compilerFlags.sorted().join(",") & "\n") - serialized.add("configure_flags: " & buildResult.buildConfig.configureFlags.sorted().join(",") & "\n") - serialized.add("target_arch: " & buildResult.buildConfig.targetArchitecture & "\n") - serialized.add("libc: " & buildResult.buildConfig.libc & "\n") - serialized.add("allocator: " & buildResult.buildConfig.allocator & "\n") - serialized.add("build_hash: " & buildResult.buildHash & "\n") - - # Store in CAS using existing system - let casObject = storeObject(serialized, manager.casRoot, compress = true) - - # Track reference - let reference = BuildReference( - buildHash: buildResult.buildHash, - casHash: casObject.hash, - packageName: buildResult.buildConfig.packageName, - packageVersion: buildResult.buildConfig.packageVersion, - timestamp: getTime(), - refCount: 1 - ) - - manager.references[buildResult.buildHash] = reference - - # Track artifact - let artifact = BuildArtifact( - buildHash: buildResult.buildHash, - casHash: casObject.hash, - size: casObject.size, - compressed: casObject.compressed, - timestamp: casObject.timestamp, - variantProfile: buildResult.buildConfig.variantProfile - ) - - manager.artifacts[buildResult.buildHash] = artifact - - return Result[Multihash, string](isOk: true, value: casObject.hash) - - except Exception as e: - return Result[Multihash, string](isOk: false, error: "Failed to store build in CAS: " & e.msg) - -# Retrieve a build from the CAS -proc retrieveBuildFromCAS*( - manager: CASIntegrationManager, - casHash: Multihash -): Result[BuildSynthesisResult, string] = - ## Retrieve a synthesized build from the CAS - ## - ## This function: - ## 1. Retrieves the build metadata from CAS - ## 2. Verifies the hash matches - ## 3. Reconstructs the build configuration - ## 4. Returns the build result - - try: - # Retrieve from CAS - let data = retrieveObject(casHash, manager.casRoot) - - # Parse the serialized data - var config = BuildConfig() - var buildHash = "" - var variantCanonical = "" - var compilerFlagsStr = "" - var configureFlagsStr = "" - - for line in data.split("\n"): - if line.startsWith("package: "): - config.packageName = line[9..^1] - elif line.startsWith("version: "): - config.packageVersion = line[9..^1] - elif line.startsWith("variant: "): - variantCanonical = line[9..^1] - elif line.startsWith("source_hash: "): - config.sourceHash = line[13..^1] - elif line.startsWith("compiler: "): - config.compilerVersion = line[10..^1] - elif line.startsWith("compiler_flags: "): - compilerFlagsStr = line[16..^1] - elif line.startsWith("configure_flags: "): - configureFlagsStr = line[17..^1] - elif line.startsWith("target_arch: "): - config.targetArchitecture = line[13..^1] - elif line.startsWith("libc: "): - config.libc = line[6..^1] - elif line.startsWith("allocator: "): - config.allocator = line[11..^1] - elif line.startsWith("build_hash: "): - buildHash = line[12..^1] - - # Reconstruct compiler and configure flags - if compilerFlagsStr != "": - config.compilerFlags = compilerFlagsStr.split(",") - if configureFlagsStr != "": - config.configureFlags = configureFlagsStr.split(",") - - # Reconstruct variant profile - var reconstructedProfile = newVariantProfile() - for domainPart in variantCanonical.split("|"): - if domainPart.contains(":"): - let parts = domainPart.split(":") - let domainName = parts[0] - let flagsStr = parts[1] - for flag in flagsStr.split(","): - if flag != "": - reconstructedProfile.addFlag(domainName, flag) - reconstructedProfile.calculateHash() - config.variantProfile = reconstructedProfile - - # Verify hash matches - let calculatedHash = calculateBuildHash(config) - if calculatedHash != buildHash: - return Result[BuildSynthesisResult, string](isOk: false, error: "Build hash mismatch: expected " & buildHash & ", got " & calculatedHash) - - return Result[BuildSynthesisResult, string](isOk: true, value: BuildSynthesisResult( - buildHash: buildHash, - casID: string(casHash), - buildConfig: config, - timestamp: getTime() - )) - - except Exception as e: - return Result[BuildSynthesisResult, string](isOk: false, error: "Failed to retrieve build from CAS: " & e.msg) - -# Verify a build exists in the CAS -proc verifyBuildInCAS*( - manager: CASIntegrationManager, - buildHash: string -): bool = - ## Verify that a build exists in the CAS - - if not manager.artifacts.hasKey(buildHash): - return false - - let artifact = manager.artifacts[buildHash] - - try: - # Try to retrieve the object - discard retrieveObject(artifact.casHash, manager.casRoot) - return true - except: - return false - -# Increment reference count for a build -proc incrementReference*( - manager: var CASIntegrationManager, - buildHash: string -): Result[void, string] = - ## Increment the reference count for a build - - if not manager.references.hasKey(buildHash): - return Result[void, string](isOk: false, error: "Build not found: " & buildHash) - - manager.references[buildHash].refCount += 1 - return Result[void, string](isOk: true) - -# Decrement reference count for a build -proc decrementReference*( - manager: var CASIntegrationManager, - buildHash: string -): Result[int, string] = - ## Decrement the reference count for a build - ## Returns the new reference count - - if not manager.references.hasKey(buildHash): - return Result[int, string](isOk: false, error: "Build not found: " & buildHash) - - manager.references[buildHash].refCount -= 1 - return Result[int, string](isOk: true, value: manager.references[buildHash].refCount) - -# Get reference count for a build -proc getReferenceCount*( - manager: CASIntegrationManager, - buildHash: string -): Option[int] = - ## Get the reference count for a build - - if manager.references.hasKey(buildHash): - return some(manager.references[buildHash].refCount) - return none(int) - -# List all tracked builds -proc listTrackedBuilds*( - manager: CASIntegrationManager -): seq[BuildReference] = - ## List all tracked builds - - result = @[] - for buildHash, reference in manager.references: - result.add(reference) - -# Get artifact metadata for a build -proc getArtifactMetadata*( - manager: CASIntegrationManager, - buildHash: string -): Option[BuildArtifact] = - ## Get artifact metadata for a build - - if manager.artifacts.hasKey(buildHash): - return some(manager.artifacts[buildHash]) - return none(BuildArtifact) - -# Calculate total size of tracked builds -proc getTotalTrackedSize*( - manager: CASIntegrationManager -): int64 = - ## Calculate total size of all tracked builds - - result = 0 - for buildHash, artifact in manager.artifacts: - result += artifact.size - -# String representation for display -proc `$`*(reference: BuildReference): string = - ## Human-readable string representation - - result = "BuildReference(\n" & - " build_hash: " & reference.buildHash & "\n" & - " cas_hash: " & string(reference.casHash) & "\n" & - " package: " & reference.packageName & " " & reference.packageVersion & "\n" & - " ref_count: " & $reference.refCount & "\n" & - ")" - -proc `$`*(artifact: BuildArtifact): string = - ## Human-readable string representation - - result = "BuildArtifact(\n" & - " build_hash: " & artifact.buildHash & "\n" & - " cas_hash: " & string(artifact.casHash) & "\n" & - " size: " & $artifact.size & " bytes\n" & - " compressed: " & $artifact.compressed & "\n" & - ")" diff --git a/src/nip/resolver/cdcl_solver.nim b/src/nip/resolver/cdcl_solver.nim deleted file mode 100644 index d071f7d..0000000 --- a/src/nip/resolver/cdcl_solver.nim +++ /dev/null @@ -1,403 +0,0 @@ -## CDCL Solver for Dependency Resolution -## -## This module implements a Conflict-Driven Clause Learning (CDCL) SAT solver -## adapted for package dependency resolution with the PubGrub algorithm. -## -## Philosophy: -## - Start with root requirements (unit clauses) -## - Make decisions (select package versions) -## - Propagate implications (unit propagation) -## - Detect conflicts -## - Learn from conflicts (add new clauses) -## - Backjump to root cause (non-chronological backtracking) -## -## Key Concepts: -## - Decision: Choosing a package version to install -## - Implication: Forced choice due to unit propagation -## - Conflict: Incompatible assignments detected -## - Learned Clause: New constraint derived from conflict analysis -## - Backjumping: Jump to earliest decision causing conflict - -import std/[tables, sets, options, sequtils, algorithm] -import ./cnf_translator -import ./solver_types -import ./variant_types -import ../manifest_parser - -type - ## Assignment type (decision vs implication) - AssignmentType* = enum - Decision, ## User choice or heuristic selection - Implication ## Forced by unit propagation - - ## A variable assignment in the solver - SolverAssignment* = object - variable*: BoolVar - value*: bool ## true = selected, false = not selected - assignmentType*: AssignmentType - decisionLevel*: int - antecedent*: Option[Clause] ## The clause that forced this (for implications) - - ## Conflict information - Conflict* = object - clause*: Clause - assignments*: seq[SolverAssignment] - - ## The CDCL solver state - CDCLSolver* = object - formula*: CNFFormula - assignments*: Table[BoolVar, SolverAssignment] - decisionLevel*: int - learnedClauses*: seq[Clause] - propagationQueue*: seq[BoolVar] - - ## Solver result - SolverResult* = object - case isSat*: bool - of true: - model*: Table[BoolVar, bool] - of false: - conflict*: Conflict - -# --- Assignment Operations --- - -proc isAssigned*(solver: CDCLSolver, variable: BoolVar): bool = - ## Check if a variable has been assigned - result = solver.assignments.hasKey(variable) - -proc getAssignment*(solver: CDCLSolver, variable: BoolVar): Option[SolverAssignment] = - ## Get the assignment for a variable - if solver.assignments.hasKey(variable): - return some(solver.assignments[variable]) - else: - return none(SolverAssignment) - -proc getValue*(solver: CDCLSolver, variable: BoolVar): Option[bool] = - ## Get the value of a variable - if solver.assignments.hasKey(variable): - return some(solver.assignments[variable].value) - else: - return none(bool) - -proc assign*(solver: var CDCLSolver, variable: BoolVar, value: bool, - assignmentType: AssignmentType, antecedent: Option[Clause] = none(Clause)) = - ## Assign a value to a variable - solver.assignments[variable] = SolverAssignment( - variable: variable, - value: value, - assignmentType: assignmentType, - decisionLevel: solver.decisionLevel, - antecedent: antecedent - ) - - # Add to propagation queue if this is a decision - if assignmentType == Decision: - solver.propagationQueue.add(variable) - -proc unassign*(solver: var CDCLSolver, variable: BoolVar) = - ## Remove an assignment - solver.assignments.del(variable) - -# --- Clause Evaluation --- - -proc evaluateLiteral*(solver: CDCLSolver, literal: Literal): Option[bool] = - ## Evaluate a literal given current assignments - ## Returns: Some(true) if satisfied, Some(false) if falsified, None if unassigned - - let varValue = solver.getValue(literal.variable) - if varValue.isNone: - return none(bool) - - let value = varValue.get() - if literal.isNegated: - return some(not value) - else: - return some(value) - -proc evaluateClause*(solver: CDCLSolver, clause: Clause): Option[bool] = - ## Evaluate a clause given current assignments - ## Returns: Some(true) if satisfied, Some(false) if falsified, None if undetermined - - var hasUnassigned = false - - for literal in clause.literals: - let litValue = solver.evaluateLiteral(literal) - - if litValue.isSome: - if litValue.get(): - # Clause is satisfied (at least one literal is true) - return some(true) - else: - hasUnassigned = true - - if hasUnassigned: - # Clause is undetermined (has unassigned literals) - return none(bool) - else: - # All literals are false, clause is falsified - return some(false) - -proc isUnitClause*(solver: CDCLSolver, clause: Clause): Option[Literal] = - ## Check if a clause is unit (exactly one unassigned literal, rest false) - ## Returns the unassigned literal if unit, None otherwise - - var unassignedLiteral: Option[Literal] = none(Literal) - var unassignedCount = 0 - - for literal in clause.literals: - let litValue = solver.evaluateLiteral(literal) - - if litValue.isNone: - # Unassigned literal - unassignedCount += 1 - unassignedLiteral = some(literal) - if unassignedCount > 1: - return none(Literal) # More than one unassigned - elif litValue.get(): - # Literal is true, clause is satisfied - return none(Literal) - - if unassignedCount == 1: - return unassignedLiteral - else: - return none(Literal) - -# --- Unit Propagation --- - -proc unitPropagate*(solver: var CDCLSolver): Option[Conflict] = - ## Perform unit propagation (Boolean Constraint Propagation) - ## Returns a conflict if one is detected, None otherwise - ## - ## Requirements: 5.1 - Use PubGrub algorithm with CDCL - - var changed = true - while changed: - changed = false - - # Check all clauses for unit clauses - for clause in solver.formula.clauses: - let clauseValue = solver.evaluateClause(clause) - - if clauseValue.isSome and not clauseValue.get(): - # Clause is falsified - conflict! - return some(Conflict( - clause: clause, - assignments: solver.assignments.values.toSeq - )) - - let unitLit = solver.isUnitClause(clause) - if unitLit.isSome: - let lit = unitLit.get() - - # Check if already assigned - if solver.isAssigned(lit.variable): - let currentValue = solver.getValue(lit.variable).get() - let requiredValue = not lit.isNegated - - if currentValue != requiredValue: - # Conflict: variable must be both true and false - return some(Conflict( - clause: clause, - assignments: solver.assignments.values.toSeq - )) - else: - # Assign the variable to satisfy the unit clause - let value = not lit.isNegated - solver.assign(lit.variable, value, Implication, some(clause)) - changed = true - - # Check learned clauses too - for clause in solver.learnedClauses: - let clauseValue = solver.evaluateClause(clause) - - if clauseValue.isSome and not clauseValue.get(): - # Clause is falsified - conflict! - return some(Conflict( - clause: clause, - assignments: solver.assignments.values.toSeq - )) - - let unitLit = solver.isUnitClause(clause) - if unitLit.isSome: - let lit = unitLit.get() - - if not solver.isAssigned(lit.variable): - let value = not lit.isNegated - solver.assign(lit.variable, value, Implication, some(clause)) - changed = true - - return none(Conflict) - -# --- Decision Heuristics --- - -proc selectUnassignedVariable*(solver: CDCLSolver): Option[BoolVar] = - ## Select an unassigned variable using a heuristic - ## For now, we use a simple first-unassigned heuristic - ## TODO: Implement VSIDS or other advanced heuristics - - for variable, _ in solver.formula.variables.pairs: - if not solver.isAssigned(variable): - return some(variable) - - return none(BoolVar) - -# --- Conflict Analysis --- - -proc analyzeConflict*(solver: CDCLSolver, conflict: Conflict): Clause = - ## Analyze a conflict and learn a new clause - ## This implements the "first UIP" (Unique Implication Point) scheme - ## - ## Requirements: 5.2 - Learn new incompatibility clause from conflicts - - # 1. Initialize resolution - # Start with the conflict clause - var currentClauseLiterals = conflict.clause.literals - - # We want to resolve literals that were assigned at the current decision level - # until only one remains (the UIP). - - # For this MVP, we'll stick to a simpler "block this assignment" strategy - # but with a bit more intelligence: we'll include the decision variables - # that led to this conflict. - - var learnedLiterals: seq[Literal] = @[] - var seenVariables = initHashSet[BoolVar]() - - # Collect all decision variables that are antecedents of the conflict - for assignment in conflict.assignments: - if assignment.assignmentType == Decision: - if assignment.variable notin seenVariables: - seenVariables.incl(assignment.variable) - # Negate the decision - learnedLiterals.add(makeLiteral(assignment.variable, isNegated = not assignment.value)) - - # If we found decisions, use them. Otherwise fall back to the conflict clause. - if learnedLiterals.len > 0: - return makeClause(learnedLiterals, reason = "Learned from conflict decision path") - else: - return conflict.clause - -proc findBackjumpLevel*(solver: CDCLSolver, learnedClause: Clause): int = - ## Find the decision level to backjump to - ## This is the second-highest decision level in the learned clause - ## - ## Requirements: 5.3 - Backjump to earliest decision causing conflict - - var levels: seq[int] = @[] - - for literal in learnedClause.literals: - if solver.isAssigned(literal.variable): - let assignment = solver.getAssignment(literal.variable).get() - if assignment.decisionLevel notin levels: - levels.add(assignment.decisionLevel) - - if levels.len == 0: - return 0 - - levels.sort() - - if levels.len == 1: - return max(0, levels[0] - 1) - else: - # Return second-highest level - return levels[levels.len - 2] - -proc backjump*(solver: var CDCLSolver, level: int) = - ## Backjump to a specific decision level - ## Remove all assignments made after that level - ## - ## Requirements: 5.3 - Backjump to earliest decision causing conflict - - var toRemove: seq[BoolVar] = @[] - - for variable, assignment in solver.assignments.pairs: - if assignment.decisionLevel > level: - toRemove.add(variable) - - for variable in toRemove: - solver.unassign(variable) - - solver.decisionLevel = level - solver.propagationQueue = @[] - -# --- Main Solver Loop --- - -proc solve*(solver: var CDCLSolver): SolverResult = - ## Main CDCL solving loop - ## Returns SAT with model if satisfiable, UNSAT with conflict if not - ## - ## Requirements: 5.1, 5.2, 5.3, 5.4, 5.5 - - # Initial unit propagation - let initialConflict = solver.unitPropagate() - if initialConflict.isSome: - # Formula is unsatisfiable at decision level 0 - return SolverResult(isSat: false, conflict: initialConflict.get()) - - # Main CDCL loop - while true: - # Check if all variables are assigned - let unassignedVar = solver.selectUnassignedVariable() - - if unassignedVar.isNone: - # All variables assigned, formula is satisfied! - var model = initTable[BoolVar, bool]() - for variable, assignment in solver.assignments.pairs: - model[variable] = assignment.value - return SolverResult(isSat: true, model: model) - - # Make a decision - solver.decisionLevel += 1 - let variable = unassignedVar.get() - solver.assign(variable, true, Decision) # Try true first - - # Propagate implications - let conflict = solver.unitPropagate() - - if conflict.isSome: - # Conflict detected! - if solver.decisionLevel == 0: - # Conflict at decision level 0 - unsatisfiable - return SolverResult(isSat: false, conflict: conflict.get()) - - # Analyze conflict and learn - let learnedClause = solver.analyzeConflict(conflict.get()) - solver.learnedClauses.add(learnedClause) - - # Backjump - let backjumpLevel = solver.findBackjumpLevel(learnedClause) - solver.backjump(backjumpLevel) - -# --- Solver Construction --- - -proc newCDCLSolver*(formula: CNFFormula): CDCLSolver = - ## Create a new CDCL solver for a CNF formula - result = CDCLSolver( - formula: formula, - assignments: initTable[BoolVar, SolverAssignment](), - decisionLevel: 0, - learnedClauses: @[], - propagationQueue: @[] - ) - -# --- String Representations --- - -proc `$`*(assignment: SolverAssignment): string = - ## String representation of an assignment - result = $assignment.variable & " = " & $assignment.value - result.add(" @" & $assignment.decisionLevel) - if assignment.assignmentType == Decision: - result.add(" (decision)") - else: - result.add(" (implied)") - -proc `$`*(conflict: Conflict): string = - ## String representation of a conflict - result = "Conflict in clause: " & $conflict.clause - -proc `$`*(solverResult: SolverResult): string = - ## String representation of solver result - if solverResult.isSat: - result = "SAT (" & $solverResult.model.len & " variables assigned)" - else: - result = "UNSAT: " & $solverResult.conflict diff --git a/src/nip/resolver/cell_manager.nim b/src/nip/resolver/cell_manager.nim deleted file mode 100644 index 90221a9..0000000 --- a/src/nip/resolver/cell_manager.nim +++ /dev/null @@ -1,498 +0,0 @@ -## Cell Management for Dependency Resolver -## -## This module provides cell management integration for the dependency resolver, -## bridging the resolver's conflict detection with the NipCell system. -## -## **Purpose:** -## - Provide normal cell management operations (not just fallback) -## - Integrate resolver with existing NipCell infrastructure -## - Support cell activation, switching, and removal -## - Clean up cell-specific packages during resolution -## -## **Requirements:** -## - 10.3: Maintain separate dependency graphs per cell -## - 10.4: Support cell switching -## - 10.5: Clean up cell-specific packages -## -## **Architecture:** -## ``` -## ┌─────────────────────────────────────────────────────────────┐ -## │ Resolver Cell Manager │ -## │ ───────────────────────────────────────────────────────── │ -## │ Coordinates resolver with NipCell system │ -## └────────────────────┬────────────────────────────────────────┘ -## │ -## v -## ┌─────────────────────────────────────────────────────────────┐ -## │ Cell Operations │ -## │ ───────────────────────────────────────────────────────── │ -## │ - Activate cell for resolution │ -## │ - Switch between cells │ -## │ - Remove cells and clean up packages │ -## │ - Resolve dependencies within cell context │ -## └─────────────────────────────────────────────────────────────┘ -## ``` - -import std/[tables, sets, options, strformat, times] -import ./nipcell_fallback -import ./dependency_graph -import ./variant_types - -type - ## Cell activation result - CellActivationResult* = object - success*: bool - cellName*: string - previousCell*: Option[string] - packagesAvailable*: int - error*: string - - ## Cell removal result - CellRemovalResult* = object - success*: bool - cellName*: string - packagesRemoved*: int - error*: string - - ## Resolver cell manager - ResolverCellManager* = ref object - graphManager*: NipCellGraphManager - activeResolutions*: Table[string, DependencyGraph] ## Active resolutions per cell - cellPackageCache*: Table[string, HashSet[string]] ## Package cache per cell - -# ============================================================================= -# Cell Manager Construction -# ============================================================================= - -proc newResolverCellManager*(cellRoot: string = ""): ResolverCellManager = - ## Create a new resolver cell manager. - ## - ## **Requirements:** 10.3, 10.4 - Maintain graphs and support switching - - result = ResolverCellManager( - graphManager: newNipCellGraphManager(cellRoot), - activeResolutions: initTable[string, DependencyGraph](), - cellPackageCache: initTable[string, HashSet[string]]() - ) - -# ============================================================================= -# Cell Activation -# ============================================================================= - -proc activateCell*( - manager: ResolverCellManager, - cellName: string -): CellActivationResult = - ## Activate a cell for dependency resolution. - ## - ## **Requirements:** 10.4 - Support cell switching - ## - ## **Effect:** - ## - Switches the active cell - ## - Loads the cell's dependency graph - ## - Makes cell packages available for resolution - ## - ## **Returns:** Activation result with status and details - - # Check if cell exists - if cellName notin manager.graphManager.cells: - return CellActivationResult( - success: false, - cellName: cellName, - previousCell: manager.graphManager.activeCell, - packagesAvailable: 0, - error: fmt"Cell '{cellName}' not found" - ) - - # Get previous cell - let previousCell = manager.graphManager.activeCell - - # Switch to new cell - let switchResult = manager.graphManager.switchCell(cellName) - - if not switchResult.success: - return CellActivationResult( - success: false, - cellName: cellName, - previousCell: previousCell, - packagesAvailable: 0, - error: switchResult.error - ) - - # Load cell packages - let packages = manager.graphManager.getCellPackages(cellName) - - # Update package cache - if cellName notin manager.cellPackageCache: - manager.cellPackageCache[cellName] = initHashSet[string]() - - for pkg in packages: - manager.cellPackageCache[cellName].incl(pkg) - - return CellActivationResult( - success: true, - cellName: cellName, - previousCell: previousCell, - packagesAvailable: packages.len, - error: "" - ) - -proc deactivateCell*(manager: ResolverCellManager): bool = - ## Deactivate the current cell. - ## - ## **Requirements:** 10.4 - Support cell switching - - if manager.graphManager.activeCell.isNone: - return false - - manager.graphManager.activeCell = none(string) - return true - -proc getActiveCellName*(manager: ResolverCellManager): Option[string] = - ## Get the name of the currently active cell. - ## - ## **Requirements:** 10.4 - Support cell switching - - return manager.graphManager.activeCell - -# ============================================================================= -# Cell Switching -# ============================================================================= - -proc switchToCell*( - manager: ResolverCellManager, - cellName: string, - preserveResolution: bool = false -): CellActivationResult = - ## Switch to a different cell. - ## - ## **Requirements:** 10.4 - Support cell switching - ## - ## **Parameters:** - ## - cellName: Name of cell to switch to - ## - preserveResolution: If true, preserve current resolution state - ## - ## **Returns:** Activation result - - # Save current resolution if requested - if preserveResolution and manager.graphManager.activeCell.isSome: - let currentCell = manager.graphManager.activeCell.get() - if currentCell in manager.activeResolutions: - # Resolution is already saved - discard - - # Activate the new cell - return manager.activateCell(cellName) - -proc listAvailableCells*(manager: ResolverCellManager): seq[string] = - ## List all available cells. - ## - ## **Requirements:** 10.4 - Support cell management - - return manager.graphManager.listCells() - -# ============================================================================= -# Cell Removal -# ============================================================================= - -proc removeCell*( - manager: ResolverCellManager, - cellName: string, - cleanupPackages: bool = true -): CellRemovalResult = - ## Remove a cell and optionally clean up its packages. - ## - ## **Requirements:** 10.5 - Clean up cell-specific packages - ## - ## **Parameters:** - ## - cellName: Name of cell to remove - ## - cleanupPackages: If true, remove all cell-specific packages - ## - ## **Returns:** Removal result with status and details - - # Check if cell exists - if cellName notin manager.graphManager.cells: - return CellRemovalResult( - success: false, - cellName: cellName, - packagesRemoved: 0, - error: fmt"Cell '{cellName}' not found" - ) - - # Get packages before removal - let packages = manager.graphManager.getCellPackages(cellName) - let packageCount = packages.len - - # Clean up packages if requested - if cleanupPackages: - for pkg in packages: - discard manager.graphManager.removePackageFromCell(cellName, pkg) - - # Remove from active resolutions - if cellName in manager.activeResolutions: - manager.activeResolutions.del(cellName) - - # Remove from package cache - if cellName in manager.cellPackageCache: - manager.cellPackageCache.del(cellName) - - # Delete the cell - let success = manager.graphManager.deleteCell(cellName) - - if not success: - return CellRemovalResult( - success: false, - cellName: cellName, - packagesRemoved: 0, - error: fmt"Failed to delete cell '{cellName}'" - ) - - return CellRemovalResult( - success: true, - cellName: cellName, - packagesRemoved: packageCount, - error: "" - ) - -# ============================================================================= -# Package Management in Cells -# ============================================================================= - -proc addPackageToActiveCell*( - manager: ResolverCellManager, - packageName: string -): bool = - ## Add a package to the currently active cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if manager.graphManager.activeCell.isNone: - return false - - let cellName = manager.graphManager.activeCell.get() - - # Add to graph manager - let success = manager.graphManager.addPackageToCell(cellName, packageName) - - if success: - # Update cache - if cellName notin manager.cellPackageCache: - manager.cellPackageCache[cellName] = initHashSet[string]() - manager.cellPackageCache[cellName].incl(packageName) - - return success - -proc removePackageFromActiveCell*( - manager: ResolverCellManager, - packageName: string -): bool = - ## Remove a package from the currently active cell. - ## - ## **Requirements:** 10.5 - Clean up cell-specific packages - - if manager.graphManager.activeCell.isNone: - return false - - let cellName = manager.graphManager.activeCell.get() - - # Remove from graph manager - let success = manager.graphManager.removePackageFromCell(cellName, packageName) - - if success: - # Update cache - if cellName in manager.cellPackageCache: - manager.cellPackageCache[cellName].excl(packageName) - - return success - -proc getActiveCellPackages*(manager: ResolverCellManager): seq[string] = - ## Get all packages in the currently active cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if manager.graphManager.activeCell.isNone: - return @[] - - let cellName = manager.graphManager.activeCell.get() - return manager.graphManager.getCellPackages(cellName) - -proc isPackageInActiveCell*( - manager: ResolverCellManager, - packageName: string -): bool = - ## Check if a package is in the currently active cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if manager.graphManager.activeCell.isNone: - return false - - let cellName = manager.graphManager.activeCell.get() - - # Check cache first for performance - if cellName in manager.cellPackageCache: - return packageName in manager.cellPackageCache[cellName] - - # Fall back to graph manager - return manager.graphManager.isPackageInCell(cellName, packageName) - -# ============================================================================= -# Resolution Integration -# ============================================================================= - -proc resolveInCell*( - manager: ResolverCellManager, - cellName: string, - rootPackage: string, - variantDemand: VariantDemand -): Option[DependencyGraph] = - ## Resolve dependencies within a specific cell context. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - ## - ## **Parameters:** - ## - cellName: Name of cell to resolve in - ## - rootPackage: Root package to resolve - ## - variantDemand: Variant requirements - ## - ## **Returns:** Resolved dependency graph or None if resolution fails - - # Check if cell exists - if cellName notin manager.graphManager.cells: - return none(DependencyGraph) - - # Get cell graph - let cellGraphOpt = manager.graphManager.getCellGraph(cellName) - if cellGraphOpt.isNone: - return none(DependencyGraph) - - # TODO: Integrate with actual resolver - # For now, return the cell's existing graph - let cellGraph = cellGraphOpt.get() - return some(cellGraph.graph) - -proc saveResolution*( - manager: ResolverCellManager, - cellName: string, - graph: DependencyGraph -) = - ## Save a resolved dependency graph for a cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - manager.activeResolutions[cellName] = graph - -proc getResolution*( - manager: ResolverCellManager, - cellName: string -): Option[DependencyGraph] = - ## Get the saved resolution for a cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if cellName in manager.activeResolutions: - return some(manager.activeResolutions[cellName]) - return none(DependencyGraph) - -# ============================================================================= -# Cell Information -# ============================================================================= - -proc getCellInfo*( - manager: ResolverCellManager, - cellName: string -): Option[NipCellGraph] = - ## Get detailed information about a cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - return manager.graphManager.getCellGraph(cellName) - -proc getCellStatistics*( - manager: ResolverCellManager, - cellName: string -): tuple[packageCount: int, lastModified: DateTime, created: DateTime] = - ## Get statistics for a cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - let cellOpt = manager.graphManager.getCellGraph(cellName) - - if cellOpt.isNone: - return (packageCount: 0, lastModified: now(), created: now()) - - let cell = cellOpt.get() - return ( - packageCount: cell.packages.len, - lastModified: cell.lastModified, - created: cell.created - ) - -# ============================================================================= -# Cleanup Operations -# ============================================================================= - -proc cleanupUnusedPackages*( - manager: ResolverCellManager, - cellName: string -): int = - ## Clean up packages that are no longer referenced in the cell's graph. - ## - ## **Requirements:** 10.5 - Clean up cell-specific packages - ## - ## **Returns:** Number of packages removed - - let cellOpt = manager.graphManager.getCellGraph(cellName) - if cellOpt.isNone: - return 0 - - let cell = cellOpt.get() - var removedCount = 0 - - # Get packages from graph (packages that are actually used) - var usedPackages = initHashSet[string]() - for term in cell.graph.terms.values: - usedPackages.incl(term.packageName) - - # Find packages in cell that aren't in the graph - for pkg in cell.packages: - if pkg notin usedPackages: - if manager.graphManager.removePackageFromCell(cellName, pkg): - removedCount += 1 - - return removedCount - -proc cleanupAllCells*(manager: ResolverCellManager): Table[string, int] = - ## Clean up unused packages in all cells. - ## - ## **Requirements:** 10.5 - Clean up cell-specific packages - ## - ## **Returns:** Map of cell name to number of packages removed - - var results = initTable[string, int]() - - for cellName in manager.graphManager.listCells(): - let removed = manager.cleanupUnusedPackages(cellName) - if removed > 0: - results[cellName] = removed - - return results - -# ============================================================================= -# String Representation -# ============================================================================= - -proc `$`*(manager: ResolverCellManager): string = - ## String representation for debugging. - - let cellCount = manager.graphManager.listCells().len - let activeCell = if manager.graphManager.activeCell.isSome: - manager.graphManager.activeCell.get() - else: - "none" - - result = "ResolverCellManager(\n" - result &= fmt" cells: {cellCount}\n" - result &= fmt" active: {activeCell}\n" - result &= fmt" resolutions: {manager.activeResolutions.len}\n" - result &= ")" diff --git a/src/nip/resolver/cnf_translator.nim b/src/nip/resolver/cnf_translator.nim deleted file mode 100644 index 17b071e..0000000 --- a/src/nip/resolver/cnf_translator.nim +++ /dev/null @@ -1,430 +0,0 @@ -## CNF Translation for Dependency Resolution -## -## This module translates dependency constraints into Conjunctive Normal Form (CNF) -## for use with CDCL-based SAT solving. -## -## Philosophy: -## - Each package+version+variant combination is a boolean variable -## - Dependencies become implication clauses (A → B ≡ ¬A ∨ B) -## - Exclusivity becomes mutual exclusion clauses (¬(A ∧ B) ≡ ¬A ∨ ¬B) -## - Variant satisfaction becomes satisfaction clauses -## -## Key Concepts: -## - A CNF formula is a conjunction of disjunctions (AND of ORs) -## - Each clause is a disjunction of literals (OR of variables/negations) -## - The solver finds an assignment that satisfies all clauses - -import std/[tables, sets, hashes, options] -import ./solver_types -import ./variant_types -import ../manifest_parser - -type - ## A boolean variable representing a specific package+version+variant - ## This is the atomic unit of the CNF formula - BoolVar* = object - package*: PackageId - version*: SemanticVersion - variant*: VariantProfile - - ## A literal is a boolean variable or its negation - Literal* = object - variable*: BoolVar - isNegated*: bool - - ## A clause is a disjunction of literals (OR) - ## Example: (¬A ∨ B ∨ ¬C) means "NOT A OR B OR NOT C" - Clause* = object - literals*: seq[Literal] - reason*: string # Human-readable explanation - - ## A CNF formula is a conjunction of clauses (AND) - ## Example: (A ∨ B) ∧ (¬A ∨ C) means "(A OR B) AND (NOT A OR C)" - CNFFormula* = object - clauses*: seq[Clause] - variables*: Table[BoolVar, int] # Variable → unique ID - nextVarId*: int - - ## The type of clause (for debugging and error reporting) - ClauseKind* = enum - DependencyClause, ## A → B (dependency implication) - ExclusivityClause, ## ¬(A ∧ B) (mutual exclusion) - SatisfactionClause, ## Variant requirements - RootClause ## User requirements - -# --- BoolVar Operations --- - -proc `==`*(a, b: BoolVar): bool = - ## Equality for boolean variables - result = a.package == b.package and - a.version == b.version and - a.variant.hash == b.variant.hash - -proc hash*(v: BoolVar): Hash = - ## Hash function for boolean variables - var h: Hash = 0 - h = h !& hash(v.package) - h = h !& hash($v.version) - h = h !& hash(v.variant.hash) - result = !$h - -proc `$`*(v: BoolVar): string = - ## String representation of a boolean variable - result = v.package & "=" & $v.version - if v.variant.domains.len > 0: - result.add(" [" & v.variant.hash & "]") - -# --- Literal Operations --- - -proc makeLiteral*(variable: BoolVar, isNegated: bool = false): Literal = - ## Create a literal from a boolean variable - result = Literal(variable: variable, isNegated: isNegated) - -proc negate*(lit: Literal): Literal = - ## Negate a literal - result = Literal(variable: lit.variable, isNegated: not lit.isNegated) - -proc `$`*(lit: Literal): string = - ## String representation of a literal - if lit.isNegated: - result = "¬" & $lit.variable - else: - result = $lit.variable - -# --- Clause Operations --- - -proc makeClause*(literals: seq[Literal], reason: string = ""): Clause = - ## Create a clause from literals - result = Clause(literals: literals, reason: reason) - -proc `$`*(clause: Clause): string = - ## String representation of a clause - result = "(" - for i, lit in clause.literals: - if i > 0: - result.add(" ∨ ") - result.add($lit) - result.add(")") - if clause.reason.len > 0: - result.add(" [" & clause.reason & "]") - -# --- CNF Formula Operations --- - -proc newCNFFormula*(): CNFFormula = - ## Create a new empty CNF formula - result = CNFFormula( - clauses: @[], - variables: initTable[BoolVar, int](), - nextVarId: 1 - ) - -proc getOrCreateVarId*(formula: var CNFFormula, variable: BoolVar): int = - ## Get or create a unique ID for a boolean variable - if formula.variables.hasKey(variable): - return formula.variables[variable] - else: - let id = formula.nextVarId - formula.variables[variable] = id - formula.nextVarId += 1 - return id - -proc addClause*(formula: var CNFFormula, clause: Clause) = - ## Add a clause to the CNF formula - formula.clauses.add(clause) - -proc `$`*(formula: CNFFormula): string = - ## String representation of a CNF formula - result = "CNF Formula (" & $formula.clauses.len & " clauses, " & - $formula.variables.len & " variables):\n" - for i, clause in formula.clauses: - result.add(" " & $i & ": " & $clause & "\n") - -# --- CNF Translation Functions --- - -proc termToBoolVar*(term: Term, version: SemanticVersion): BoolVar = - ## Convert a term to a boolean variable - ## This creates a specific package+version+variant combination - result = BoolVar( - package: term.package, - version: version, - variant: term.constraint.variantReq - ) - -proc translateDependency*( - formula: var CNFFormula, - dependent: PackageId, - dependentVersion: SemanticVersion, - dependentVariant: VariantProfile, - dependency: PackageId, - dependencyVersion: SemanticVersion, - dependencyVariant: VariantProfile -): Clause = - ## Translate a dependency into a CNF clause - ## "A depends on B" becomes "¬A ∨ B" (if A then B) - ## - ## Requirements: 6.2 - WHEN encoding dependencies THEN the system SHALL create implication clauses (A → B) - - let varA = BoolVar( - package: dependent, - version: dependentVersion, - variant: dependentVariant - ) - - let varB = BoolVar( - package: dependency, - version: dependencyVersion, - variant: dependencyVariant - ) - - # Register variables - discard formula.getOrCreateVarId(varA) - discard formula.getOrCreateVarId(varB) - - # Create clause: ¬A ∨ B - let clause = makeClause( - @[ - makeLiteral(varA, isNegated = true), # ¬A - makeLiteral(varB, isNegated = false) # B - ], - reason = dependent & " " & $dependentVersion & " depends on " & - dependency & " " & $dependencyVersion - ) - - formula.addClause(clause) - return clause - -proc translateExclusivity*( - formula: var CNFFormula, - packageA: PackageId, - versionA: SemanticVersion, - variantA: VariantProfile, - packageB: PackageId, - versionB: SemanticVersion, - variantB: VariantProfile, - reason: string = "" -): Clause = - ## Translate mutual exclusion into a CNF clause - ## "A and B are mutually exclusive" becomes "¬A ∨ ¬B" (not both) - ## - ## Requirements: 6.3 - WHEN encoding exclusivity THEN the system SHALL create mutual exclusion clauses (¬(A ∧ B)) - - let varA = BoolVar( - package: packageA, - version: versionA, - variant: variantA - ) - - let varB = BoolVar( - package: packageB, - version: versionB, - variant: variantB - ) - - # Register variables - discard formula.getOrCreateVarId(varA) - discard formula.getOrCreateVarId(varB) - - # Create clause: ¬A ∨ ¬B - let clause = makeClause( - @[ - makeLiteral(varA, isNegated = true), # ¬A - makeLiteral(varB, isNegated = true) # ¬B - ], - reason = if reason.len > 0: reason else: "Mutually exclusive: " & - packageA & " and " & packageB - ) - - formula.addClause(clause) - return clause - -proc translateVariantSatisfaction*( - formula: var CNFFormula, - package: PackageId, - version: SemanticVersion, - requiredVariant: VariantProfile, - availableVariant: VariantProfile -): Clause = - ## Translate variant satisfaction into a CNF clause - ## "If we select this package, its variant must satisfy requirements" - ## - ## Requirements: 6.4 - WHEN encoding variant satisfaction THEN the system SHALL create satisfaction clauses - - let varRequired = BoolVar( - package: package, - version: version, - variant: requiredVariant - ) - - let varAvailable = BoolVar( - package: package, - version: version, - variant: availableVariant - ) - - # Register variables - discard formula.getOrCreateVarId(varRequired) - discard formula.getOrCreateVarId(varAvailable) - - # Check if available variant satisfies required variant - # For now, we check if all required domains/flags are present - var satisfies = true - for domain, variantDomain in requiredVariant.domains.pairs: - if not availableVariant.domains.hasKey(domain): - satisfies = false - break - - for flag in variantDomain.flags: - if flag notin availableVariant.domains[domain].flags: - satisfies = false - break - - if satisfies: - # If available satisfies required, create: ¬required ∨ available - # Meaning: if we need required, we can use available - let clause = makeClause( - @[ - makeLiteral(varRequired, isNegated = true), - makeLiteral(varAvailable, isNegated = false) - ], - reason = "Variant " & availableVariant.hash & " satisfies " & requiredVariant.hash - ) - formula.addClause(clause) - return clause - else: - # If available doesn't satisfy required, create: ¬required ∨ ¬available - # Meaning: we can't have both (they're incompatible) - let clause = makeClause( - @[ - makeLiteral(varRequired, isNegated = true), - makeLiteral(varAvailable, isNegated = true) - ], - reason = "Variant " & availableVariant.hash & " does not satisfy " & requiredVariant.hash - ) - formula.addClause(clause) - return clause - -proc translateRootRequirement*( - formula: var CNFFormula, - package: PackageId, - version: SemanticVersion, - variant: VariantProfile -): Clause = - ## Translate a root requirement into a CNF clause - ## "User requires package P" becomes "P" (unit clause) - ## - ## Requirements: 6.1 - WHEN translating to CNF THEN the system SHALL create boolean variables for each term - - let variable = BoolVar( - package: package, - version: version, - variant: variant - ) - - # Register variable - discard formula.getOrCreateVarId(variable) - - # Create unit clause: P - let clause = makeClause( - @[makeLiteral(variable, isNegated = false)], - reason = "User requires " & package & " " & $version - ) - - formula.addClause(clause) - return clause - -proc translateIncompatibility*(formula: var CNFFormula, incomp: Incompatibility): Clause = - ## Translate an incompatibility into a CNF clause - ## An incompatibility ¬(T1 ∧ T2 ∧ ... ∧ Tn) becomes (¬T1 ∨ ¬T2 ∨ ... ∨ ¬Tn) - ## - ## This is the general translation that handles all incompatibility types - - var literals: seq[Literal] = @[] - - for term in incomp.terms: - # For each term in the incompatibility, we need to create a literal - # The term already has a constraint with version and variant - # We need to pick a specific version that satisfies the constraint - - # We create a boolean variable representing the term's constraint - # In a full implementation, this would map to specific package versions - # For now, we use the term's constraint as the identity of the variable - - # We need to ensure we have a valid version for the BoolVar - # Since Incompatibility terms might be ranges, we might need a different approach - # or map to a specific representative version. - - # For this MVP, we'll assume the term maps to a specific "decision" variable - # that the solver is tracking. - - let variable = BoolVar( - package: term.package, - version: parseSemanticVersion("0.0.0"), # Placeholder/Any - variant: term.constraint.variantReq - ) - - discard formula.getOrCreateVarId(variable) - - # If term is positive (P satisfies C), then in the incompatibility ¬(P satisfies C), - # we want ¬(Variable). So we add ¬Variable to the clause. - # If term is negative (P satisfies NOT C), then in the incompatibility ¬(P satisfies NOT C), - # we want ¬(¬Variable) = Variable. So we add Variable to the clause. - - let isNegated = term.isPositive - literals.add(makeLiteral(variable, isNegated = isNegated)) - - let clause = makeClause(literals, reason = incomp.externalContext) - formula.addClause(clause) - return clause - -import ./dependency_graph - -proc translateGraph*(formula: var CNFFormula, graph: DependencyGraph) = - ## Translate a dependency graph into CNF clauses - ## This converts the graph structure (nodes and edges) into boolean logic - - for termId, term in graph.terms.pairs: - # 1. Create variable for each term - let version = term.version - - let variable = BoolVar( - package: term.packageName, - version: version, - variant: term.variantProfile - ) - discard formula.getOrCreateVarId(variable) - - # 2. Translate dependencies (Edges) - # A -> B becomes ¬A ∨ B - for edge in graph.getOutgoingEdges(termId): - let depTerm = graph.terms[edge.toTerm] - let depVersion = depTerm.version - - discard translateDependency( - formula, - term.packageName, version, term.variantProfile, - depTerm.packageName, depVersion, depTerm.variantProfile - ) - -# --- Validation --- - -proc isValidCNF*(formula: CNFFormula): bool = - ## Validate that the CNF formula is well-formed - ## - ## Requirements: 6.5 - WHEN CNF is complete THEN the system SHALL be ready for CDCL solving - - # Check that we have at least one clause - if formula.clauses.len == 0: - return false - - # Check that each clause has at least one literal - for clause in formula.clauses: - if clause.literals.len == 0: - return false - - # Check that all variables in clauses are registered - for clause in formula.clauses: - for literal in clause.literals: - if not formula.variables.hasKey(literal.variable): - return false - - return true diff --git a/src/nip/resolver/conflict_detection.nim b/src/nip/resolver/conflict_detection.nim deleted file mode 100644 index 6351178..0000000 --- a/src/nip/resolver/conflict_detection.nim +++ /dev/null @@ -1,469 +0,0 @@ -## Conflict Detection for Dependency Resolution -## -## This module implements specific conflict detection for the NIP dependency resolver. -## It detects and reports various types of conflicts that can occur during resolution: -## - Version conflicts: Incompatible version requirements -## - Variant conflicts: Incompatible variant flags -## - Circular dependencies: Cycles in the dependency graph -## - Missing packages: Packages not found in any source -## -## Philosophy: -## - Detect conflicts early and specifically -## - Provide actionable error messages -## - Suggest solutions when possible -## - Track conflict origins for debugging -## -## Requirements: -## - 7.1: Report version conflicts with incompatible version requirements -## - 7.2: Report variant conflicts with incompatible variant flags -## - 7.3: Report circular dependencies with cycle path -## - 7.4: Report missing packages with suggestions -## - 7.5: Provide actionable suggestions for resolution - -import std/[tables, sets, options, sequtils, algorithm, strutils, strformat] -import ./solver_types -import ./variant_types -import ../manifest_parser - -type - ## The kind of conflict detected - ConflictKind* = enum - VersionConflict, ## Incompatible version requirements - VariantConflict, ## Incompatible variant flags - CircularDependency, ## Cycle in dependency graph - MissingPackage, ## Package not found in any source - BuildHashMismatch ## Installed build doesn't match required - - ## Detailed information about a conflict - ConflictReport* = object - kind*: ConflictKind - packages*: seq[string] ## Packages involved in the conflict - details*: string ## Detailed description of the conflict - suggestions*: seq[string] ## Actionable suggestions for resolution - conflictingTerms*: seq[Term] ## The specific terms that conflict - cyclePath*: Option[seq[string]] ## For circular dependencies: the cycle path - -# --- Version Conflict Detection --- - -proc detectVersionConflict*( - package: PackageId, - constraints: seq[VersionConstraint] -): Option[ConflictReport] = - ## Detect if a set of version constraints are incompatible - ## - ## Requirements: 7.1 - Report version conflicts with incompatible version requirements - ## - ## Returns a ConflictReport if the constraints are incompatible, None otherwise - - if constraints.len < 2: - return none(ConflictReport) - - # Check if all constraints can be satisfied simultaneously - # For now, we use a simple approach: check if any two constraints are incompatible - - for i in 0 ..< constraints.len: - for j in (i + 1) ..< constraints.len: - let constraint1 = constraints[i] - let constraint2 = constraints[j] - - # Check if these constraints can both be satisfied - # This is a simplified check - a full implementation would need - # proper semantic version range intersection logic - - case constraint1.operator: - of OpExact: - # Exact version must match - case constraint2.operator: - of OpExact: - if constraint1.version != constraint2.version: - return some(ConflictReport( - kind: VersionConflict, - packages: @[package], - details: fmt"Package '{package}' has conflicting exact version requirements: {constraint1.version} and {constraint2.version}", - suggestions: @[ - fmt"Check which packages require {package} {constraint1.version}", - fmt"Check which packages require {package} {constraint2.version}", - "Consider using a version that satisfies both requirements", - "Or use NipCell isolation to install different versions in separate environments" - ], - conflictingTerms: @[], - cyclePath: none(seq[string]) - )) - of OpGreaterEq: - if constraint1.version < constraint2.version: - return some(ConflictReport( - kind: VersionConflict, - packages: @[package], - details: fmt"Package '{package}' requires exact version {constraint1.version} but also requires >= {constraint2.version}", - suggestions: @[ - fmt"Update requirement to {constraint2.version} or later", - "Check if {constraint1.version} is still maintained", - "Consider upgrading to a newer version" - ], - conflictingTerms: @[], - cyclePath: none(seq[string]) - )) - else: - discard # Other operators would need more complex logic - else: - discard # Other operators would need more complex logic - - return none(ConflictReport) - -# --- Variant Conflict Detection --- - -proc detectVariantConflict*( - package: PackageId, - demands: seq[VariantDemand] -): Option[ConflictReport] = - ## Detect if a set of variant demands are incompatible - ## - ## Requirements: 7.2 - Report variant conflicts with incompatible variant flags - ## - ## Returns a ConflictReport if the demands are incompatible, None otherwise - - if demands.len < 2: - return none(ConflictReport) - - # Check for exclusive domain conflicts - var domainValues: Table[string, seq[string]] = initTable[string, seq[string]]() - - for demand in demands: - for domain, variantDomain in demand.variantProfile.domains.pairs: - if domain notin domainValues: - domainValues[domain] = @[] - - for flag in variantDomain.flags: - if flag notin domainValues[domain]: - domainValues[domain].add(flag) - - # Check for conflicts in exclusive domains - for domain, values in domainValues.pairs: - # Check if this is an exclusive domain (by checking first demand) - var isExclusive = false - for demand in demands: - if domain in demand.variantProfile.domains: - isExclusive = demand.variantProfile.domains[domain].exclusivity == Exclusive - break - - if isExclusive and values.len > 1: - # Exclusive domain has multiple values - conflict! - let conflictingDemands = demands.filterIt(domain in it.variantProfile.domains) - let valuesList = values.join(", ") - - return some(ConflictReport( - kind: VariantConflict, - packages: @[package], - details: fmt"Package '{package}' has conflicting exclusive variant flags in domain '{domain}': {valuesList}", - suggestions: @[ - fmt"Choose one of the conflicting values: {valuesList}", - "Check which packages require each variant", - "Consider using NipCell isolation to install different variants in separate environments", - "Or rebuild the package with a compatible variant" - ], - conflictingTerms: @[], - cyclePath: none(seq[string]) - )) - - return none(ConflictReport) - -# --- Circular Dependency Detection --- - -proc detectCircularDependency*( - graph: Table[PackageId, seq[PackageId]], - startPackage: PackageId -): Option[ConflictReport] = - ## Detect if there is a circular dependency starting from a package - ## - ## Requirements: 7.3 - Report circular dependencies with cycle path - ## - ## Returns a ConflictReport with the cycle path if a cycle is found, None otherwise - - var visited: HashSet[PackageId] = initHashSet[PackageId]() - var recursionStack: HashSet[PackageId] = initHashSet[PackageId]() - var path: seq[PackageId] = @[] - - proc dfs(package: PackageId): Option[seq[PackageId]] = - visited.incl(package) - recursionStack.incl(package) - path.add(package) - - if package in graph: - for dependency in graph[package]: - if dependency notin visited: - let cyclePath = dfs(dependency) - if cyclePath.isSome: - return cyclePath - elif dependency in recursionStack: - # Found a cycle! - let cycleStart = path.find(dependency) - if cycleStart >= 0: - let cycle = path[cycleStart..^1] & @[dependency] - return some(cycle) - - discard path.pop() - recursionStack.excl(package) - return none(seq[PackageId]) - - let cyclePath = dfs(startPackage) - - if cyclePath.isSome: - let cycle = cyclePath.get() - let cycleStr = cycle.join(" -> ") - return some(ConflictReport( - kind: CircularDependency, - packages: cycle, - details: fmt"Circular dependency detected: {cycleStr}", - suggestions: @[ - "Break the cycle by removing or modifying one of the dependencies", - "Check if any dependencies are optional and can be made optional", - "Consider splitting the package into smaller packages", - "Review the dependency declarations for correctness" - ], - conflictingTerms: @[], - cyclePath: cyclePath - )) - - return none(ConflictReport) - -# --- Missing Package Detection --- - -proc detectMissingPackage*( - package: PackageId, - availablePackages: HashSet[PackageId] -): Option[ConflictReport] = - ## Detect if a required package is missing from all sources - ## - ## Requirements: 7.4 - Report missing packages with suggestions - ## - ## Returns a ConflictReport if the package is missing, None otherwise - - if package in availablePackages: - return none(ConflictReport) - - # Find similar package names for suggestions - var suggestions: seq[string] = @[] - - # Simple similarity check: packages with similar names - let packageLower = package.toLowerAscii() - var similarPackages: seq[string] = @[] - - for available in availablePackages: - let availableLower = available.toLowerAscii() - - # Check for substring matches or similar names - if availableLower.contains(packageLower) or packageLower.contains(availableLower): - similarPackages.add(available) - - # Check for edit distance (simple check for typos) - if abs(available.len - package.len) <= 2: - var matches = 0 - for i in 0 ..< min(available.len, package.len): - if available[i] == package[i]: - matches += 1 - - if matches >= min(available.len, package.len) - 2: - similarPackages.add(available) - - # Build suggestions - suggestions.add(fmt"Package '{package}' not found in any configured repository") - - if similarPackages.len > 0: - let similarStr = similarPackages.join(", ") - suggestions.add(fmt"Did you mean: {similarStr}?") - - suggestions.add("Check if the package name is spelled correctly") - suggestions.add("Check if the package is available in your configured repositories") - suggestions.add("Try updating your package repository metadata") - suggestions.add("Check if the package has been renamed or moved") - - return some(ConflictReport( - kind: MissingPackage, - packages: @[package], - details: fmt"Package '{package}' not found in any source", - suggestions: suggestions, - conflictingTerms: @[], - cyclePath: none(seq[string]) - )) - -# --- Build Hash Mismatch Detection --- - -proc detectBuildHashMismatch*( - package: PackageId, - expectedHash: string, - actualHash: string -): Option[ConflictReport] = - ## Detect if an installed package's build hash doesn't match the expected hash - ## - ## Requirements: 7.5 - Provide actionable suggestions for resolution - ## - ## Returns a ConflictReport if hashes don't match, None otherwise - - if expectedHash == actualHash: - return none(ConflictReport) - - return some(ConflictReport( - kind: BuildHashMismatch, - packages: @[package], - details: fmt"Package '{package}' build hash mismatch: expected {expectedHash}, got {actualHash}", - suggestions: @[ - "The installed package may have been modified or corrupted", - "Try reinstalling the package", - "Check if the package source has changed", - "Verify the integrity of your package cache", - "Consider running 'nip verify' to check all packages" - ], - conflictingTerms: @[], - cyclePath: none(seq[string]) - )) - -# --- Conflict Reporting --- - -proc formatConflict*(report: ConflictReport): string = - ## Format a conflict report as a human-readable error message - ## - ## Requirements: 7.1, 7.2, 7.3, 7.4, 7.5 - - result = "" - - case report.kind: - of VersionConflict: - result = fmt""" -❌ [VersionConflict] Cannot satisfy conflicting version requirements -🔍 Context: {report.details} -💡 Suggestions:""" - for suggestion in report.suggestions: - result.add(fmt"\n • {suggestion}") - - of VariantConflict: - result = fmt""" -❌ [VariantConflict] Cannot unify conflicting variant demands -🔍 Context: {report.details} -💡 Suggestions:""" - for suggestion in report.suggestions: - result.add(fmt"\n • {suggestion}") - - of CircularDependency: - result = fmt""" -❌ [CircularDependency] Circular dependency detected -🔍 Context: {report.details} -💡 Suggestions:""" - for suggestion in report.suggestions: - result.add(fmt"\n • {suggestion}") - - of MissingPackage: - result = fmt""" -❌ [MissingPackage] Package not found -🔍 Context: {report.details} -💡 Suggestions:""" - for suggestion in report.suggestions: - result.add(fmt"\n • {suggestion}") - - of BuildHashMismatch: - result = fmt""" -❌ [BuildHashMismatch] Build hash verification failed -🔍 Context: {report.details} -💡 Suggestions:""" - for suggestion in report.suggestions: - result.add(fmt"\n • {suggestion}") - - return result - -# --- Conflict Extraction --- - -proc extractMinimalConflict*( - incompatibilities: seq[Incompatibility] -): Option[seq[Incompatibility]] = - ## Extract the minimal set of incompatibilities that cause a conflict - ## - ## Requirements: 7.5 - Provide minimal conflicting requirements - ## - ## This removes redundant incompatibilities to show only the essential conflict. - ## Uses a greedy algorithm to find a minimal unsatisfiable core (MUC). - ## - ## Algorithm: - ## 1. Start with all incompatibilities - ## 2. Try removing each incompatibility one at a time - ## 3. If the remaining set is still unsatisfiable, keep it removed - ## 4. Repeat until no more incompatibilities can be removed - ## - ## This is a greedy approximation of the MUC problem (which is NP-hard). - ## For practical purposes, this gives good results quickly. - - if incompatibilities.len == 0: - return none(seq[Incompatibility]) - - if incompatibilities.len == 1: - return some(incompatibilities) - - # Start with all incompatibilities - var minimal = incompatibilities - var changed = true - - # Iteratively try to remove incompatibilities - while changed: - changed = false - var i = 0 - - while i < minimal.len: - # Try removing incompatibility at index i - let candidate = minimal[0 ..< i] & minimal[(i + 1) ..< minimal.len] - - # Check if the candidate set is still unsatisfiable - # For now, we use a simple heuristic: if there are still conflicting terms, - # the set is likely still unsatisfiable - - # Collect all packages mentioned in the candidate set - var packages: HashSet[string] = initHashSet[string]() - for incomp in candidate: - for term in incomp.terms: - packages.incl(term.package) - - # If we still have packages with conflicting requirements, keep the candidate - # This is a simplified check - a full implementation would need to re-solve - if packages.len > 0: - minimal = candidate - changed = true - break - - i += 1 - - return some(minimal) - -# --- Conflict Analysis --- - -proc analyzeConflictOrigins*( - report: ConflictReport, - packageManifests: Table[PackageId, seq[VariantDemand]] -): seq[string] = - ## Analyze the origins of a conflict and provide detailed context - ## - ## Requirements: 7.5 - Provide actionable suggestions - - var analysis: seq[string] = @[] - - case report.kind: - of VersionConflict: - for package in report.packages: - if package in packageManifests: - analysis.add(fmt"Package '{package}' has {packageManifests[package].len} version demands") - - of VariantConflict: - for package in report.packages: - if package in packageManifests: - analysis.add(fmt"Package '{package}' has {packageManifests[package].len} variant demands") - - of CircularDependency: - if report.cyclePath.isSome: - let cycle = report.cyclePath.get() - let cycleStr = cycle.join(" -> ") - analysis.add(fmt"Cycle involves {cycle.len} packages: {cycleStr}") - - of MissingPackage: - analysis.add(fmt"Package '{report.packages[0]}' is required but not available") - - of BuildHashMismatch: - analysis.add(fmt"Package '{report.packages[0]}' integrity check failed") - - return analysis - diff --git a/src/nip/resolver/dependency_graph.nim b/src/nip/resolver/dependency_graph.nim deleted file mode 100644 index 7f165ce..0000000 --- a/src/nip/resolver/dependency_graph.nim +++ /dev/null @@ -1,328 +0,0 @@ -## Dependency Graph - Core data structure for package dependencies -## -## This module implements the dependency graph used by the resolver to track -## package dependencies, detect cycles, and calculate installation order. - -import tables -import sets -import sequtils -import strutils -import options -import ./variant_types -import ../manifest_parser - -# ============================================================================ -# Type Definitions -# ============================================================================ - -type - PackageTermId* = string - ## Unique identifier for a package term - - DependencyType* = enum - Required, - Optional - - PackageTerm* = object - ## A specific package + version + variant combination - id*: PackageTermId - packageName*: string - version*: SemanticVersion - variantHash*: string # xxh4-128 hash of variant profile - variantProfile*: VariantProfile - optional*: bool - source*: string - - DependencyEdge* = object - ## An edge from one package to its dependency - fromTerm*: PackageTermId - toTerm*: PackageTermId - dependencyType*: DependencyType - constraint*: string # Version constraint (e.g. ">=1.0.0") - - DependencyGraph* = object - ## The complete dependency graph - terms*: Table[PackageTermId, PackageTerm] - edges*: seq[DependencyEdge] - incomingEdges*: Table[PackageTermId, seq[DependencyEdge]] - outgoingEdges*: Table[PackageTermId, seq[DependencyEdge]] - - GraphStats* = object - ## Statistics about the dependency graph - terms*: int - edges*: int - roots*: int - leaves*: int - maxDepth*: int - hasCycle*: bool - -# ============================================================================ -# Helper Functions -# ============================================================================ - -proc createTermId*(packageName, variantHash: string): PackageTermId = - ## Create a term ID from components - result = packageName & ":" & variantHash - -proc termKey*(term: PackageTerm): PackageTermId = - ## Generate unique key for a term - result = term.id - -proc `==`*(a, b: PackageTerm): bool = - ## Compare two terms - result = a.id == b.id - -# ============================================================================ -# Graph Operations -# ============================================================================ - -proc newDependencyGraph*(): DependencyGraph = - ## Create an empty dependency graph - result = DependencyGraph( - terms: initTable[PackageTermId, PackageTerm](), - edges: @[], - incomingEdges: initTable[PackageTermId, seq[DependencyEdge]](), - outgoingEdges: initTable[PackageTermId, seq[DependencyEdge]]() - ) - -proc addTerm*(graph: var DependencyGraph, term: PackageTerm) = - ## Add a term to the graph - if term.id notin graph.terms: - graph.terms[term.id] = term - graph.incomingEdges[term.id] = @[] - graph.outgoingEdges[term.id] = @[] - -proc addEdge*(graph: var DependencyGraph, edge: DependencyEdge) = - ## Add an edge to the graph - # Ensure both nodes exist (should be added before edge) - if edge.fromTerm notin graph.terms or edge.toTerm notin graph.terms: - # In a robust system we might raise error or auto-add - return - - # Add to edge lists - graph.edges.add(edge) - graph.outgoingEdges[edge.fromTerm].add(edge) - graph.incomingEdges[edge.toTerm].add(edge) - -proc getTerm*(graph: DependencyGraph, termId: PackageTermId): Option[PackageTerm] = - ## Get a term by ID - if termId in graph.terms: - return some(graph.terms[termId]) - else: - return none(PackageTerm) - -proc getIncomingEdges*(graph: DependencyGraph, termId: PackageTermId): seq[DependencyEdge] = - ## Get all edges pointing to this node - if termId in graph.incomingEdges: - result = graph.incomingEdges[termId] - else: - result = @[] - -proc getOutgoingEdges*(graph: DependencyGraph, termId: PackageTermId): seq[DependencyEdge] = - ## Get all edges from this node - if termId in graph.outgoingEdges: - result = graph.outgoingEdges[termId] - else: - result = @[] - -proc getDependencies*(graph: DependencyGraph, termId: PackageTermId): seq[PackageTerm] = - ## Get all direct dependencies of a package - let edges = graph.getOutgoingEdges(termId) - result = edges.mapIt(graph.terms[it.toTerm]) - -proc getDependents*(graph: DependencyGraph, termId: PackageTermId): seq[PackageTerm] = - ## Get all packages that depend on this one - let edges = graph.getIncomingEdges(termId) - result = edges.mapIt(graph.terms[it.fromTerm]) - -# ============================================================================ -# Cycle Detection -# ============================================================================ - -proc hasCycle*(graph: DependencyGraph): bool = - ## Check if the graph has any cycles using DFS - var visited = initHashSet[PackageTermId]() - var recursionStack = initHashSet[PackageTermId]() - - proc dfs(key: PackageTermId): bool = - visited.incl(key) - recursionStack.incl(key) - - if key in graph.outgoingEdges: - for edge in graph.outgoingEdges[key]: - let targetKey = edge.toTerm - if targetKey notin visited: - if dfs(targetKey): - return true - elif targetKey in recursionStack: - return true - - recursionStack.excl(key) - return false - - for key in graph.terms.keys: - if key notin visited: - if dfs(key): - return true - - return false - -proc findCycle*(graph: DependencyGraph): seq[PackageTerm] = - ## Find a cycle in the graph (if one exists) - var visited = initHashSet[PackageTermId]() - var recursionStack = initHashSet[PackageTermId]() - var path: seq[PackageTerm] = @[] - - proc dfs(key: PackageTermId): seq[PackageTerm] = - visited.incl(key) - recursionStack.incl(key) - path.add(graph.terms[key]) - - if key in graph.outgoingEdges: - for edge in graph.outgoingEdges[key]: - let targetKey = edge.toTerm - if targetKey notin visited: - let cycle = dfs(targetKey) - if cycle.len > 0: - return cycle - elif targetKey in recursionStack: - # Found cycle - extract it from path - var cycleStart = 0 - for i in 0.. 0: - return cycle - - return @[] - -# ============================================================================ -# Graph Analysis -# ============================================================================ - -proc topologicalSort*(graph: DependencyGraph): seq[PackageTermId] = - ## Perform topological sort on the graph - ## Returns a sequence of term IDs in topological order - ## Raises ValueError if cycle is detected - - var visited = initHashSet[PackageTermId]() - var recursionStack = initHashSet[PackageTermId]() - var resultSeq: seq[PackageTermId] = @[] - - proc dfs(termId: PackageTermId) = - visited.incl(termId) - recursionStack.incl(termId) - - if termId in graph.outgoingEdges: - for edge in graph.outgoingEdges[termId]: - let targetId = edge.toTerm - if targetId notin visited: - dfs(targetId) - elif targetId in recursionStack: - raise newException(ValueError, "Cycle detected during topological sort") - - recursionStack.excl(termId) - resultSeq.add(termId) - - for termId in graph.terms.keys: - if termId notin visited: - dfs(termId) - - # Reverse to get correct order (dependencies first? No, topological sort usually gives dependencies last if using this DFS) - # Wait, standard DFS post-order gives reverse topological sort. - # So if A -> B, B finishes first, then A. Result: B, A. - # If we want installation order (dependencies first), we want B, A. - # So this resultSeq is already in installation order (reverse topological sort). - # Wait, topological sort of A -> B is A, B. - # Installation order for A -> B (A depends on B) is B, A. - # So we want B, A. - # DFS post-order: B is visited, finishes. Added to result. A is visited, calls B (visited), finishes. Added to result. - # Result: B, A. - # So this IS the installation order. - - return resultSeq - -proc nodeCount*(graph: DependencyGraph): int = - ## Get the number of nodes/terms in the graph - result = graph.terms.len - -proc edgeCount*(graph: DependencyGraph): int = - ## Get the number of edges in the graph - result = graph.edges.len - -proc getRoots*(graph: DependencyGraph): seq[PackageTerm] = - ## Get all root nodes (nodes with no incoming edges) - result = @[] - for term in graph.terms.values: - if graph.getIncomingEdges(term.id).len == 0: - result.add(term) - -proc getLeaves*(graph: DependencyGraph): seq[PackageTerm] = - ## Get all leaf nodes (nodes with no outgoing edges) - result = @[] - for term in graph.terms.values: - if graph.getOutgoingEdges(term.id).len == 0: - result.add(term) - -proc getDepth*(graph: DependencyGraph, termId: PackageTermId): int = - ## Calculate the depth of a node (longest path from root) - var visited = initHashSet[PackageTermId]() - - proc dfs(currentId: PackageTermId): int = - if currentId in visited: - return 0 - visited.incl(currentId) - - let edges = graph.getOutgoingEdges(currentId) - if edges.len == 0: - return 0 - - var maxDepth = 0 - for edge in edges: - let depth = dfs(edge.toTerm) - if depth + 1 > maxDepth: - maxDepth = depth + 1 - - return maxDepth - - return dfs(termId) - -proc getStats*(graph: DependencyGraph): GraphStats = - ## Get statistics about the graph - result = GraphStats( - terms: graph.terms.len, - edges: graph.edges.len, - roots: graph.getRoots().len, - leaves: graph.getLeaves().len, - maxDepth: 0, # TODO: Calculate max depth efficiently - hasCycle: graph.hasCycle() - ) - -# ============================================================================ -# String Representation -# ============================================================================ - -proc `$`*(term: PackageTerm): string = - ## Convert term to string - result = term.packageName & "@" & $term.version & "#" & term.variantHash[0..min(7, term.variantHash.high)] - -proc `$`*(graph: DependencyGraph): string = - ## Convert graph to string representation - let stats = graph.getStats() - result = "DependencyGraph(\n" - result.add(" terms: " & $stats.terms & "\n") - result.add(" edges: " & $stats.edges & "\n") - result.add(" roots: " & $stats.roots & "\n") - result.add(" leaves: " & $stats.leaves & "\n") - result.add(" hasCycle: " & $stats.hasCycle & "\n") - result.add(")") diff --git a/src/nip/resolver/flexible_adapter.nim b/src/nip/resolver/flexible_adapter.nim deleted file mode 100644 index 8623138..0000000 --- a/src/nip/resolver/flexible_adapter.nim +++ /dev/null @@ -1,148 +0,0 @@ -## Flexible Source Adapter -## -## This module implements the flexible adapter for source-based package systems -## like Gentoo and NPK. Flexible sources can build packages on demand with -## custom variant profiles. -## -## Philosophy: -## - Flexible = build on demand with any variant -## - Maximum customization (USE flags, compiler options) -## - Slower deployment (build time required) -## - Perfect for custom configurations -## -## Examples: -## - Gentoo: Build with custom USE flags -## - NPK: Build with custom variant profiles -## - Source-only packages: Always build from source - -import std/[options, tables] -import ./source_adapter -import ./variant_types - -type - # Build function signature for flexible sources - BuildFunction* = proc(demand: VariantDemand): Result[CasId, BuildError] {.closure.} - - # Flexible adapter for source-based builds - FlexibleAdapter* = ref object of SourceAdapter - availablePackages*: Table[string, PackageMetadata] ## Packages that can be built - buildFunc*: BuildFunction ## Function to build packages - -# Constructor -proc newFlexibleAdapter*( - name: string, - priority: int = 30, - buildFunc: BuildFunction = nil -): FlexibleAdapter = - ## Create a new flexible adapter - ## - ## Args: - ## name: Source name (e.g., "gentoo", "npk", "source") - ## priority: Selection priority (default: 30, lower than frozen) - ## buildFunc: Function to build packages (optional, for testing) - - result = FlexibleAdapter( - name: name, - class: Flexible, - priority: priority, - availablePackages: initTable[string, PackageMetadata](), - buildFunc: buildFunc - ) - -# Add a package that can be built -proc addPackage*(adapter: FlexibleAdapter, metadata: PackageMetadata) = - ## Add a package that can be built from source - ## - ## For flexible sources, the variant profile in metadata indicates - ## what variants are possible, not what's pre-built. - - adapter.availablePackages[metadata.name] = metadata - -# Check if adapter can satisfy a demand -method canSatisfy*(adapter: FlexibleAdapter, demand: VariantDemand): PackageAvailability = - ## Check if this flexible source can satisfy a variant demand - ## - ## For flexible sources, we can build any variant as long as the package exists. - ## Returns Available if package exists, Unavailable otherwise. - - if adapter.availablePackages.hasKey(demand.packageName): - return Available - else: - return Unavailable - -# Get package metadata for a demand -method getVariant*(adapter: FlexibleAdapter, demand: VariantDemand): Option[PackageMetadata] = - ## Get package metadata for a specific variant demand - ## - ## For flexible sources, we return metadata indicating the package can be built - ## with the requested variant profile. - - if not adapter.availablePackages.hasKey(demand.packageName): - return none(PackageMetadata) - - # Return metadata with the requested variant - var metadata = adapter.availablePackages[demand.packageName] - - # Update available variants to include the requested one - # (flexible sources can build any variant) - metadata.availableVariants = @[demand.variantProfile] - - return some(metadata) - -# Synthesize a package with requested variant -method synthesize*(adapter: FlexibleAdapter, demand: VariantDemand): Result[CasId, BuildError] = - ## Build a package with the requested variant profile - ## - ## This is the core capability of flexible sources - building packages - ## on demand with custom configurations. - ## - ## Returns CasId on success, BuildError on failure. - - # Check if package exists - if not adapter.availablePackages.hasKey(demand.packageName): - return err[CasId, BuildError](BuildError( - message: "Package not found: " & demand.packageName, - exitCode: 1, - buildLog: "Package " & demand.packageName & " is not available in source " & adapter.name - )) - - # Use custom build function if provided (for testing) - if adapter.buildFunc != nil: - return adapter.buildFunc(demand) - - # Default implementation: simulate successful build - # In production, this would invoke the actual build system - let metadata = adapter.availablePackages[demand.packageName] - let casId = newCasId(adapter.name & "-" & demand.packageName & "-" & demand.variantProfile.hash) - - return ok[CasId, BuildError](casId) - -# Helper to create a mock build function for testing -proc mockBuildSuccess*(packageName: string, casId: string): BuildFunction = - ## Create a mock build function that always succeeds - ## - ## Useful for testing without actual build infrastructure - - result = proc(demand: VariantDemand): Result[CasId, BuildError] = - if demand.packageName == packageName: - return ok[CasId, BuildError](newCasId(casId)) - else: - return err[CasId, BuildError](BuildError( - message: "Package not found: " & demand.packageName, - exitCode: 1, - buildLog: "Mock build function only handles " & packageName - )) - -# Helper to create a mock build function that fails -proc mockBuildFailure*(errorMessage: string, exitCode: int = 1): BuildFunction = - ## Create a mock build function that always fails - ## - ## Useful for testing build failure scenarios - - result = proc(demand: VariantDemand): Result[CasId, BuildError] = - return err[CasId, BuildError](BuildError( - message: errorMessage, - exitCode: exitCode, - buildLog: "Mock build failure: " & errorMessage - )) - diff --git a/src/nip/resolver/frozen_adapter.nim b/src/nip/resolver/frozen_adapter.nim deleted file mode 100644 index cfc4e0c..0000000 --- a/src/nip/resolver/frozen_adapter.nim +++ /dev/null @@ -1,140 +0,0 @@ -## Frozen Source Adapter -## -## This module implements the frozen adapter for pre-built binary sources -## like Nix and Arch Linux. Frozen sources provide packages with fixed -## variant profiles - you get what's available or nothing. -## -## Philosophy: -## - Frozen = pre-built binaries with fixed configurations -## - Fast deployment (no build time) -## - Limited flexibility (can't customize variants) -## - Perfect for common use cases -## -## Examples: -## - Nix: Provides binaries for common configurations -## - Arch/AUR: Pre-built packages with standard flags -## - Debian/Ubuntu: Binary packages with fixed options - -import std/[options, tables] -import ./source_adapter -import ./variant_types - -type - # Frozen adapter for pre-built binary sources - FrozenAdapter* = ref object of SourceAdapter - packages*: Table[string, seq[PackageMetadata]] ## Available packages by name - -# Constructor -proc newFrozenAdapter*(name: string, priority: int = 50): FrozenAdapter = - ## Create a new frozen adapter - ## - ## Args: - ## name: Source name (e.g., "nix", "arch", "debian") - ## priority: Selection priority (default: 50) - - result = FrozenAdapter( - name: name, - class: Frozen, - priority: priority, - packages: initTable[string, seq[PackageMetadata]]() - ) - -# Add a package to the frozen source -proc addPackage*(adapter: FrozenAdapter, metadata: PackageMetadata) = - ## Add a package with its available variants to the frozen source - ## - ## This simulates the package database of a frozen source. - ## In production, this would query the actual source (Nix cache, Arch repos, etc.) - - if not adapter.packages.hasKey(metadata.name): - adapter.packages[metadata.name] = @[] - - adapter.packages[metadata.name].add(metadata) - -# Check if adapter can satisfy a demand -method canSatisfy*(adapter: FrozenAdapter, demand: VariantDemand): PackageAvailability = - ## Check if this frozen source can satisfy a variant demand - ## - ## Returns: - ## Available: Package exists with exact variant match - ## WrongVariant: Package exists but variant doesn't match - ## Unavailable: Package doesn't exist in this source - - # Check if package exists - if not adapter.packages.hasKey(demand.packageName): - return Unavailable - - # Check if any available variant matches the demand - let availablePackages = adapter.packages[demand.packageName] - - for pkg in availablePackages: - # Check each available variant - for availableVariant in pkg.availableVariants: - if availableVariant == demand.variantProfile: - return Available - - # Package exists but no matching variant - return WrongVariant - -# Get package metadata for a demand -method getVariant*(adapter: FrozenAdapter, demand: VariantDemand): Option[PackageMetadata] = - ## Get package metadata for a specific variant demand - ## - ## Returns Some(metadata) if exact variant match found, None otherwise - - # Check if package exists - if not adapter.packages.hasKey(demand.packageName): - return none(PackageMetadata) - - # Find matching variant - let availablePackages = adapter.packages[demand.packageName] - - for pkg in availablePackages: - for availableVariant in pkg.availableVariants: - if availableVariant == demand.variantProfile: - return some(pkg) - - # No matching variant found - return none(PackageMetadata) - -# Synthesize is not supported for frozen adapters -method synthesize*(adapter: FrozenAdapter, demand: VariantDemand): Result[CasId, BuildError] = - ## Frozen adapters cannot build packages - they only provide pre-built binaries - ## - ## This method always returns an error for frozen adapters. - ## Use flexible adapters if you need to build from source. - - return err[CasId, BuildError](BuildError( - message: "Cannot synthesize packages from frozen source: " & adapter.name, - exitCode: 1, - buildLog: "Frozen sources only provide pre-built binaries. Use a flexible source to build from source." - )) - -# Helper to create a simple package metadata -proc newPackageMetadata*( - name: string, - version: string, - variants: seq[VariantProfile], - dependencies: seq[VariantDemand] = @[], - sourceHash: string = "", - buildTime: int = 0 -): PackageMetadata = - ## Create package metadata for a frozen source - ## - ## Args: - ## name: Package name - ## version: Package version - ## variants: Available variant profiles - ## dependencies: Package dependencies - ## sourceHash: Source hash (optional) - ## buildTime: Build time in seconds (0 for frozen) - - PackageMetadata( - name: name, - version: version, - availableVariants: variants, - dependencies: dependencies, - sourceHash: sourceHash, - buildTime: buildTime - ) - diff --git a/src/nip/resolver/graph_builder.nim b/src/nip/resolver/graph_builder.nim deleted file mode 100644 index ac55397..0000000 --- a/src/nip/resolver/graph_builder.nim +++ /dev/null @@ -1,258 +0,0 @@ -## Dependency Graph Builder -## -## This module implements the graph builder that constructs dependency graphs -## from package demands. It recursively fetches dependencies, unifies variants, -## and builds the complete dependency graph. -## -## Philosophy: -## - Start with root demands (user requests) -## - Recursively fetch dependencies from package metadata -## - Group demands by package name for variant unification -## - Build complete graph with all dependencies resolved -## - Detect conflicts and cycles early -## -## The graph builder is the bridge between user requests and the solver. - -import std/[tables, sets, options, sequtils] -import ./dependency_graph -import ./variant_types -import ./variant_hash -import ./source_adapter -import ../manifest_parser - -type - # Result of graph building operation - GraphBuildResult* = object - graph*: DependencyGraph - conflicts*: seq[UnificationResult] - warnings*: seq[string] - - # Error during graph building - GraphBuildError* = object - message*: string - packageName*: string - context*: string - - # Package metadata provider interface - PackageProvider* = proc(packageName: string): Option[seq[VariantDemand]] {.closure.} - -# Build dependency graph from root demands -proc buildDependencyGraph*( - rootDemands: seq[VariantDemand], - packageProvider: PackageProvider -): GraphBuildResult = - ## Build a complete dependency graph from root package demands - ## - ## This function: - ## 1. Starts with root demands (user requests) - ## 2. Recursively fetches dependencies for each package - ## 3. Groups demands by package name - ## 4. Unifies variant profiles for each package - ## 5. Creates terms and edges in the dependency graph - ## - ## Args: - ## rootDemands: Initial package demands from user - ## packageProvider: Function to get dependencies for a package - ## - ## Returns: - ## GraphBuildResult with complete graph and any conflicts - - var graph = newDependencyGraph() - var conflicts: seq[UnificationResult] = @[] - var warnings: seq[string] = @[] - var visited = initHashSet[string]() # Track visited packages to avoid infinite recursion - var allDemands = initTable[string, seq[VariantDemand]]() # Group demands by package name - - # Recursive function to collect all demands - proc collectDemands(demands: seq[VariantDemand]) = - for demand in demands: - # Skip if already processed this package - if demand.packageName in visited: - # Add to existing demands for unification - if not allDemands.hasKey(demand.packageName): - allDemands[demand.packageName] = @[] - allDemands[demand.packageName].add(demand) - continue - - visited.incl(demand.packageName) - - # Add this demand - if not allDemands.hasKey(demand.packageName): - allDemands[demand.packageName] = @[] - allDemands[demand.packageName].add(demand) - - # Get dependencies for this package - let dependencies = packageProvider(demand.packageName) - if dependencies.isSome: - # Recursively collect dependencies - collectDemands(dependencies.get) - - # Start collection with root demands - collectDemands(rootDemands) - - # Process each package: unify variants and create terms - var packageTerms = initTable[string, PackageTermId]() - - for packageName, demands in allDemands.pairs: - # Unify all variant demands for this package - let unificationResult = unify(demands) - - case unificationResult.kind: - of Unified: - # Create unified term - var profile = unificationResult.profile - profile.calculateHash() - let termId = createTermId(packageName, profile.hash) - let term = PackageTerm( - id: termId, - packageName: packageName, - version: SemanticVersion(major: 0, minor: 0, patch: 0), # Placeholder, needs real version resolution - variantHash: profile.hash, - variantProfile: profile, - optional: demands.anyIt(it.optional), - source: "unified" # Will be determined by source selection - ) - - graph.addTerm(term) - packageTerms[packageName] = termId - - of Conflict: - # Record conflict for later handling - conflicts.add(unificationResult) - warnings.add("Variant conflict for package " & packageName & ": " & unificationResult.reason) - - # Create dependency edges - for packageName, demands in allDemands.pairs: - if not packageTerms.hasKey(packageName): - continue # Skip packages with conflicts - - let fromTermId = packageTerms[packageName] - - # Get dependencies for this package - let dependencies = packageProvider(packageName) - if dependencies.isSome: - for depDemand in dependencies.get: - if packageTerms.hasKey(depDemand.packageName): - let toTermId = packageTerms[depDemand.packageName] - - # Determine dependency type - let depType = if depDemand.optional: Optional else: Required - - let edge = DependencyEdge( - fromTerm: fromTermId, - toTerm: toTermId, - dependencyType: depType, - constraint: "" # TODO: Add constraint string - ) - - graph.addEdge(edge) - - return GraphBuildResult( - graph: graph, - conflicts: conflicts, - warnings: warnings - ) - -# Simplified graph builder for testing -proc buildSimpleGraph*( - rootDemands: seq[VariantDemand], - dependencyMap: Table[string, seq[VariantDemand]] -): GraphBuildResult = - ## Simplified graph builder using a static dependency map - ## - ## This is useful for testing where we want to control - ## the dependency relationships explicitly. - ## - ## Args: - ## rootDemands: Initial package demands - ## dependencyMap: Map of package name to its dependencies - - let provider: PackageProvider = proc(packageName: string): Option[seq[VariantDemand]] = - if dependencyMap.hasKey(packageName): - return some(dependencyMap[packageName]) - else: - return none(seq[VariantDemand]) - - return buildDependencyGraph(rootDemands, provider) - -# Validate graph structure -proc validateGraph*(graph: DependencyGraph): bool = - ## Validate that the dependency graph is well-formed - ## - ## Checks: - ## - All edge endpoints exist as terms - ## - No self-loops - ## - Edge lookup tables are consistent - - # Check all edges have valid endpoints - for edge in graph.edges: - if not graph.terms.hasKey(edge.fromTerm): - return false - if not graph.terms.hasKey(edge.toTerm): - return false - - # Check for self-loops - if edge.fromTerm == edge.toTerm: - return false - - # Check edge lookup table consistency - for termId in graph.terms.keys: - let outgoing = graph.getOutgoingEdges(termId) - let incoming = graph.getIncomingEdges(termId) - - # Verify outgoing edges are in main edge list - for edge in outgoing: - if edge notin graph.edges: - return false - - # Verify incoming edges are in main edge list - for edge in incoming: - if edge notin graph.edges: - return false - - return true - -# Get root terms (terms with no incoming edges) -proc getRootTerms*(graph: DependencyGraph): seq[PackageTermId] = - ## Get all root terms (terms with no incoming dependencies) - ## - ## These are typically the packages directly requested by the user. - - result = @[] - for termId in graph.terms.keys: - if graph.getIncomingEdges(termId).len == 0: - result.add(termId) - -# Get leaf terms (terms with no outgoing edges) -proc getLeafTerms*(graph: DependencyGraph): seq[PackageTermId] = - ## Get all leaf terms (terms with no outgoing dependencies) - ## - ## These are typically low-level libraries with no dependencies. - - result = @[] - for termId in graph.terms.keys: - if graph.getOutgoingEdges(termId).len == 0: - result.add(termId) - -# Get terms by package name -proc getTermsByPackage*(graph: DependencyGraph, packageName: string): seq[PackageTerm] = - ## Get all terms for a specific package name - ## - ## This can return multiple terms if the same package appears - ## with different variant profiles. - - result = @[] - for term in graph.terms.values: - if term.packageName == packageName: - result.add(term) - -# String representation for debugging -proc `$`*(buildResult: GraphBuildResult): string = - ## String representation of graph build result - - result = "GraphBuildResult(" - result.add("terms=" & $buildResult.graph.getStats().terms) - result.add(", edges=" & $buildResult.graph.getStats().edges) - result.add(", conflicts=" & $buildResult.conflicts.len) - result.add(", warnings=" & $buildResult.warnings.len) - result.add(")") \ No newline at end of file diff --git a/src/nip/resolver/lru_cache.nim b/src/nip/resolver/lru_cache.nim deleted file mode 100644 index b2da4e6..0000000 --- a/src/nip/resolver/lru_cache.nim +++ /dev/null @@ -1,584 +0,0 @@ -## LRU Cache Implementation -## -## This module provides a generic Least Recently Used (LRU) cache with: -## - O(1) get/put operations -## - Automatic eviction of least recently used entries -## - Configurable maximum size -## - Thread-safe operations (optional) -## -## **Design:** -## - Doubly-linked list for LRU ordering -## - Hash table for O(1) key lookup -## - Move-to-front on access (most recently used) -## - Evict from tail when capacity exceeded -## -## **Use Cases:** -## - Dependency resolution caching -## - Unification result caching -## - Build hash caching - -import tables -import options -import locks -import strutils # For formatFloat - -type - LRUNode[K, V] = ref object - ## Node in the doubly-linked list - key: K - value: V - prev: LRUNode[K, V] - next: LRUNode[K, V] - - LRUCache*[K, V] = ref object - ## Generic LRU cache with automatic eviction - capacity: int - cache: Table[K, LRUNode[K, V]] - head: LRUNode[K, V] # Most recently used (dummy head) - tail: LRUNode[K, V] # Least recently used (dummy tail) - lock: Lock # For thread-safe operations - threadSafe: bool - - CacheStats* = object - ## Cache performance statistics - hits*: int - misses*: int - evictions*: int - size*: int - capacity*: int - -# ============================================================================ -# LRU Cache Construction -# ============================================================================ - -proc newLRUCache*[K, V](capacity: int, threadSafe: bool = false): LRUCache[K, V] = - ## Create a new LRU cache with specified capacity. - ## - ## **Parameters:** - ## - capacity: Maximum number of entries (must be > 0) - ## - threadSafe: Enable thread-safe operations (default: false) - ## - ## **Returns:** New LRU cache instance - ## - ## **Example:** - ## ```nim - ## let cache = newLRUCache[string, int](capacity = 100) - ## cache.put("key", 42) - ## let value = cache.get("key") - ## ``` - - assert capacity > 0, "Cache capacity must be positive" - - result = LRUCache[K, V]( - capacity: capacity, - cache: initTable[K, LRUNode[K, V]](), - threadSafe: threadSafe - ) - - # Create dummy head and tail nodes - result.head = LRUNode[K, V]() - result.tail = LRUNode[K, V]() - result.head.next = result.tail - result.tail.prev = result.head - - if threadSafe: - initLock(result.lock) - -# ============================================================================ -# Internal List Operations -# ============================================================================ - -proc removeNode[K, V](cache: LRUCache[K, V], node: LRUNode[K, V]) = - ## Remove node from doubly-linked list (internal) - let prev = node.prev - let next = node.next - prev.next = next - next.prev = prev - -proc addToHead[K, V](cache: LRUCache[K, V], node: LRUNode[K, V]) = - ## Add node to head of list (most recently used) - node.prev = cache.head - node.next = cache.head.next - cache.head.next.prev = node - cache.head.next = node - -proc moveToHead[K, V](cache: LRUCache[K, V], node: LRUNode[K, V]) = - ## Move existing node to head (mark as most recently used) - cache.removeNode(node) - cache.addToHead(node) - -proc removeTail[K, V](cache: LRUCache[K, V]): LRUNode[K, V] = - ## Remove and return tail node (least recently used) - result = cache.tail.prev - cache.removeNode(result) - -# ============================================================================ -# Public Cache Operations -# ============================================================================ - -proc get*[K, V](cache: LRUCache[K, V], key: K): Option[V] = - ## Get value from cache, marking it as recently used. - ## - ## **Parameters:** - ## - key: Key to lookup - ## - ## **Returns:** Some(value) if found, None if not found - ## - ## **Complexity:** O(1) - ## - ## **Side Effect:** Moves accessed entry to front (most recently used) - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - if key in cache.cache: - let node = cache.cache[key] - cache.moveToHead(node) - return some(node.value) - else: - return none(V) - -proc put*[K, V](cache: LRUCache[K, V], key: K, value: V) = - ## Put value into cache, evicting least recently used if necessary. - ## - ## **Parameters:** - ## - key: Key to store - ## - value: Value to store - ## - ## **Complexity:** O(1) - ## - ## **Side Effect:** May evict least recently used entry if at capacity - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - if key in cache.cache: - # Update existing entry - let node = cache.cache[key] - node.value = value - cache.moveToHead(node) - else: - # Add new entry - let newNode = LRUNode[K, V](key: key, value: value) - cache.cache[key] = newNode - cache.addToHead(newNode) - - # Evict if over capacity - if cache.cache.len > cache.capacity: - let tail = cache.removeTail() - cache.cache.del(tail.key) - -proc contains*[K, V](cache: LRUCache[K, V], key: K): bool = - ## Check if key exists in cache without affecting LRU order. - ## - ## **Parameters:** - ## - key: Key to check - ## - ## **Returns:** true if key exists, false otherwise - ## - ## **Complexity:** O(1) - ## - ## **Note:** Does NOT mark entry as recently used - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - return key in cache.cache - -proc delete*[K, V](cache: LRUCache[K, V], key: K): bool = - ## Delete entry from cache. - ## - ## **Parameters:** - ## - key: Key to delete - ## - ## **Returns:** true if entry was deleted, false if not found - ## - ## **Complexity:** O(1) - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - if key in cache.cache: - let node = cache.cache[key] - cache.removeNode(node) - cache.cache.del(key) - return true - else: - return false - -proc clear*[K, V](cache: LRUCache[K, V]) = - ## Clear all entries from cache. - ## - ## **Complexity:** O(n) - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - cache.cache.clear() - cache.head.next = cache.tail - cache.tail.prev = cache.head - -proc len*[K, V](cache: LRUCache[K, V]): int = - ## Get current number of entries in cache. - ## - ## **Returns:** Number of entries - ## - ## **Complexity:** O(1) - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - return cache.cache.len - -proc capacity*[K, V](cache: LRUCache[K, V]): int = - ## Get maximum capacity of cache. - ## - ## **Returns:** Maximum number of entries - - return cache.capacity - -proc isFull*[K, V](cache: LRUCache[K, V]): bool = - ## Check if cache is at capacity. - ## - ## **Returns:** true if cache is full, false otherwise - - return cache.len >= cache.capacity - -# ============================================================================ -# Cache Statistics -# ============================================================================ - -type - LRUCacheWithStats*[K, V] = ref object - ## LRU cache with performance statistics tracking - cache: LRUCache[K, V] - hits: int - misses: int - evictions: int - -proc newLRUCacheWithStats*[K, V](capacity: int, threadSafe: bool = false): LRUCacheWithStats[K, V] = - ## Create LRU cache with statistics tracking. - ## - ## **Parameters:** - ## - capacity: Maximum number of entries - ## - threadSafe: Enable thread-safe operations - ## - ## **Returns:** New cache with stats tracking - - result = LRUCacheWithStats[K, V]( - cache: newLRUCache[K, V](capacity, threadSafe), - hits: 0, - misses: 0, - evictions: 0 - ) - -proc get*[K, V](cache: LRUCacheWithStats[K, V], key: K): Option[V] = - ## Get value from cache with statistics tracking. - - let result = cache.cache.get(key) - if result.isSome: - cache.hits += 1 - else: - cache.misses += 1 - return result - -proc put*[K, V](cache: LRUCacheWithStats[K, V], key: K, value: V) = - ## Put value into cache with statistics tracking. - - let wasFull = cache.cache.isFull - let hadKey = key in cache.cache - - cache.cache.put(key, value) - - if wasFull and not hadKey: - cache.evictions += 1 - -proc getStats*[K, V](cache: LRUCacheWithStats[K, V]): CacheStats = - ## Get cache performance statistics. - ## - ## **Returns:** Statistics including hits, misses, evictions - - result = CacheStats( - hits: cache.hits, - misses: cache.misses, - evictions: cache.evictions, - size: cache.cache.len, - capacity: cache.cache.capacity - ) - -proc hitRate*[K, V](cache: LRUCacheWithStats[K, V]): float = - ## Calculate cache hit rate. - ## - ## **Returns:** Hit rate as percentage (0.0 - 1.0) - - let total = cache.hits + cache.misses - if total == 0: - return 0.0 - return cache.hits.float / total.float - -proc resetStats*[K, V](cache: LRUCacheWithStats[K, V]) = - ## Reset statistics counters to zero. - - cache.hits = 0 - cache.misses = 0 - cache.evictions = 0 - -proc clear*[K, V](cache: LRUCacheWithStats[K, V]) = - ## Clear all entries from cache (keeps statistics). - - cache.cache.clear() - -proc delete*[K, V](cache: LRUCacheWithStats[K, V], key: K): bool = - ## Delete entry from cache. - result = cache.cache.delete(key) - -# ============================================================================ -# Iteration Support -# ============================================================================ - -iterator items*[K, V](cache: LRUCache[K, V]): (K, V) = - ## Iterate over cache entries (no particular order). - ## - ## **Note:** Does NOT affect LRU order - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - for key, node in cache.cache.pairs: - yield (key, node.value) - -iterator itemsLRU*[K, V](cache: LRUCache[K, V]): (K, V) = - ## Iterate over cache entries in LRU order (most recent first). - ## - ## **Note:** Does NOT affect LRU order - - if cache.threadSafe: - acquire(cache.lock) - - defer: - if cache.threadSafe: - release(cache.lock) - - var current = cache.head.next - while current != cache.tail: - yield (current.key, current.value) - current = current.next - -# ============================================================================ -# Debug and Inspection -# ============================================================================ - -proc `$`*[K, V](cache: LRUCache[K, V]): string = - ## String representation of cache for debugging. - - result = "LRUCache(size=" & $cache.len & ", capacity=" & $cache.capacity & ")" - -proc `$`*(stats: CacheStats): string = - ## String representation of cache statistics. - - let hitRate = if stats.hits + stats.misses > 0: - (stats.hits.float / (stats.hits + stats.misses).float * 100.0) - else: - 0.0 - - result = "CacheStats(hits=" & $stats.hits & - ", misses=" & $stats.misses & - ", evictions=" & $stats.evictions & - ", size=" & $stats.size & - ", capacity=" & $stats.capacity & - ", hitRate=" & hitRate.formatFloat(ffDecimal, 2) & "%)" - -# ============================================================================ -# Unit Tests -# ============================================================================ - -when isMainModule: - import unittest - - suite "LRU Cache Basic Operations": - test "Create cache with capacity": - let cache = newLRUCache[string, int](capacity = 3) - check cache.len == 0 - check cache.capacity == 3 - check not cache.isFull - - test "Put and get single entry": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - - let value = cache.get("key1") - check value.isSome - check value.get == 100 - - test "Get non-existent key returns None": - let cache = newLRUCache[string, int](capacity = 3) - let value = cache.get("missing") - check value.isNone - - test "Update existing key": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - cache.put("key1", 200) - - let value = cache.get("key1") - check value.get == 200 - check cache.len == 1 - - test "Contains check": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - - check "key1" in cache - check "missing" notin cache - - test "Delete entry": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - - check cache.delete("key1") - check "key1" notin cache - check not cache.delete("missing") - - test "Clear cache": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - cache.put("key2", 200) - - cache.clear() - check cache.len == 0 - check "key1" notin cache - - suite "LRU Eviction": - test "Evict least recently used when at capacity": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - cache.put("key2", 200) - cache.put("key3", 300) - cache.put("key4", 400) # Should evict key1 - - check cache.len == 3 - check "key1" notin cache - check "key2" in cache - check "key3" in cache - check "key4" in cache - - test "Access updates LRU order": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - cache.put("key2", 200) - cache.put("key3", 300) - - # Access key1 to make it most recently used - discard cache.get("key1") - - # Add key4, should evict key2 (least recently used) - cache.put("key4", 400) - - check "key1" in cache - check "key2" notin cache - check "key3" in cache - check "key4" in cache - - test "Update preserves entry": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - cache.put("key2", 200) - cache.put("key3", 300) - - # Update key1 - cache.put("key1", 150) - - # Add key4, should evict key2 - cache.put("key4", 400) - - check "key1" in cache - check cache.get("key1").get == 150 - - suite "Cache Statistics": - test "Track hits and misses": - let cache = newLRUCacheWithStats[string, int](capacity = 3) - cache.put("key1", 100) - - discard cache.get("key1") # Hit - discard cache.get("key2") # Miss - discard cache.get("key1") # Hit - - let stats = cache.getStats() - check stats.hits == 2 - check stats.misses == 1 - check cache.hitRate() > 0.6 - - test "Track evictions": - let cache = newLRUCacheWithStats[string, int](capacity = 2) - cache.put("key1", 100) - cache.put("key2", 200) - cache.put("key3", 300) # Eviction - - let stats = cache.getStats() - check stats.evictions == 1 - - test "Reset statistics": - let cache = newLRUCacheWithStats[string, int](capacity = 3) - cache.put("key1", 100) - discard cache.get("key1") - - cache.resetStats() - - let stats = cache.getStats() - check stats.hits == 0 - check stats.misses == 0 - - suite "Iteration": - test "Iterate over entries": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - cache.put("key2", 200) - cache.put("key3", 300) - - var count = 0 - for (key, value) in cache.items: - count += 1 - - check count == 3 - - test "Iterate in LRU order": - let cache = newLRUCache[string, int](capacity = 3) - cache.put("key1", 100) - cache.put("key2", 200) - cache.put("key3", 300) - - var keys: seq[string] - for (key, value) in cache.itemsLRU: - keys.add(key) - - # Most recent first - check keys[0] == "key3" - check keys[2] == "key1" diff --git a/src/nip/resolver/nimpak_bridge_adapter.nim b/src/nip/resolver/nimpak_bridge_adapter.nim deleted file mode 100644 index 2907aa3..0000000 --- a/src/nip/resolver/nimpak_bridge_adapter.nim +++ /dev/null @@ -1,112 +0,0 @@ -## Nimpak Bridge Adapter -## -## This module bridges the new Resolver system with the existing Nimpak adapters -## (AUR, Pacman, Nix, etc.). It allows the Resolver to query and install packages -## from these external sources using the unified SourceAdapter interface. -## -## Philosophy: -## - Reuse existing robust adapters -## - Provide immediate access to 100,000+ packages -## - Unified interface for all package sources - -import std/[options, json, strutils, sequtils, times] -import ./source_adapter -import ./variant_types -import ../../nimpak/adapters/aur -import ../../nimpak/grafting -import ../../nimpak/cas as nimpak_cas - -type - NimpakBridgeAdapter* = ref object of SourceAdapter - aurAdapter*: AURAdapter - # Future: Add pacmanAdapter, nixAdapter, etc. - -# Constructor -proc newNimpakBridgeAdapter*(priority: int = 40): NimpakBridgeAdapter = - ## Create a new bridge adapter - result = NimpakBridgeAdapter( - name: "nimpak-bridge", - class: Flexible, # AUR is source-based, so Flexible - priority: priority, - aurAdapter: newAURAdapter() - ) - -# Helper to convert AUR info to PackageMetadata -proc toPackageMetadata(info: JsonNode): PackageMetadata = - var variants: seq[VariantProfile] = @[] - - # Create a default variant profile - var defaultProfile = newVariantProfile() - defaultProfile.calculateHash() - variants.add(defaultProfile) - - # Convert dependencies - var dependencies: seq[VariantDemand] = @[] - if info.hasKey("depends"): - for dep in info["depends"]: - dependencies.add(VariantDemand( - packageName: dep.getStr(), - variantProfile: newVariantProfile(), # Default profile for deps - optional: false - )) - - if info.hasKey("makedepends"): - for dep in info["makedepends"]: - dependencies.add(VariantDemand( - packageName: dep.getStr(), - variantProfile: newVariantProfile(), - optional: false # Build deps are required for build - )) - - result = PackageMetadata( - name: info["name"].getStr(), - version: info["version"].getStr(), - availableVariants: variants, - dependencies: dependencies, - sourceHash: "aur-" & info["version"].getStr(), # Simple hash for now - buildTime: 300 # Estimate 5 mins - ) - -# Check if adapter can satisfy a demand -method canSatisfy*(adapter: NimpakBridgeAdapter, demand: VariantDemand): PackageAvailability = - ## Check if AUR has the package - - # For now, we only check AUR - # In future, we'll check other adapters too - - let validationResult = adapter.aurAdapter.validatePackage(demand.packageName) - if validationResult.isOk and validationResult.value: - return Available - else: - return Unavailable - -# Get package metadata -method getVariant*(adapter: NimpakBridgeAdapter, demand: VariantDemand): Option[PackageMetadata] = - ## Get package info from AUR - - let infoResult = adapter.aurAdapter.getPackageInfo(demand.packageName) - if infoResult.isOk: - return some(toPackageMetadata(infoResult.value)) - else: - return none(PackageMetadata) - -# Synthesize (Build/Graft) package -method synthesize*(adapter: NimpakBridgeAdapter, demand: VariantDemand): source_adapter.Result[CasId, BuildError] = - ## Graft package from AUR - - # We use a dummy cache for now as the bridge doesn't manage the cache directly - # The AUR adapter manages its own caching - let cache = GraftingCache() - - let graftResult = adapter.aurAdapter.graftPackage(demand.packageName, cache) - - if graftResult.success: - # Return the package ID as the CAS ID (since we don't have the real CAS ID from graft yet) - # In a real implementation, graftPackage should return the CAS ID - return source_adapter.ok[CasId, BuildError](newCasId("aur-" & demand.packageName)) - else: - return source_adapter.err[CasId, BuildError](BuildError( - message: "Failed to graft package from AUR", - exitCode: 1, - buildLog: graftResult.errors.join("\n") - )) diff --git a/src/nip/resolver/nipcell_fallback.nim b/src/nip/resolver/nipcell_fallback.nim deleted file mode 100644 index 09c38b0..0000000 --- a/src/nip/resolver/nipcell_fallback.nim +++ /dev/null @@ -1,618 +0,0 @@ -## NipCell Fallback for Unresolvable Conflicts -## -## This module implements the NipCell isolation fallback mechanism for the -## NIP dependency resolver. When variant unification fails due to irreconcilable -## conflicts, this module suggests and manages NipCell isolation as an alternative. -## -## **Philosophy:** -## - When the Paradox Engine cannot synthesize a unified solution, we offer -## isolation as a pragmatic escape hatch -## - NipCells provide separate dependency graphs for conflicting packages -## - Users maintain control over when to use isolation vs. forcing unification -## -## **Requirements:** -## - 10.1: Detect unresolvable conflicts and suggest NipCell isolation -## - 10.2: Create separate NipCells for conflicting packages -## - 10.3: Maintain separate dependency graphs per cell -## - 10.4: Support cell switching for different environments -## - 10.5: Clean up cell-specific packages when removing cells -## -## **Architecture:** -## ``` -## ┌─────────────────────────────────────────────────────────────┐ -## │ Conflict Detection │ -## │ ───────────────────────────────────────────────────────── │ -## │ Detect unresolvable variant conflicts │ -## │ Analyze conflict severity and isolation candidates │ -## └────────────────────┬────────────────────────────────────────┘ -## │ -## v -## ┌─────────────────────────────────────────────────────────────┐ -## │ Isolation Suggestion │ -## │ ───────────────────────────────────────────────────────── │ -## │ Suggest NipCell isolation with clear explanation │ -## │ Provide actionable commands for user │ -## └────────────────────┬────────────────────────────────────────┘ -## │ -## v -## ┌─────────────────────────────────────────────────────────────┐ -## │ Cell Management │ -## │ ───────────────────────────────────────────────────────── │ -## │ Create cells, maintain separate graphs, handle switching │ -## └─────────────────────────────────────────────────────────────┘ -## ``` - -import std/[tables, sets, options, sequtils, algorithm, strutils, strformat, times, os, json] -import ./conflict_detection -import ./dependency_graph -import ./solver_types - -type - ## Severity of a conflict for isolation decision - ConflictSeverity* = enum - Low, ## Minor conflict, may be resolvable with flag changes - Medium, ## Significant conflict, isolation recommended - High, ## Severe conflict, isolation strongly recommended - Critical ## Irreconcilable conflict, isolation required - - ## A candidate package for isolation - IsolationCandidate* = object - packageName*: string - conflictingWith*: seq[string] - severity*: ConflictSeverity - suggestedCellName*: string - reason*: string - - ## Suggestion for NipCell isolation - IsolationSuggestion* = object - candidates*: seq[IsolationCandidate] - primaryConflict*: ConflictReport - suggestedCells*: seq[SuggestedCell] - explanation*: string - commands*: seq[string] - - ## A suggested cell configuration - SuggestedCell* = object - name*: string - packages*: seq[string] - description*: string - isolationLevel*: string - - ## A NipCell with its own dependency graph - NipCellGraph* = object - cellName*: string - cellId*: string - graph*: DependencyGraph - packages*: HashSet[string] - created*: DateTime - lastModified*: DateTime - metadata*: Table[string, string] - - ## Manager for multiple NipCell graphs - NipCellGraphManager* = ref object - cells*: Table[string, NipCellGraph] - activeCell*: Option[string] - cellRoot*: string - globalPackages*: HashSet[string] ## Packages available in all cells - - ## Result of cell creation - CellCreationResult* = object - success*: bool - cellName*: string - cellId*: string - error*: string - - ## Result of cell switching - CellSwitchResult* = object - success*: bool - previousCell*: Option[string] - newCell*: string - error*: string - -# ============================================================================= -# Conflict Severity Analysis -# ============================================================================= - -proc analyzeConflictSeverity*(conflict: ConflictReport): ConflictSeverity = - ## Analyze the severity of a conflict to determine isolation necessity. - ## - ## **Requirements:** 10.1 - Detect unresolvable conflicts - ## - ## **Severity Levels:** - ## - Low: Version conflicts that might be resolved with constraint relaxation - ## - Medium: Variant conflicts in non-exclusive domains - ## - High: Variant conflicts in exclusive domains - ## - Critical: Circular dependencies or fundamental incompatibilities - - case conflict.kind: - of VersionConflict: - # Version conflicts are usually resolvable - return Low - - of VariantConflict: - # Check if it's an exclusive domain conflict - if conflict.details.contains("exclusive"): - return High - else: - return Medium - - of CircularDependency: - # Circular dependencies are critical - return Critical - - of MissingPackage: - # Missing packages are low severity (just need to find the package) - return Low - - of BuildHashMismatch: - # Build hash mismatches are medium severity - return Medium - -proc shouldSuggestIsolation*(severity: ConflictSeverity): bool = - ## Determine if isolation should be suggested based on severity. - ## - ## **Requirements:** 10.1 - Suggest NipCell isolation for unresolvable conflicts - - case severity: - of Low: - return false - of Medium: - return true - of High: - return true - of Critical: - return true - -# ============================================================================= -# Isolation Candidate Detection -# ============================================================================= - -proc detectIsolationCandidates*( - conflicts: seq[ConflictReport] -): seq[IsolationCandidate] = - ## Detect packages that are good candidates for isolation. - ## - ## **Requirements:** 10.1, 10.2 - Detect conflicts and suggest isolation - ## - ## **Algorithm:** - ## 1. Group conflicts by package - ## 2. Analyze severity of each conflict - ## 3. Identify packages that would benefit from isolation - ## 4. Generate suggested cell names - - result = @[] - - # Group conflicts by package - var packageConflicts: Table[string, seq[ConflictReport]] = initTable[string, seq[ConflictReport]]() - - for conflict in conflicts: - for pkg in conflict.packages: - if pkg notin packageConflicts: - packageConflicts[pkg] = @[] - packageConflicts[pkg].add(conflict) - - # Analyze each package - for pkg, pkgConflicts in packageConflicts.pairs: - # Find the most severe conflict - var maxSeverity = Low - var conflictingPackages: seq[string] = @[] - var reasons: seq[string] = @[] - - for conflict in pkgConflicts: - let severity = analyzeConflictSeverity(conflict) - if severity > maxSeverity: - maxSeverity = severity - - for otherPkg in conflict.packages: - if otherPkg != pkg and otherPkg notin conflictingPackages: - conflictingPackages.add(otherPkg) - - reasons.add(conflict.details) - - # Only suggest isolation for medium+ severity - if shouldSuggestIsolation(maxSeverity): - let candidate = IsolationCandidate( - packageName: pkg, - conflictingWith: conflictingPackages, - severity: maxSeverity, - suggestedCellName: pkg & "-cell", - reason: reasons.join("; ") - ) - result.add(candidate) - -# ============================================================================= -# Isolation Suggestion Generation -# ============================================================================= - -proc generateIsolationSuggestion*( - conflict: ConflictReport, - candidates: seq[IsolationCandidate] -): IsolationSuggestion = - ## Generate a complete isolation suggestion with commands. - ## - ## **Requirements:** 10.1, 10.2 - Suggest NipCell isolation - ## - ## **Returns:** Complete suggestion with explanation and CLI commands - - var suggestedCells: seq[SuggestedCell] = @[] - var commands: seq[string] = @[] - - # Group candidates by suggested cell - for candidate in candidates: - let cell = SuggestedCell( - name: candidate.suggestedCellName, - packages: @[candidate.packageName], - description: fmt"Isolated environment for {candidate.packageName}", - isolationLevel: if candidate.severity == Critical: "strict" else: "standard" - ) - suggestedCells.add(cell) - - # Generate CLI commands - commands.add(fmt"nip cell create {candidate.suggestedCellName} --isolation={cell.isolationLevel}") - commands.add(fmt"nip cell activate {candidate.suggestedCellName}") - commands.add(fmt"nip install {candidate.packageName}") - - # Build explanation - var explanation = "The following packages have irreconcilable conflicts:\n\n" - - for candidate in candidates: - explanation.add(" • " & candidate.packageName) - if candidate.conflictingWith.len > 0: - let conflictList = candidate.conflictingWith.join(", ") - explanation.add(" (conflicts with: " & conflictList & ")") - explanation.add("\n") - - explanation.add("\nNipCell isolation allows you to install these packages in separate environments,\n") - explanation.add("each with its own dependency graph. This avoids the conflict while maintaining\n") - explanation.add("full functionality of each package.\n") - - return IsolationSuggestion( - candidates: candidates, - primaryConflict: conflict, - suggestedCells: suggestedCells, - explanation: explanation, - commands: commands - ) - -proc formatIsolationSuggestion*(suggestion: IsolationSuggestion): string = - ## Format an isolation suggestion for display. - ## - ## **Requirements:** 10.1 - Provide actionable suggestions - - result = """ -🔀 [IsolationSuggested] NipCell isolation recommended - -""" - result.add(suggestion.explanation) - result.add("\n💡 Suggested commands:\n\n") - - for cmd in suggestion.commands: - result.add(fmt" $ {cmd}\n") - - result.add("\n📦 Suggested cells:\n\n") - - for cell in suggestion.suggestedCells: - result.add(" • " & cell.name & ": " & cell.description & "\n") - let pkgList = cell.packages.join(", ") - result.add(" Packages: " & pkgList & "\n") - result.add(" Isolation: " & cell.isolationLevel & "\n\n") - -# ============================================================================= -# NipCell Graph Management -# ============================================================================= - -proc newNipCellGraph*(cellName: string, cellId: string = ""): NipCellGraph = - ## Create a new NipCell graph. - ## - ## **Requirements:** 10.2, 10.3 - Create cells with separate graphs - - let id = if cellId == "": cellName & "-" & $now().toTime().toUnix() else: cellId - - result = NipCellGraph( - cellName: cellName, - cellId: id, - graph: newDependencyGraph(), - packages: initHashSet[string](), - created: now(), - lastModified: now(), - metadata: initTable[string, string]() - ) - -proc newNipCellGraphManager*(cellRoot: string = ""): NipCellGraphManager = - ## Create a new NipCell graph manager. - ## - ## **Requirements:** 10.3, 10.4 - Maintain separate graphs and support switching - - let root = if cellRoot == "": getHomeDir() / ".nip" / "cells" else: cellRoot - - result = NipCellGraphManager( - cells: initTable[string, NipCellGraph](), - activeCell: none(string), - cellRoot: root, - globalPackages: initHashSet[string]() - ) - -proc createCell*( - manager: NipCellGraphManager, - cellName: string, - description: string = "" -): CellCreationResult = - ## Create a new NipCell with its own dependency graph. - ## - ## **Requirements:** 10.2 - Create separate NipCells for conflicting packages - - # Check if cell already exists - if cellName in manager.cells: - return CellCreationResult( - success: false, - cellName: cellName, - cellId: "", - error: fmt"Cell '{cellName}' already exists" - ) - - # Create new cell graph - let cellGraph = newNipCellGraph(cellName) - - # Add description to metadata - var graph = cellGraph - if description != "": - graph.metadata["description"] = description - - # Store in manager - manager.cells[cellName] = graph - - return CellCreationResult( - success: true, - cellName: cellName, - cellId: graph.cellId, - error: "" - ) - -proc deleteCell*( - manager: NipCellGraphManager, - cellName: string -): bool = - ## Delete a NipCell and clean up its packages. - ## - ## **Requirements:** 10.5 - Clean up cell-specific packages when removing cells - - if cellName notin manager.cells: - return false - - # If this is the active cell, deactivate it - if manager.activeCell.isSome and manager.activeCell.get() == cellName: - manager.activeCell = none(string) - - # Remove the cell - manager.cells.del(cellName) - - return true - -proc switchCell*( - manager: NipCellGraphManager, - cellName: string -): CellSwitchResult = - ## Switch to a different NipCell. - ## - ## **Requirements:** 10.4 - Support cell switching - - # Check if cell exists - if cellName notin manager.cells: - return CellSwitchResult( - success: false, - previousCell: manager.activeCell, - newCell: cellName, - error: fmt"Cell '{cellName}' not found" - ) - - let previousCell = manager.activeCell - manager.activeCell = some(cellName) - - return CellSwitchResult( - success: true, - previousCell: previousCell, - newCell: cellName, - error: "" - ) - -proc getActiveCell*(manager: NipCellGraphManager): Option[string] = - ## Get the currently active cell name. - ## - ## **Requirements:** 10.4 - Support cell switching - - return manager.activeCell - -proc getCellGraph*( - manager: NipCellGraphManager, - cellName: string -): Option[NipCellGraph] = - ## Get the dependency graph for a specific cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if cellName in manager.cells: - return some(manager.cells[cellName]) - return none(NipCellGraph) - -proc getActiveCellGraph*(manager: NipCellGraphManager): Option[NipCellGraph] = - ## Get the dependency graph for the active cell. - ## - ## **Requirements:** 10.3, 10.4 - Maintain graphs and support switching - - if manager.activeCell.isSome: - return manager.getCellGraph(manager.activeCell.get()) - return none(NipCellGraph) - -proc listCells*(manager: NipCellGraphManager): seq[string] = - ## List all available cells. - ## - ## **Requirements:** 10.4 - Support cell management - - result = @[] - for cellName in manager.cells.keys: - result.add(cellName) - result.sort() - -# ============================================================================= -# Package Management in Cells -# ============================================================================= - -proc addPackageToCell*( - manager: NipCellGraphManager, - cellName: string, - packageName: string -): bool = - ## Add a package to a cell's dependency graph. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if cellName notin manager.cells: - return false - - manager.cells[cellName].packages.incl(packageName) - manager.cells[cellName].lastModified = now() - - return true - -proc removePackageFromCell*( - manager: NipCellGraphManager, - cellName: string, - packageName: string -): bool = - ## Remove a package from a cell's dependency graph. - ## - ## **Requirements:** 10.5 - Clean up cell-specific packages - - if cellName notin manager.cells: - return false - - if packageName notin manager.cells[cellName].packages: - return false - - manager.cells[cellName].packages.excl(packageName) - manager.cells[cellName].lastModified = now() - - return true - -proc getCellPackages*( - manager: NipCellGraphManager, - cellName: string -): seq[string] = - ## Get all packages in a cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if cellName notin manager.cells: - return @[] - - result = toSeq(manager.cells[cellName].packages) - result.sort() - -proc isPackageInCell*( - manager: NipCellGraphManager, - cellName: string, - packageName: string -): bool = - ## Check if a package is in a specific cell. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs per cell - - if cellName notin manager.cells: - return false - - return packageName in manager.cells[cellName].packages - -# ============================================================================= -# Conflict-Triggered Fallback -# ============================================================================= - -proc checkForIsolationFallback*( - conflicts: seq[ConflictReport] -): Option[IsolationSuggestion] = - ## Check if conflicts warrant NipCell isolation and generate suggestion. - ## - ## **Requirements:** 10.1 - Detect unresolvable conflicts and suggest isolation - ## - ## **Returns:** Isolation suggestion if warranted, None otherwise - - if conflicts.len == 0: - return none(IsolationSuggestion) - - # Detect isolation candidates - let candidates = detectIsolationCandidates(conflicts) - - if candidates.len == 0: - return none(IsolationSuggestion) - - # Generate suggestion based on the first (primary) conflict - let suggestion = generateIsolationSuggestion(conflicts[0], candidates) - - return some(suggestion) - -proc handleUnresolvableConflict*( - manager: NipCellGraphManager, - conflict: ConflictReport, - autoCreate: bool = false -): tuple[suggestion: IsolationSuggestion, cellsCreated: seq[string]] = - ## Handle an unresolvable conflict by suggesting or creating cells. - ## - ## **Requirements:** 10.1, 10.2 - Detect conflicts and create cells - ## - ## **Parameters:** - ## - manager: The cell graph manager - ## - conflict: The conflict to handle - ## - autoCreate: If true, automatically create suggested cells - ## - ## **Returns:** Tuple of suggestion and list of created cell names - - let candidates = detectIsolationCandidates(@[conflict]) - let suggestion = generateIsolationSuggestion(conflict, candidates) - - var cellsCreated: seq[string] = @[] - - if autoCreate: - for cell in suggestion.suggestedCells: - let createResult = manager.createCell(cell.name, cell.description) - if createResult.success: - cellsCreated.add(cell.name) - - return (suggestion: suggestion, cellsCreated: cellsCreated) - -# ============================================================================= -# Cell Serialization (for persistence) -# ============================================================================= - -proc toJson*(cell: NipCellGraph): JsonNode = - ## Serialize a NipCell graph to JSON. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs - - result = %*{ - "cellName": cell.cellName, - "cellId": cell.cellId, - "packages": toSeq(cell.packages), - "created": $cell.created, - "lastModified": $cell.lastModified, - "metadata": cell.metadata - } - -proc fromJson*(json: JsonNode): NipCellGraph = - ## Deserialize a NipCell graph from JSON. - ## - ## **Requirements:** 10.3 - Maintain separate dependency graphs - - result = NipCellGraph( - cellName: json["cellName"].getStr(), - cellId: json["cellId"].getStr(), - graph: newDependencyGraph(), - packages: initHashSet[string](), - created: now(), # Would need proper parsing - lastModified: now(), - metadata: initTable[string, string]() - ) - - for pkg in json["packages"]: - result.packages.incl(pkg.getStr()) - - for key, value in json["metadata"].pairs: - result.metadata[key] = value.getStr() diff --git a/src/nip/resolver/optimizations.nim b/src/nip/resolver/optimizations.nim deleted file mode 100644 index 4406376..0000000 --- a/src/nip/resolver/optimizations.nim +++ /dev/null @@ -1,465 +0,0 @@ -## Resolver Optimizations -## -## This module contains optimized implementations of resolver operations -## identified as hot paths through profiling. -## -## **Optimizations:** -## - Bit vector variant unification (O(1) instead of O(n)) -## - Indexed conflict detection (O(n) instead of O(n²)) -## - Cached hash calculations -## - Memory pool allocations -## - Parallel dependency fetching - -import tables -import sets -import bitops -import strutils -import strformat -import ./variant_types -import ./dependency_graph -import ../manifest_parser # For SemanticVersion - -# ============================================================================ -# Bit Vector Variant Unification (Optimization 1) -# ============================================================================ - -type - VariantBitVector* = object - ## Bit vector representation of variant flags for O(1) operations - bits: uint64 - flagMap: Table[string, int] # Flag name → bit position - -const MAX_FLAGS = 64 # Maximum number of flags (uint64 limit) - -proc toBitVector*(demand: VariantDemand): VariantBitVector = - ## Convert variant demand to bit vector representation - ## - ## **Performance:** O(n) where n = number of flags - ## **Benefit:** Enables O(1) unification operations - - result.bits = 0 - result.flagMap = initTable[string, int]() - - var bitPos = 0 - for domainName, domain in demand.variantProfile.domains.pairs: - for flag in domain.flags: - if bitPos >= MAX_FLAGS: - break # Limit to 64 flags - result.flagMap[domainName & ":" & flag] = bitPos - result.bits = result.bits or (1'u64 shl bitPos) - bitPos += 1 - -proc unifyBitVectors*(v1, v2: VariantBitVector): VariantBitVector = - ## Unify two bit vectors using bitwise OR - ## - ## **Performance:** O(1) - single bitwise operation - ## **Speedup:** ~10-100x faster than string comparison - - result.bits = v1.bits or v2.bits - - # Merge flag maps - result.flagMap = v1.flagMap - for flag, pos in v2.flagMap: - if flag notin result.flagMap: - result.flagMap[flag] = pos - -proc toVariantDemand*(bv: VariantBitVector): VariantDemand = - ## Convert bit vector back to variant demand - - result = VariantDemand( - packageName: "", - variantProfile: VariantProfile( - domains: initTable[string, VariantDomain](), - hash: "" - ), - optional: false - ) - - # Extract flags from bit vector - for flagKey, pos in bv.flagMap: - if (bv.bits and (1'u64 shl pos)) != 0: - let parts = flagKey.split(":") - if parts.len == 2: - let domainName = parts[0] - let flag = parts[1] - if domainName notin result.variantProfile.domains: - result.variantProfile.domains[domainName] = VariantDomain( - name: domainName, - exclusivity: NonExclusive, - flags: initHashSet[string]() - ) - result.variantProfile.domains[domainName].flags.incl(flag) - -proc unifyVariantsFast*(v1, v2: VariantDemand): UnificationResult = - ## Fast variant unification using bit vectors - ## - ## **Performance:** O(n) where n = number of flags - ## **Speedup:** ~10-100x faster than naive string comparison - ## - ## **Example:** - ## ```nim - ## let v1 = VariantDemand(packageName: "nginx", ...) - ## let v2 = VariantDemand(packageName: "nginx", ...) - ## let result = unifyVariantsFast(v1, v2) - ## ``` - - # Convert to bit vectors - let bv1 = toBitVector(v1) - let bv2 = toBitVector(v2) - - # Unify using bitwise OR (O(1)) - let unified = unifyBitVectors(bv1, bv2) - - # Convert back to variant demand - var unifiedDemand = toVariantDemand(unified) - - # Copy package name from v1 - unifiedDemand.packageName = v1.packageName - - return UnificationResult( - kind: Unified, - profile: unifiedDemand.variantProfile - ) - -# ============================================================================ -# Indexed Conflict Detection (Optimization 2) -# ============================================================================ - -type - PackageIndex* = object - ## Index for fast package lookup by name - byName: Table[string, seq[PackageTerm]] - - VersionConflict* = object - ## Version conflict between two package terms - package1*: string - version1*: string - package2*: string - version2*: string - -proc buildPackageIndex*(packages: seq[PackageTerm]): PackageIndex = - ## Build index for fast package lookup - ## - ## **Performance:** O(n) where n = number of packages - ## **Benefit:** Enables O(1) lookup by name - - result.byName = initTable[string, seq[PackageTerm]]() - - for pkg in packages: - if pkg.packageName notin result.byName: - result.byName[pkg.packageName] = @[] - result.byName[pkg.packageName].add(pkg) - -proc detectVersionConflictsFast*(index: PackageIndex): seq[VersionConflict] = - ## Fast version conflict detection using index - ## - ## **Performance:** O(n) where n = number of packages - ## **Speedup:** ~n times faster than O(n²) naive approach - ## - ## **Example:** - ## ```nim - ## let packages = @[pkg1, pkg2, pkg3] - ## let index = buildPackageIndex(packages) - ## let conflicts = detectVersionConflictsFast(index) - ## ``` - - result = @[] - - # Only check packages with same name (O(n) instead of O(n²)) - for name, versions in index.byName: - if versions.len > 1: - # Multiple versions of same package - potential conflict - for i in 0.. 0: - return pool.freeList.pop() - - # Check if current block is full - if pool.currentIndex >= pool.blockSize: - # Allocate new block - pool.blocks.add(newSeq[T](pool.blockSize)) - pool.currentBlock += 1 - pool.currentIndex = 0 - - # Allocate from current block - result = addr pool.blocks[pool.currentBlock][pool.currentIndex] - pool.currentIndex += 1 - -proc deallocate*[T](pool: MemoryPool[T], obj: ptr T) = - ## Return object to pool - - pool.freeList.add(obj) - -proc clear*[T](pool: MemoryPool[T]) = - ## Clear pool and reset allocations - - pool.currentBlock = 0 - pool.currentIndex = 0 - pool.freeList.setLen(0) - -# ============================================================================ -# Parallel Dependency Fetching (Optimization 5) -# ============================================================================ - -# Parallel Dependency Fetching (Optimization 5) -# Note: Disabled for MVP - requires PackageSpec and ResolvedPackage types -# when compileOption("threads"): -# import threadpool -# -# proc fetchDependenciesParallel*(packages: seq[PackageSpec]): seq[ResolvedPackage] = -# ## Fetch dependencies in parallel -# ## **Performance:** ~n times faster where n = number of cores -# result = newSeq[ResolvedPackage](packages.len) -# var futures = newSeq[FlowVar[ResolvedPackage]](packages.len) -# for i, pkg in packages: -# futures[i] = spawn resolvePackage(pkg) -# for i in 0.. 0: - # Handle conflicts from unification - orchestrator.metrics.failedResolutions += 1 - orchestrator.metrics.conflictCount += buildResult.conflicts.len - return err[ResolutionResult, ResolutionError](ResolutionError( - kind: ConflictError, - packageName: rootPackage, - constraint: constraint, - conflict: none(ConflictReport), # TODO: Map UnificationResult to ConflictReport - suggestions: buildResult.warnings - )) - - var graph = buildResult.graph - endOperation(graphOpId) - - # Step 3: Translate to CNF - let cnfOpId = startOperation(SolverExecution, "cnf-translation") - var formula = newCNFFormula() - translateGraph(formula, graph) - - # Add root requirement - # We need to find the root term in the graph - # For now, we'll assume the first term added or use getRoots - let roots = graph.getRoots() - if roots.len > 0: - let rootTerm = roots[0] - let rootVersion = rootTerm.version - - discard translateRootRequirement( - formula, - rootTerm.packageName, - rootVersion, - rootTerm.variantProfile - ) - endOperation(cnfOpId) - - # Step 4: Solve constraints - let solverOpId = startOperation(SolverExecution, "solve-constraints") - var solver = newCDCLSolver(formula) - let solverResult = solver.solve() - endOperation(solverOpId) - - if not solverResult.isSat: - orchestrator.metrics.failedResolutions += 1 - # Convert solver conflict to report - let conflictReport = ConflictReport( - kind: VersionConflict, # Default to version conflict for now - packages: @[], # TODO: Extract packages from conflict - details: "Solver found a conflict: " & $solverResult.conflict.clause, - suggestions: @["Check package dependencies for conflicts"] - ) - - return err[ResolutionResult, ResolutionError](ResolutionError( - kind: ConflictError, - packageName: rootPackage, - constraint: constraint, - details: formatConflict(conflictReport), - conflict: some(conflictReport) - )) - - # Step 5: Synthesize builds (skipped for now) - - # Step 6: Calculate install order - let sortOpId = startOperation(TopologicalSort, "topo-sort") - let installOrder: seq[PackageTerm] = @[] # Placeholder - endOperation(sortOpId) - - # Step 7: Cache result - let cacheStoreOpId = startOperation(CacheOperation, "cache-store") - orchestrator.cache.put(cacheKey, graph) - endOperation(cacheStoreOpId) - - let resolutionTime = cpuTime() - startTime - orchestrator.metrics.totalTime += resolutionTime - orchestrator.metrics.successfulResolutions += 1 - - return ok[ResolutionResult, ResolutionError](ResolutionResult( - graph: graph, - installOrder: installOrder, - cacheHit: false, - resolutionTime: resolutionTime, - packageCount: graph.nodeCount() - )) - -# ============================================================================ -# Error Handling -# ============================================================================ - -proc formatError*(error: ResolutionError): string = - ## Format resolution error for user display. - ## - ## **Parameters:** - ## - error: Resolution error - ## - ## **Returns:** Formatted error message with suggestions - - case error.kind: - of ConflictError: - result = "❌ Dependency conflicts detected:\n\n" - - if error.conflict.isSome: - result.add("\n" & formatConflict(error.conflict.get())) - result &= "\n" - - result &= "\n💡 Suggestions:\n" - for suggestion in error.suggestions: - result &= " • " & suggestion & "\n" - - of PackageNotFoundError: - result = fmt"❌ Package not found: {error.packageName}\n\n" - result &= "💡 Suggestions:\n" - result &= " • Check package name spelling\n" - result &= " • Update repository metadata: nip update\n" - result &= fmt" • Search for similar packages: nip search {error.packageName}\n" - - of BuildFailureError: - result = fmt"❌ Build failed for {error.packageName}:\n\n" - result &= error.buildLog - result &= "\n\n💡 Suggestions:\n" - result &= " • Check build dependencies\n" - result &= " • Review build log for errors\n" - result &= " • Try different variant flags\n" - - of TimeoutError: - result = "❌ Resolution timeout exceeded\n\n" - result &= "💡 Suggestions:\n" - result &= " • Increase timeout: nip config set timeout 600\n" - result &= " • Check network connectivity\n" - result &= " • Simplify dependency constraints\n" - - of CacheError: - result = "❌ Cache error occurred\n\n" - result &= "💡 Suggestions:\n" - result &= " • Clear cache: nip cache clear\n" - result &= " • Check disk space\n" - result &= " • Disable cache temporarily: nip --no-cache resolve ...\n" - - of NetworkError: - result = "❌ Network error occurred\n\n" - result &= "💡 Suggestions:\n" - result &= " • Check internet connectivity\n" - result &= " • Verify repository URLs\n" - result &= " • Try again later\n" - -# ============================================================================ -# Metrics and Monitoring -# ============================================================================ - -proc getMetrics*(orchestrator: ResolutionOrchestrator): ResolverMetrics = - ## Get resolver performance metrics. - ## - ## **Returns:** Current metrics - - return orchestrator.metrics - -proc resetMetrics*(orchestrator: ResolutionOrchestrator) = - ## Reset metrics counters. - - orchestrator.metrics = ResolverMetrics() - -proc printMetrics*(orchestrator: ResolutionOrchestrator) = - ## Print metrics summary. - - let m = orchestrator.metrics - - echo "" - echo "=" .repeat(60) - echo "RESOLVER METRICS" - echo "=" .repeat(60) - echo "" - echo fmt"Total resolutions: {m.totalResolutions}" - echo fmt"Successful: {m.successfulResolutions}" - echo fmt"Failed: {m.failedResolutions}" - echo "" - - if m.totalResolutions > 0: - let avgTime = m.totalTime / m.totalResolutions.float - let successRate = (m.successfulResolutions.float / m.totalResolutions.float) * 100.0 - - echo fmt"Average time: {avgTime * 1000:.2f}ms" - echo fmt"Success rate: {successRate:.1f}%" - echo "" - - let totalCacheAccess = m.cacheHits + m.cacheMisses - if totalCacheAccess > 0: - let cacheHitRate = (m.cacheHits.float / totalCacheAccess.float) * 100.0 - - echo fmt"Cache hits: {m.cacheHits}" - echo fmt"Cache misses: {m.cacheMisses}" - echo fmt"Cache hit rate: {cacheHitRate:.1f}%" - echo "" - - if m.totalResolutions > 0: - let conflictRate = (m.conflictCount.float / m.totalResolutions.float) * 100.0 - echo fmt"Conflicts: {m.conflictCount} ({conflictRate:.1f}%)" - - echo "" - -# ============================================================================ -# Configuration Management -# ============================================================================ - -proc updateConfig*(orchestrator: ResolutionOrchestrator, config: ResolverConfig) = - ## Update resolver configuration. - ## - ## **Parameters:** - ## - config: New configuration - ## - ## **Effect:** Updates configuration and reinitializes cache if needed - - orchestrator.config = config - - # Update cache settings - orchestrator.cache.setEnabled(config.enableCache) - -proc getConfig*(orchestrator: ResolutionOrchestrator): ResolverConfig = - ## Get current resolver configuration. - - return orchestrator.config - -# ============================================================================ -# Cache Management -# ============================================================================ - -proc clearCache*(orchestrator: ResolutionOrchestrator) = - ## Clear resolver cache. - - orchestrator.cache.clear() - -proc getCacheMetrics*(orchestrator: ResolutionOrchestrator): CacheMetrics = - ## Get cache performance metrics. - - return orchestrator.cache.getMetrics() - -# ============================================================================ -# Repository Management -# ============================================================================ - -proc updateRepositories*(orchestrator: ResolutionOrchestrator, repos: seq[Repository]) = - ## Update available repositories. - ## - ## **Parameters:** - ## - repos: New repository list - ## - ## **Effect:** Updates repositories and invalidates cache - - orchestrator.repositories = repos - - # Invalidate cache (repo state changed) - let newRepoHash = calculateGlobalRepoStateHash(repos.mapIt(it.name & ":" & it.url)) - orchestrator.cache.updateRepoHash(newRepoHash) - -proc getRepositories*(orchestrator: ResolutionOrchestrator): seq[Repository] = - ## Get current repositories. - - return orchestrator.repositories - -# ============================================================================ -# Debug and Inspection -# ============================================================================ - -proc `$`*(orchestrator: ResolutionOrchestrator): string = - ## String representation for debugging. - - result = "ResolutionOrchestrator(\n" - result &= fmt" repositories: {orchestrator.repositories.len}\n" - result &= fmt" cache enabled: {orchestrator.config.enableCache}\n" - result &= fmt" parallel enabled: {orchestrator.config.enableParallel}\n" - result &= fmt" total resolutions: {orchestrator.metrics.totalResolutions}\n" - result &= ")" - -# ============================================================================ -# Unit Tests -# ============================================================================ - -when isMainModule: - import unittest - - suite "Resolution Orchestrator": - test "Create orchestrator": - let cas = newCASStorage("/tmp/test-orchestrator-cas") - let repos: seq[Repository] = @[] - let config = defaultConfig() - - let orchestrator = newResolutionOrchestrator(cas, repos, config) - - check orchestrator.getConfig().enableCache == true - check orchestrator.getMetrics().totalResolutions == 0 - - test "Resolve with empty graph": - let cas = newCASStorage("/tmp/test-orchestrator-cas-2") - let repos: seq[Repository] = @[] - let config = defaultConfig() - - let orchestrator = newResolutionOrchestrator(cas, repos, config) - - let result = orchestrator.resolve( - "test-pkg", - "*", - VariantDemand( - useFlags: @[], - libc: "musl", - allocator: "jemalloc", - targetArch: "x86_64", - buildFlags: @[] - ) - ) - - check result.isOk - check result.get.packageCount == 0 - check result.get.cacheHit == false - - test "Cache hit on second resolution": - let cas = newCASStorage("/tmp/test-orchestrator-cas-3") - let repos: seq[Repository] = @[] - let config = defaultConfig() - - let orchestrator = newResolutionOrchestrator(cas, repos, config) - - let demand = VariantDemand( - useFlags: @[], - libc: "musl", - allocator: "jemalloc", - targetArch: "x86_64", - buildFlags: @[] - ) - - # First resolution (cache miss) - let result1 = orchestrator.resolve("test-pkg", "*", demand) - check result1.isOk - check result1.get.cacheHit == false - - # Second resolution (cache hit) - let result2 = orchestrator.resolve("test-pkg", "*", demand) - check result2.isOk - check result2.get.cacheHit == true - - # Verify metrics - let metrics = orchestrator.getMetrics() - check metrics.totalResolutions == 2 - check metrics.cacheHits == 1 - check metrics.cacheMisses == 1 - - test "Update configuration": - let cas = newCASStorage("/tmp/test-orchestrator-cas-4") - let repos: seq[Repository] = @[] - let config = defaultConfig() - - let orchestrator = newResolutionOrchestrator(cas, repos, config) - - var newConfig = config - newConfig.enableCache = false - - orchestrator.updateConfig(newConfig) - - check orchestrator.getConfig().enableCache == false - - test "Clear cache": - let cas = newCASStorage("/tmp/test-orchestrator-cas-5") - let repos: seq[Repository] = @[] - let config = defaultConfig() - - let orchestrator = newResolutionOrchestrator(cas, repos, config) - - let demand = VariantDemand( - useFlags: @[], - libc: "musl", - allocator: "jemalloc", - targetArch: "x86_64", - buildFlags: @[] - ) - - # Resolve to populate cache - discard orchestrator.resolve("test-pkg", "*", demand) - - # Clear cache - orchestrator.clearCache() - - # Next resolution should be cache miss - let result = orchestrator.resolve("test-pkg", "*", demand) - check result.isOk - check result.get.cacheHit == false diff --git a/src/nip/resolver/persistent_cache.nim b/src/nip/resolver/persistent_cache.nim deleted file mode 100644 index 5f5a36d..0000000 --- a/src/nip/resolver/persistent_cache.nim +++ /dev/null @@ -1,477 +0,0 @@ -## Persistent Cache Index with SQLite -## -## This module provides an optional SQLite-backed persistent cache index -## that survives across nip invocations. This enables: -## - Fast cache lookups without CAS scanning -## - Cache statistics persistence -## - Cache metadata storage -## - Cross-session cache reuse -## -## **Architecture:** -## - SQLite database stores cache keys → CAS IDs mapping -## - Actual graph data stored in CAS (content-addressable) -## - Index provides O(1) lookup without CAS scanning -## -## **Use Cases:** -## - Persistent caching across nip invocations -## - Fast cache warmup on startup -## - Cache statistics tracking over time -## - Debugging and cache inspection - -import db_sqlite -import options -import times -import ./variant_types -import ./dependency_graph - -type - PersistentCacheIndex* = ref object - ## SQLite-backed persistent cache index - db: DbConn - dbPath: string - enabled: bool - - CacheEntry* = object - ## Cache entry metadata - cacheKey*: string - casId*: string - timestamp*: DateTime - hitCount*: int - lastAccess*: DateTime - - CacheIndexStats* = object - ## Persistent cache statistics - totalEntries*: int - totalHits*: int - oldestEntry*: DateTime - newestEntry*: DateTime - dbSize*: int64 - -# ============================================================================ -# Database Schema -# ============================================================================ - -const SCHEMA_VERSION = 1 - -const CREATE_TABLES = """ -CREATE TABLE IF NOT EXISTS cache_entries ( - cache_key TEXT PRIMARY KEY, - cas_id TEXT NOT NULL, - timestamp INTEGER NOT NULL, - hit_count INTEGER DEFAULT 0, - last_access INTEGER NOT NULL -); - -CREATE INDEX IF NOT EXISTS idx_last_access ON cache_entries(last_access); -CREATE INDEX IF NOT EXISTS idx_timestamp ON cache_entries(timestamp); - -CREATE TABLE IF NOT EXISTS cache_metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); - -INSERT OR IGNORE INTO cache_metadata (key, value) VALUES ('schema_version', '1'); -""" - -# ============================================================================ -# Index Construction -# ============================================================================ - -proc newPersistentCacheIndex*( - dbPath: string, - enabled: bool = true -): PersistentCacheIndex = - ## Create or open persistent cache index. - ## - ## **Parameters:** - ## - dbPath: Path to SQLite database file - ## - enabled: Enable/disable persistent caching - ## - ## **Returns:** New persistent cache index - ## - ## **Example:** - ## ```nim - ## let index = newPersistentCacheIndex("/var/lib/nip/cache.db") - ## ``` - - result = PersistentCacheIndex( - dbPath: dbPath, - enabled: enabled - ) - - if enabled: - # Open or create database - result.db = open(dbPath, "", "", "") - - # Create schema - result.db.exec(sql(CREATE_TABLES)) - -proc close*(index: PersistentCacheIndex) = - ## Close database connection. - - if index.enabled and not index.db.isNil: - index.db.close() - -# ============================================================================ -# Cache Operations -# ============================================================================ - -proc get*(index: PersistentCacheIndex, cacheKey: string): Option[string] = - ## Get CAS ID for cache key. - ## - ## **Parameters:** - ## - cacheKey: Cache key to lookup - ## - ## **Returns:** Some(casId) if found, None if not found - ## - ## **Side Effect:** Updates hit count and last access time - - if not index.enabled: - return none(string) - - let row = index.db.getRow(sql""" - SELECT cas_id FROM cache_entries WHERE cache_key = ? - """, cacheKey) - - if row[0].len > 0: - # Update hit count and last access - index.db.exec(sql""" - UPDATE cache_entries - SET hit_count = hit_count + 1, - last_access = ? - WHERE cache_key = ? - """, now().toTime().toUnix(), cacheKey) - - return some(row[0]) - else: - return none(string) - -proc put*(index: PersistentCacheIndex, cacheKey: string, casId: string) = - ## Store cache key → CAS ID mapping. - ## - ## **Parameters:** - ## - cacheKey: Cache key - ## - casId: CAS ID where graph is stored - ## - ## **Effect:** Inserts or updates cache entry - - if not index.enabled: - return - - let now = now().toTime().toUnix() - - index.db.exec(sql""" - INSERT OR REPLACE INTO cache_entries - (cache_key, cas_id, timestamp, hit_count, last_access) - VALUES (?, ?, ?, COALESCE((SELECT hit_count FROM cache_entries WHERE cache_key = ?), 0), ?) - """, cacheKey, casId, now, cacheKey, now) - -proc delete*(index: PersistentCacheIndex, cacheKey: string): bool = - ## Delete cache entry. - ## - ## **Parameters:** - ## - cacheKey: Cache key to delete - ## - ## **Returns:** true if entry was deleted, false if not found - - if not index.enabled: - return false - - let rowsBefore = index.db.getValue(sql"SELECT COUNT(*) FROM cache_entries").parseInt - - index.db.exec(sql""" - DELETE FROM cache_entries WHERE cache_key = ? - """, cacheKey) - - let rowsAfter = index.db.getValue(sql"SELECT COUNT(*) FROM cache_entries").parseInt - - return rowsBefore > rowsAfter - -proc clear*(index: PersistentCacheIndex) = - ## Clear all cache entries. - - if not index.enabled: - return - - index.db.exec(sql"DELETE FROM cache_entries") - -proc prune*(index: PersistentCacheIndex, olderThan: Duration): int = - ## Prune cache entries older than specified duration. - ## - ## **Parameters:** - ## - olderThan: Delete entries older than this duration - ## - ## **Returns:** Number of entries deleted - ## - ## **Example:** - ## ```nim - ## let deleted = index.prune(initDuration(days = 30)) - ## echo "Deleted ", deleted, " entries older than 30 days" - ## ``` - - if not index.enabled: - return 0 - - let cutoff = (now() - olderThan).toTime().toUnix() - - let rowsBefore = index.db.getValue(sql"SELECT COUNT(*) FROM cache_entries").parseInt - - index.db.exec(sql""" - DELETE FROM cache_entries WHERE last_access < ? - """, cutoff) - - let rowsAfter = index.db.getValue(sql"SELECT COUNT(*) FROM cache_entries").parseInt - - return rowsBefore - rowsAfter - -proc pruneLRU*(index: PersistentCacheIndex, keepCount: int): int = - ## Prune least recently used entries, keeping only specified count. - ## - ## **Parameters:** - ## - keepCount: Number of entries to keep - ## - ## **Returns:** Number of entries deleted - - if not index.enabled: - return 0 - - let rowsBefore = index.db.getValue(sql"SELECT COUNT(*) FROM cache_entries").parseInt - - if rowsBefore <= keepCount: - return 0 - - index.db.exec(sql""" - DELETE FROM cache_entries - WHERE cache_key NOT IN ( - SELECT cache_key FROM cache_entries - ORDER BY last_access DESC - LIMIT ? - ) - """, keepCount) - - let rowsAfter = index.db.getValue(sql"SELECT COUNT(*) FROM cache_entries").parseInt - - return rowsBefore - rowsAfter - -# ============================================================================ -# Statistics and Inspection -# ============================================================================ - -proc getStats*(index: PersistentCacheIndex): CacheIndexStats = - ## Get cache index statistics. - ## - ## **Returns:** Statistics including entry count, hits, age - - if not index.enabled: - return CacheIndexStats() - - let totalEntries = index.db.getValue(sql"SELECT COUNT(*) FROM cache_entries").parseInt - let totalHits = index.db.getValue(sql"SELECT SUM(hit_count) FROM cache_entries").parseInt - - let oldestTimestamp = index.db.getValue(sql"SELECT MIN(timestamp) FROM cache_entries") - let newestTimestamp = index.db.getValue(sql"SELECT MAX(timestamp) FROM cache_entries") - - let oldestEntry = if oldestTimestamp.len > 0: - fromUnix(oldestTimestamp.parseInt).local - else: - now() - - let newestEntry = if newestTimestamp.len > 0: - fromUnix(newestTimestamp.parseInt).local - else: - now() - - # Get database file size - let dbSize = 0'i64 # TODO: Get actual file size - - result = CacheIndexStats( - totalEntries: totalEntries, - totalHits: totalHits, - oldestEntry: oldestEntry, - newestEntry: newestEntry, - dbSize: dbSize - ) - -proc listEntries*(index: PersistentCacheIndex, limit: int = 100): seq[CacheEntry] = - ## List cache entries (most recently accessed first). - ## - ## **Parameters:** - ## - limit: Maximum number of entries to return - ## - ## **Returns:** Sequence of cache entries - - if not index.enabled: - return @[] - - result = @[] - - for row in index.db.fastRows(sql""" - SELECT cache_key, cas_id, timestamp, hit_count, last_access - FROM cache_entries - ORDER BY last_access DESC - LIMIT ? - """, limit): - result.add(CacheEntry( - cacheKey: row[0], - casId: row[1], - timestamp: fromUnix(row[2].parseInt).local, - hitCount: row[3].parseInt, - lastAccess: fromUnix(row[4].parseInt).local - )) - -proc getMostUsed*(index: PersistentCacheIndex, limit: int = 10): seq[CacheEntry] = - ## Get most frequently used cache entries. - ## - ## **Parameters:** - ## - limit: Maximum number of entries to return - ## - ## **Returns:** Sequence of cache entries sorted by hit count - - if not index.enabled: - return @[] - - result = @[] - - for row in index.db.fastRows(sql""" - SELECT cache_key, cas_id, timestamp, hit_count, last_access - FROM cache_entries - ORDER BY hit_count DESC - LIMIT ? - """, limit): - result.add(CacheEntry( - cacheKey: row[0], - casId: row[1], - timestamp: fromUnix(row[2].parseInt).local, - hitCount: row[3].parseInt, - lastAccess: fromUnix(row[4].parseInt).local - )) - -# ============================================================================ -# Maintenance Operations -# ============================================================================ - -proc vacuum*(index: PersistentCacheIndex) = - ## Vacuum database to reclaim space. - - if not index.enabled: - return - - index.db.exec(sql"VACUUM") - -proc analyze*(index: PersistentCacheIndex) = - ## Analyze database for query optimization. - - if not index.enabled: - return - - index.db.exec(sql"ANALYZE") - -# ============================================================================ -# Debug and Inspection -# ============================================================================ - -proc `$`*(stats: CacheIndexStats): string = - ## String representation of cache index statistics. - - result = "CacheIndexStats(\n" - result &= " total entries: " & $stats.totalEntries & "\n" - result &= " total hits: " & $stats.totalHits & "\n" - result &= " oldest entry: " & $stats.oldestEntry.format("yyyy-MM-dd HH:mm:ss") & "\n" - result &= " newest entry: " & $stats.newestEntry.format("yyyy-MM-dd HH:mm:ss") & "\n" - result &= " db size: " & $(stats.dbSize div 1024) & " KB\n" - result &= ")" - -proc `$`*(entry: CacheEntry): string = - ## String representation of cache entry. - - result = "CacheEntry(\n" - result &= " cache key: " & entry.cacheKey[0..min(31, entry.cacheKey.len-1)] & "...\n" - result &= " CAS ID: " & entry.casId[0..min(31, entry.casId.len-1)] & "...\n" - result &= " timestamp: " & entry.timestamp.format("yyyy-MM-dd HH:mm:ss") & "\n" - result &= " hit count: " & $entry.hitCount & "\n" - result &= " last access: " & entry.lastAccess.format("yyyy-MM-dd HH:mm:ss") & "\n" - result &= ")" - -# ============================================================================ -# Unit Tests -# ============================================================================ - -when isMainModule: - import unittest - import os - - suite "Persistent Cache Index": - setup: - let testDb = "/tmp/test-cache-" & $now().toTime().toUnix() & ".db" - - teardown: - if fileExists(testDb): - removeFile(testDb) - - test "Create index": - let index = newPersistentCacheIndex(testDb) - check index.enabled - index.close() - - test "Put and get entry": - let index = newPersistentCacheIndex(testDb) - - index.put("key1", "cas-id-123") - - let result = index.get("key1") - check result.isSome - check result.get == "cas-id-123" - - index.close() - - test "Get non-existent entry": - let index = newPersistentCacheIndex(testDb) - - let result = index.get("missing") - check result.isNone - - index.close() - - test "Update existing entry": - let index = newPersistentCacheIndex(testDb) - - index.put("key1", "cas-id-123") - index.put("key1", "cas-id-456") - - let result = index.get("key1") - check result.get == "cas-id-456" - - index.close() - - test "Delete entry": - let index = newPersistentCacheIndex(testDb) - - index.put("key1", "cas-id-123") - check index.delete("key1") - check index.get("key1").isNone - - index.close() - - test "Clear all entries": - let index = newPersistentCacheIndex(testDb) - - index.put("key1", "cas-id-123") - index.put("key2", "cas-id-456") - - index.clear() - - check index.get("key1").isNone - check index.get("key2").isNone - - index.close() - - test "Get statistics": - let index = newPersistentCacheIndex(testDb) - - index.put("key1", "cas-id-123") - index.put("key2", "cas-id-456") - - let stats = index.getStats() - check stats.totalEntries == 2 - - index.close() diff --git a/src/nip/resolver/profiler.nim b/src/nip/resolver/profiler.nim deleted file mode 100644 index 0a935e3..0000000 --- a/src/nip/resolver/profiler.nim +++ /dev/null @@ -1,440 +0,0 @@ -## Resolver Profiling Infrastructure -## -## This module provides profiling tools for measuring resolver performance -## and identifying optimization opportunities. -## -## **Features:** -## - Operation timing with high precision -## - Call count tracking -## - Hot path identification (top 10 by time and frequency) -## - Optimization recommendations -## - CSV export for detailed analysis - -import times -import tables -import algorithm -import strformat -import strutils - -# ============================================================================ -# Profiling Data Structures -# ============================================================================ - -type - OperationKind* = enum - ## Types of resolver operations to profile - VariantUnification - GraphConstruction - ConflictDetection - TopologicalSort - SolverExecution - BuildSynthesis - CacheOperation - HashCalculation - PackageResolution - DependencyFetch - - OperationTiming* = object - ## Timing data for a single operation - kind*: OperationKind - name*: string - startTime*: float - endTime*: float - duration*: float - - OperationStats* = object - ## Aggregated statistics for an operation type - kind*: OperationKind - name*: string - callCount*: int - totalTime*: float - minTime*: float - maxTime*: float - avgTime*: float - percentOfTotal*: float - - Profiler* = ref object - ## Main profiler object - enabled*: bool - timings*: seq[OperationTiming] - startTime*: float - endTime*: float - totalTime*: float - -# ============================================================================ -# Global Profiler Instance -# ============================================================================ - -var globalProfiler* = Profiler( - enabled: false, - timings: @[], - startTime: 0.0, - endTime: 0.0, - totalTime: 0.0 -) - -# ============================================================================ -# Profiler Control -# ============================================================================ - -proc enableProfiler*() = - ## Enable profiling - globalProfiler.enabled = true - globalProfiler.timings = @[] - globalProfiler.startTime = epochTime() - -proc disableProfiler*() = - ## Disable profiling - globalProfiler.enabled = false - globalProfiler.endTime = epochTime() - globalProfiler.totalTime = globalProfiler.endTime - globalProfiler.startTime - -proc isEnabled*(): bool = - ## Check if profiler is enabled - return globalProfiler.enabled - -proc clearProfiler*() = - ## Clear all profiling data - globalProfiler.timings = @[] - globalProfiler.startTime = 0.0 - globalProfiler.endTime = 0.0 - globalProfiler.totalTime = 0.0 - -# ============================================================================ -# Operation Timing -# ============================================================================ - -proc startOperation*(kind: OperationKind, name: string = ""): int = - ## Start timing an operation - ## - ## Returns an operation ID that should be passed to endOperation() - ## - ## **Example:** - ## ```nim - ## let opId = startOperation(VariantUnification, "unify-nginx") - ## # ... do work ... - ## endOperation(opId) - ## ``` - - if not globalProfiler.enabled: - return -1 - - let timing = OperationTiming( - kind: kind, - name: name, - startTime: epochTime(), - endTime: 0.0, - duration: 0.0 - ) - - globalProfiler.timings.add(timing) - return globalProfiler.timings.len - 1 - -proc endOperation*(opId: int) = - ## End timing an operation - ## - ## **Example:** - ## ```nim - ## let opId = startOperation(VariantUnification) - ## # ... do work ... - ## endOperation(opId) - ## ``` - - if not globalProfiler.enabled or opId < 0 or opId >= globalProfiler.timings.len: - return - - let endTime = epochTime() - globalProfiler.timings[opId].endTime = endTime - globalProfiler.timings[opId].duration = endTime - globalProfiler.timings[opId].startTime - -template profileOperation*(kind: OperationKind, name: string, body: untyped) = - ## Profile a block of code - ## - ## **Example:** - ## ```nim - ## profileOperation(VariantUnification, "unify-nginx"): - ## let result = unifyVariants(demands) - ## ``` - - let opId = startOperation(kind, name) - try: - body - finally: - endOperation(opId) - -# ============================================================================ -# Statistics Calculation -# ============================================================================ - -proc calculateStats*(): seq[OperationStats] = - ## Calculate aggregated statistics for all operations - ## - ## Returns statistics sorted by total time (descending) - - if globalProfiler.timings.len == 0: - return @[] - - # Group timings by operation kind - var statsByKind = initTable[OperationKind, OperationStats]() - - for timing in globalProfiler.timings: - if timing.kind notin statsByKind: - statsByKind[timing.kind] = OperationStats( - kind: timing.kind, - name: $timing.kind, - callCount: 0, - totalTime: 0.0, - minTime: high(float), - maxTime: 0.0, - avgTime: 0.0, - percentOfTotal: 0.0 - ) - - var stats = statsByKind[timing.kind] - stats.callCount += 1 - stats.totalTime += timing.duration - stats.minTime = min(stats.minTime, timing.duration) - stats.maxTime = max(stats.maxTime, timing.duration) - statsByKind[timing.kind] = stats - - # Calculate averages and percentages - let totalTime = globalProfiler.totalTime - - for kind, stats in statsByKind.mpairs: - stats.avgTime = stats.totalTime / stats.callCount.float - stats.percentOfTotal = (stats.totalTime / totalTime) * 100.0 - - # Convert to sequence and sort by total time - result = @[] - for stats in statsByKind.values: - result.add(stats) - - result.sort do (a, b: OperationStats) -> int: - if a.totalTime > b.totalTime: -1 - elif a.totalTime < b.totalTime: 1 - else: 0 - -proc getHotPaths*(limit: int = 10): seq[OperationStats] = - ## Get top N operations by total time - ## - ## **Example:** - ## ```nim - ## let hotPaths = getHotPaths(10) - ## for path in hotPaths: - ## echo fmt"{path.name}: {path.totalTime:.3f}s ({path.percentOfTotal:.1f}%)" - ## ``` - - let allStats = calculateStats() - - if allStats.len <= limit: - return allStats - - return allStats[0..15% of time - ## ``` - - let allStats = calculateStats() - - result = @[] - for stats in allStats: - if stats.percentOfTotal >= threshold: - result.add(stats) - -# ============================================================================ -# Reporting -# ============================================================================ - -proc printReport*() = - ## Print profiling report to stdout - - if globalProfiler.timings.len == 0: - echo "No profiling data available" - return - - echo "" - echo "=" .repeat(80) - echo "RESOLVER PROFILING REPORT" - echo "=" .repeat(80) - echo "" - echo fmt"Total time: {globalProfiler.totalTime:.3f}s" - echo fmt"Total operations: {globalProfiler.timings.len}" - echo "" - - # Print statistics table - echo "Operation Statistics:" - echo "-" .repeat(80) - echo "Operation Calls Total Avg Min Max %" - echo "-" .repeat(80) - - let stats = calculateStats() - for s in stats: - echo fmt"{s.name:<30} {s.callCount:>8} {s.totalTime:>10.3f}s {s.avgTime:>10.6f}s {s.minTime:>10.6f}s {s.maxTime:>10.6f}s {s.percentOfTotal:>5.1f}%" - - echo "-" .repeat(80) - echo "" - - # Print hot paths - echo "Hot Paths (Top 10 by time):" - echo "-" .repeat(80) - - let hotPaths = getHotPaths(10) - for i, path in hotPaths: - echo fmt"{i+1:>2}. {path.name:<30} {path.totalTime:>10.3f}s ({path.percentOfTotal:>5.1f}%)" - - echo "" - - # Print bottlenecks - let bottlenecks = getBottlenecks(15.0) - if bottlenecks.len > 0: - echo "Bottlenecks (>15% of total time):" - echo "-" .repeat(80) - - for bottleneck in bottlenecks: - echo fmt"⚠️ {bottleneck.name}: {bottleneck.totalTime:.3f}s ({bottleneck.percentOfTotal:.1f}%)" - - echo "" - -proc getOptimizationRecommendations*(): seq[string] = - ## Get optimization recommendations based on profiling data - ## - ## **Example:** - ## ```nim - ## let recommendations = getOptimizationRecommendations() - ## for rec in recommendations: - ## echo rec - ## ``` - - result = @[] - - let bottlenecks = getBottlenecks(15.0) - - if bottlenecks.len == 0: - result.add("✅ No major bottlenecks detected (all operations <15% of total time)") - return - - for bottleneck in bottlenecks: - case bottleneck.kind: - of VariantUnification: - result.add(fmt"🔧 Optimize variant unification ({bottleneck.percentOfTotal:.1f}% of time)") - result.add(" → Consider bit vector representation for O(1) operations") - result.add(" → Cache unification results for repeated demands") - - of GraphConstruction: - result.add(fmt"🔧 Optimize graph construction ({bottleneck.percentOfTotal:.1f}% of time)") - result.add(" → Use indexed lookups instead of linear scans") - result.add(" → Parallelize independent subgraph construction") - - of ConflictDetection: - result.add(fmt"🔧 Optimize conflict detection ({bottleneck.percentOfTotal:.1f}% of time)") - result.add(" → Build package index for O(n) instead of O(n²) checks") - result.add(" → Use bloom filters for quick negative checks") - - of SolverExecution: - result.add(fmt"🔧 Optimize solver execution ({bottleneck.percentOfTotal:.1f}% of time)") - result.add(" → Implement clause learning and caching") - result.add(" → Use better heuristics for variable selection") - - of HashCalculation: - result.add(fmt"🔧 Optimize hash calculation ({bottleneck.percentOfTotal:.1f}% of time)") - result.add(" → Cache hash results for repeated inputs") - result.add(" → Use faster hash algorithm (xxh3 instead of blake2b)") - - of CacheOperation: - result.add(fmt"🔧 Optimize cache operations ({bottleneck.percentOfTotal:.1f}% of time)") - result.add(" → Increase cache size to improve hit rate") - result.add(" → Use more efficient cache data structure") - - else: - result.add(fmt"🔧 Optimize {bottleneck.name} ({bottleneck.percentOfTotal:.1f}% of time)") - -proc printOptimizationRecommendations*() = - ## Print optimization recommendations - - echo "" - echo "=" .repeat(80) - echo "OPTIMIZATION RECOMMENDATIONS" - echo "=" .repeat(80) - echo "" - - let recommendations = getOptimizationRecommendations() - for rec in recommendations: - echo rec - - echo "" - -# ============================================================================ -# CSV Export -# ============================================================================ - -proc exportToCSV*(filename: string) = - ## Export profiling data to CSV file - ## - ## **Example:** - ## ```nim - ## exportToCSV("profiling_results.csv") - ## ``` - - var csv = "Operation,Name,CallCount,TotalTime,AvgTime,MinTime,MaxTime,PercentOfTotal\n" - - let stats = calculateStats() - for s in stats: - csv.add(fmt"{s.kind},{s.name},{s.callCount},{s.totalTime},{s.avgTime},{s.minTime},{s.maxTime},{s.percentOfTotal}\n") - - writeFile(filename, csv) - echo fmt"Profiling data exported to {filename}" - -proc exportDetailedToCSV*(filename: string) = - ## Export detailed timing data to CSV file - ## - ## **Example:** - ## ```nim - ## exportDetailedToCSV("profiling_detailed.csv") - ## ``` - - var csv = "Operation,Name,StartTime,EndTime,Duration\n" - - for timing in globalProfiler.timings: - csv.add(fmt"{timing.kind},{timing.name},{timing.startTime},{timing.endTime},{timing.duration}\n") - - writeFile(filename, csv) - echo fmt"Detailed profiling data exported to {filename}" - -# ============================================================================ -# Example Usage -# ============================================================================ - -when isMainModule: - import std/random - - # Enable profiler - enableProfiler() - - # Simulate some operations - for i in 0..<100: - profileOperation(VariantUnification, fmt"unify-{i}"): - sleep(rand(1..10)) # Simulate work - - if i mod 10 == 0: - profileOperation(GraphConstruction, fmt"graph-{i}"): - sleep(rand(5..15)) - - if i mod 20 == 0: - profileOperation(ConflictDetection, fmt"conflict-{i}"): - sleep(rand(10..30)) - - # Disable profiler - disableProfiler() - - # Print report - printReport() - printOptimizationRecommendations() - - # Export to CSV - exportToCSV("profiling_results.csv") - exportDetailedToCSV("profiling_detailed.csv") diff --git a/src/nip/resolver/resolution_cache.nim b/src/nip/resolver/resolution_cache.nim deleted file mode 100644 index eab472b..0000000 --- a/src/nip/resolver/resolution_cache.nim +++ /dev/null @@ -1,459 +0,0 @@ -## Resolution Cache with CAS Integration -## -## This module provides a two-tier caching system for dependency resolution: -## - **L1 Cache**: In-memory LRU cache for hot resolution results -## - **L2 Cache**: CAS-backed persistent storage for cold resolution results -## -## **Cache Key Strategy:** -## - Cache key includes global repository state hash -## - Any metadata change invalidates all cache entries automatically -## - Variant demand is canonicalized for deterministic keys -## -## **Performance:** -## - L1 hit: ~1μs (in-memory lookup) -## - L2 hit: ~100μs (CAS retrieval + deserialization) -## - Cache miss: ~100ms-1s (full resolution) -## -## **Invalidation:** -## - Automatic on repository metadata changes -## - Manual via clear() or invalidate() - -import options -import tables -import ./variant_types -import ./dependency_graph -import ./serialization -import ./lru_cache -import strutils - -type - ResolutionCache* = ref object - ## Two-tier cache for dependency resolution results - ## Note: L2 (CAS) integration is simplified for MVP - l1Cache: LRUCacheWithStats[string, DependencyGraph] - enabled: bool - l1Capacity: int - currentRepoHash: string - - CacheKey* = object - ## Key for caching resolution results - rootPackage*: string - rootConstraint*: string - repoStateHash*: string - variantDemand*: VariantDemand - - CacheResult*[T] = object - ## Result of cache lookup with source information - value*: Option[T] - source*: CacheSource - - CacheSource* = enum - ## Where the cached value came from - L1Hit, ## In-memory LRU cache - L2Hit, ## CAS persistent storage - CacheMiss ## Not found in cache - - CacheMetrics* = object - ## Cache performance metrics - l1Hits*: int - l2Hits*: int - misses*: int - l1Size*: int - l1Capacity*: int - l1HitRate*: float - totalHitRate*: float - -# ============================================================================ -# Cache Construction -# ============================================================================ - -proc newResolutionCache*( - l1Capacity: int = 100, - enabled: bool = true -): ResolutionCache = - ## Create a new resolution cache (L1 in-memory only for MVP). - ## - ## **Note:** L2 (CAS) integration simplified for MVP - ## - ## **Parameters:** - ## - l1Capacity: Maximum entries in L1 (in-memory) cache - ## - enabled: Enable/disable caching (for testing) - ## - ## **Returns:** New resolution cache instance - ## - ## **Example:** - ## ```nim - ## let cache = newResolutionCache(l1Capacity = 100) - ## ``` - - result = ResolutionCache( - l1Cache: newLRUCacheWithStats[string, DependencyGraph](l1Capacity), - enabled: enabled, - l1Capacity: l1Capacity, - currentRepoHash: "" - ) - -# ============================================================================ -# Cache Operations -# ============================================================================ - -proc calculateCacheKey*(key: CacheKey): string = - ## Calculate cache key hash from CacheKey object - serialization.calculateCacheKey( - key.rootPackage, - key.rootConstraint, - key.repoStateHash, - key.variantDemand - ) - -proc get*( - cache: ResolutionCache, - key: CacheKey -): CacheResult[DependencyGraph] = - ## Get dependency graph from cache (L1 → L2 → miss). - ## - ## **Parameters:** - ## - key: Cache key (includes repo state hash) - ## - ## **Returns:** Cache result with value and source - ## - ## **Lookup Order:** - ## 1. Check L1 (in-memory LRU cache) - ## 2. Check L2 (CAS persistent storage) - ## 3. Return cache miss - ## - ## **Complexity:** - ## - L1 hit: O(1) ~1μs - ## - L2 hit: O(1) ~100μs (CAS lookup + deserialization) - ## - Miss: O(1) ~1μs - - if not cache.enabled: - return CacheResult[DependencyGraph]( - value: none(DependencyGraph), - source: CacheMiss - ) - - # Calculate cache key hash - let cacheKeyHash = calculateCacheKey(key) - - # Try L1 cache (in-memory) - let l1Result = cache.l1Cache.get(cacheKeyHash) - if l1Result.isSome: - return CacheResult[DependencyGraph]( - value: l1Result, - source: L1Hit - ) - - # L2 cache (CAS) - Simplified for MVP - # TODO: Implement CAS integration when CASStorage type is available - - # Cache miss - return CacheResult[DependencyGraph]( - value: none(DependencyGraph), - source: CacheMiss - ) - -proc put*( - cache: ResolutionCache, - key: CacheKey, - graph: DependencyGraph -) = - ## Put dependency graph into cache (L1 + L2). - ## - ## **Parameters:** - ## - key: Cache key (includes repo state hash) - ## - graph: Dependency graph to cache - ## - ## **Storage:** - ## - L1: Stored in in-memory LRU cache - ## - L2: Serialized to MessagePack and stored in CAS - ## - ## **Complexity:** O(n) where n = graph size (serialization cost) - - if not cache.enabled: - return - - # Calculate cache key hash - let cacheKeyHash = calculateCacheKey(key) - - # Store in L1 cache (in-memory) - cache.l1Cache.put(cacheKeyHash, graph) - - # L2 cache (CAS) - Simplified for MVP - # TODO: Implement CAS storage when CASStorage type is available - # let serialized = toMessagePack(graph) - # discard cache.casStorage.store(cacheKeyHash, serialized) - -proc invalidate*(cache: ResolutionCache, key: CacheKey) = - ## Invalidate specific cache entry. - ## - ## **Parameters:** - ## - key: Cache key to invalidate - ## - ## **Effect:** Removes entry from L1 cache (L2 remains for potential reuse) - - if not cache.enabled: - return - - let cacheKeyHash = calculateCacheKey(key) - discard cache.l1Cache.delete(cacheKeyHash) - -proc clear*(cache: ResolutionCache) = - ## Clear all cache entries (L1 only, L2 remains). - ## - ## **Effect:** Clears in-memory L1 cache, CAS L2 cache remains - ## - ## **Note:** L2 cache is not cleared to preserve disk-backed cache - ## across nip invocations. Use clearAll() to clear both tiers. - - cache.l1Cache.clear() - cache.l1Cache.resetStats() - -proc clearAll*(cache: ResolutionCache) = - ## Clear all cache entries (L1 + L2). - ## - ## **Effect:** Clears both in-memory and CAS-backed caches - ## - ## **Warning:** This removes all cached resolution results from disk - - cache.clear() - # Note: CAS storage doesn't have a clearAll() method - # Individual entries are garbage collected based on reference tracking - -proc updateRepoHash*(cache: ResolutionCache, newHash: string) = - ## Update current repository state hash. - ## - ## **Parameters:** - ## - newHash: New global repository state hash - ## - ## **Effect:** If hash changed, clears L1 cache (automatic invalidation) - ## - ## **Rationale:** Repository metadata change invalidates all cached results - - if cache.currentRepoHash != newHash: - cache.currentRepoHash = newHash - cache.clear() # Invalidate all L1 entries - -proc isEnabled*(cache: ResolutionCache): bool = - ## Check if caching is enabled. - - return cache.enabled - -proc setEnabled*(cache: ResolutionCache, enabled: bool) = - ## Enable or disable caching. - ## - ## **Parameters:** - ## - enabled: true to enable, false to disable - ## - ## **Effect:** When disabled, all cache operations become no-ops - - cache.enabled = enabled - -# ============================================================================ -# Cache Metrics -# ============================================================================ - -proc getMetrics*(cache: ResolutionCache): CacheMetrics = - ## Get cache performance metrics. - ## - ## **Returns:** Metrics including hit rates, sizes, and sources - - let l1Stats = cache.l1Cache.getStats() - - # Calculate L2 hits (total hits - L1 hits) - # Note: This is approximate since we don't track L2 hits separately - let totalAccesses = l1Stats.hits + l1Stats.misses - let l2Hits = 0 # TODO: Track L2 hits separately - - let totalHits = l1Stats.hits + l2Hits - let totalHitRate = if totalAccesses > 0: - totalHits.float / totalAccesses.float - else: - 0.0 - - result = CacheMetrics( - l1Hits: l1Stats.hits, - l2Hits: l2Hits, - misses: l1Stats.misses, - l1Size: l1Stats.size, - l1Capacity: l1Stats.capacity, - l1HitRate: cache.l1Cache.hitRate(), - totalHitRate: totalHitRate - ) - -proc `$`*(metrics: CacheMetrics): string = - ## String representation of cache metrics. - - result = "CacheMetrics(\n" - result &= " L1 hits: " & $metrics.l1Hits & "\n" - result &= " L2 hits: " & $metrics.l2Hits & "\n" - result &= " Misses: " & $metrics.misses & "\n" - result &= " L1 size: " & $metrics.l1Size & "/" & $metrics.l1Capacity & "\n" - result &= " L1 hit rate: " & (metrics.l1HitRate * 100.0).formatFloat(ffDecimal, 2) & "%\n" - result &= " Total hit rate: " & (metrics.totalHitRate * 100.0).formatFloat(ffDecimal, 2) & "%\n" - result &= ")" - -# ============================================================================ -# Convenience Helpers -# ============================================================================ - -proc getCached*( - cache: ResolutionCache, - rootPackage: string, - rootConstraint: string, - repoStateHash: string, - variantDemand: VariantDemand -): CacheResult[DependencyGraph] = - ## Convenience method to get cached graph with individual parameters. - ## - ## **Parameters:** - ## - rootPackage: Root package name - ## - rootConstraint: Root package constraint - ## - repoStateHash: Global repository state hash - ## - variantDemand: Variant demand for resolution - ## - ## **Returns:** Cache result with value and source - - let key = CacheKey( - rootPackage: rootPackage, - rootConstraint: rootConstraint, - repoStateHash: repoStateHash, - variantDemand: variantDemand - ) - - return cache.get(key) - -proc putCached*( - cache: ResolutionCache, - rootPackage: string, - rootConstraint: string, - repoStateHash: string, - variantDemand: VariantDemand, - graph: DependencyGraph -) = - ## Convenience method to put graph into cache with individual parameters. - ## - ## **Parameters:** - ## - rootPackage: Root package name - ## - rootConstraint: Root package constraint - ## - repoStateHash: Global repository state hash - ## - variantDemand: Variant demand for resolution - ## - graph: Dependency graph to cache - - let key = CacheKey( - rootPackage: rootPackage, - rootConstraint: rootConstraint, - repoStateHash: repoStateHash, - variantDemand: variantDemand - ) - - cache.put(key, graph) - -# ============================================================================ -# Debug and Inspection -# ============================================================================ - -proc `$`*(cache: ResolutionCache): string = - ## String representation of cache for debugging. - - result = "ResolutionCache(\n" - result &= " enabled: " & $cache.enabled & "\n" - result &= " L1 capacity: " & $cache.l1Capacity & "\n" - result &= " L1 size: " & $cache.l1Cache.getStats().size & "\n" - result &= " current repo hash: " & cache.currentRepoHash & "\n" - result &= ")" - -# ============================================================================ -# Unit Tests -# ============================================================================ - -when isMainModule: - import unittest - - suite "Resolution Cache Basic Operations": - test "Create cache": - let cas = newCASStorage("/tmp/test-cas") - let cache = newResolutionCache(cas, l1Capacity = 10) - - check cache.isEnabled - check cache.l1Capacity == 10 - - test "Cache miss on empty cache": - let cas = newCASStorage("/tmp/test-cas") - let cache = newResolutionCache(cas) - - let key = CacheKey( - rootPackage: "nginx", - rootConstraint: ">=1.24.0", - repoStateHash: "hash123", - variantDemand: VariantDemand( - useFlags: @[], - libc: "musl", - allocator: "jemalloc", - targetArch: "x86_64", - buildFlags: @[] - ) - ) - - let result = cache.get(key) - check result.value.isNone - check result.source == CacheMiss - - test "Put and get from L1 cache": - let cas = newCASStorage("/tmp/test-cas") - let cache = newResolutionCache(cas) - - let key = CacheKey( - rootPackage: "nginx", - rootConstraint: ">=1.24.0", - repoStateHash: "hash123", - variantDemand: VariantDemand( - useFlags: @[], - libc: "musl", - allocator: "jemalloc", - targetArch: "x86_64", - buildFlags: @[] - ) - ) - - let graph = DependencyGraph( - rootPackage: PackageId(name: "nginx", version: "1.24.0", variant: "default"), - nodes: @[], - timestamp: 1700000000 - ) - - cache.put(key, graph) - - let result = cache.get(key) - check result.value.isSome - check result.source == L1Hit - check result.value.get.rootPackage.name == "nginx" - - test "Disabled cache returns miss": - let cas = newCASStorage("/tmp/test-cas") - let cache = newResolutionCache(cas, enabled = false) - - let key = CacheKey( - rootPackage: "nginx", - rootConstraint: ">=1.24.0", - repoStateHash: "hash123", - variantDemand: VariantDemand( - useFlags: @[], - libc: "musl", - allocator: "jemalloc", - targetArch: "x86_64", - buildFlags: @[] - ) - ) - - let graph = DependencyGraph( - rootPackage: PackageId(name: "nginx", version: "1.24.0", variant: "default"), - nodes: @[], - timestamp: 1700000000 - ) - - cache.put(key, graph) - - let result = cache.get(key) - check result.value.isNone - check result.source == CacheMiss diff --git a/src/nip/resolver/resolver_integration.nim b/src/nip/resolver/resolver_integration.nim deleted file mode 100644 index 9084737..0000000 --- a/src/nip/resolver/resolver_integration.nim +++ /dev/null @@ -1,330 +0,0 @@ -## Resolver Integration -## -## This module integrates the dependency graph, CNF translation, and CDCL solver -## to provide end-to-end dependency resolution. -## -## Philosophy: -## - Graph construction identifies package relationships -## - CNF translation converts constraints to boolean logic -## - PubGrub-style CDCL solver finds satisfying assignments -## - Solution is converted back to installation order -## -## The integration flow: -## 1. Build dependency graph from package requirements -## 2. Translate graph to CNF formula -## 3. Solve CNF using CDCL solver -## 4. Convert SAT model to package selections -## 5. Perform topological sort for installation order - -import std/[tables, sets, options, sequtils, strutils] -import ./dependency_graph -import ./cnf_translator -import ./cdcl_solver -import ./variant_types -import ../manifest_parser - -type - ## Resolution request from user - ResolutionRequest* = object - rootPackages*: seq[PackageSpec] - constraints*: seq[VariantConstraint] - - ## A package specification for resolution - PackageSpec* = object - packageName*: string - versionConstraint*: VersionConstraint - variantProfile*: VariantProfile - - ## A variant constraint - VariantConstraint* = object - packageName*: string - requiredFlags*: VariantProfile - - ## Resolution result - ResolutionResult* = object - case success*: bool - of true: - packages*: seq[ResolvedPackage] - installOrder*: seq[string] ## Topologically sorted - of false: - conflict*: ConflictReport - - ## A resolved package ready for installation - ResolvedPackage* = object - packageName*: string - version*: SemanticVersion - variant*: VariantProfile - source*: string - - ## Conflict report for user - ConflictReport* = object - conflictType*: ConflictType - packages*: seq[string] - details*: string - suggestions*: seq[string] - - ConflictType* = enum - VersionConflict, - VariantConflict, - CircularDependency, - Unsatisfiable - -# --- Graph to CNF Translation --- - -proc graphToCNF*(graph: DependencyGraph): CNFFormula = - ## Convert a dependency graph to a CNF formula - ## - ## Requirements: 5.1 - Use PubGrub algorithm with CDCL - - var formula = newCNFFormula() - - # For each term in the graph, create a boolean variable - for termId, term in graph.terms.pairs: - let variable = BoolVar( - package: term.packageName, - version: term.version, # Use actual version from term - variant: term.variantProfile - ) - discard formula.getOrCreateVarId(variable) - - # For each edge, create an implication clause - for edge in graph.edges: - let fromTerm = graph.terms[edge.fromTerm] - let toTerm = graph.terms[edge.toTerm] - - discard formula.translateDependency( - dependent = fromTerm.packageName, - dependentVersion = fromTerm.version, # Use actual version - dependentVariant = fromTerm.variantProfile, - dependency = toTerm.packageName, - dependencyVersion = toTerm.version, # Use actual version - dependencyVariant = toTerm.variantProfile - ) - - return formula - -# --- Solution to Package Selection --- - -proc modelToPackages*(model: Table[BoolVar, bool], graph: DependencyGraph): seq[ResolvedPackage] = - ## Convert a SAT model to a list of resolved packages - ## - ## Requirements: 5.4 - Produce deterministic installation order - - var packages: seq[ResolvedPackage] = @[] - var seen = initHashSet[string]() # Track package names to avoid duplicates - - for variable, value in model.pairs: - if value: # Only include selected packages - # Create a unique key for this package (name + version + variant) - let key = variable.package & "-" & $variable.version & "-" & variable.variant.hash - - if key notin seen: - seen.incl(key) - - # Find corresponding term in graph to get source - var foundSource = "unknown" - for termId, term in graph.terms.pairs: - if term.packageName == variable.package and - term.version == variable.version and - term.variantProfile.hash == variable.variant.hash: - foundSource = term.source - break - - packages.add(ResolvedPackage( - packageName: variable.package, - version: variable.version, - variant: variable.variant, - source: foundSource # Use actual source from graph - )) - - return packages - -# --- Main Resolution Function --- - -proc resolve*(request: ResolutionRequest, graph: DependencyGraph): ResolutionResult = - ## Main resolution function - integrates all components - ## - ## This is the complete end-to-end resolution pipeline: - ## 1. Build dependency graph (already done, passed as parameter) - ## 2. Translate graph to CNF formula - ## 3. Solve CNF using CDCL solver - ## 4. Convert SAT model to package selections - ## 5. Perform topological sort for installation order - ## - ## Requirements: 5.1, 5.4, 5.5 - - # Step 1: Check for circular dependencies in graph - if graph.hasCycle(): - let cycle = graph.findCycle() - var cyclePackages: seq[string] = @[] - if cycle.len > 0: - for term in cycle: - cyclePackages.add(term.packageName) - - return ResolutionResult( - success: false, - conflict: ConflictReport( - conflictType: CircularDependency, - packages: cyclePackages, - details: "Circular dependency detected: " & cyclePackages.join(" -> "), - suggestions: @[ - "Break the circular dependency by making one dependency optional", - "Check if this is a bug in package metadata" - ] - ) - ) - - # Step 2: Translate graph to CNF - var formula = graphToCNF(graph) - - # Step 3: Add root requirements as unit clauses - # Find the root package terms in the graph and add them as unit clauses - for spec in request.rootPackages: - # Find matching term in graph - var foundTerm = false - for termId, term in graph.terms.pairs: - if term.packageName == spec.packageName: - # Add this term as a unit clause (must be selected) - discard formula.translateRootRequirement( - package = term.packageName, - version = term.version, - variant = term.variantProfile - ) - foundTerm = true - break - - if not foundTerm: - # Root package not in graph - this shouldn't happen - return ResolutionResult( - success: false, - conflict: ConflictReport( - conflictType: Unsatisfiable, - packages: @[spec.packageName], - details: "Root package " & spec.packageName & " not found in dependency graph", - suggestions: @["Check package name", "Ensure package exists in repository"] - ) - ) - - # Step 4: Validate CNF is well-formed - if not formula.isValidCNF(): - return ResolutionResult( - success: false, - conflict: ConflictReport( - conflictType: Unsatisfiable, - packages: @[], - details: "Invalid CNF formula generated", - suggestions: @["Check package specifications", "Report this as a bug"] - ) - ) - - # Step 5: Solve using CDCL - var solver = newCDCLSolver(formula) - let solverResult = solver.solve() - - # Step 6: Handle result - if solverResult.isSat: - # Success! Convert model to packages - let packages = modelToPackages(solverResult.model, graph) - - # Step 7: Compute installation order using topological sort - # Build a subgraph containing only selected packages - var selectedGraph = newDependencyGraph() - var selectedTermIds = initHashSet[PackageTermId]() - - # Add selected terms to subgraph - for pkg in packages: - for termId, term in graph.terms.pairs: - if term.packageName == pkg.packageName and - term.variantProfile.hash == pkg.variant.hash: - selectedGraph.addTerm(term) - selectedTermIds.incl(termId) - break - - # Add edges between selected terms - for edge in graph.edges: - if edge.fromTerm in selectedTermIds and edge.toTerm in selectedTermIds: - selectedGraph.addEdge(edge) - - # Perform topological sort on selected subgraph - try: - let sortedTermIds = selectedGraph.topologicalSort() - var installOrder: seq[string] = @[] - - for termId in sortedTermIds: - let term = selectedGraph.getTerm(termId) - if term.isSome: - installOrder.add(term.get().packageName) - - return ResolutionResult( - success: true, - packages: packages, - installOrder: installOrder - ) - except ValueError as e: - # This shouldn't happen since we already checked for cycles - return ResolutionResult( - success: false, - conflict: ConflictReport( - conflictType: CircularDependency, - packages: @[], - details: "Unexpected cycle in selected packages: " & e.msg, - suggestions: @["Report this as a bug"] - ) - ) - else: - # Conflict detected - analyze and report - let conflict = solverResult.conflict - - # Extract package names from conflict - var conflictPackages: seq[string] = @[] - for assignment in conflict.assignments: - if assignment.decisionLevel > 0: # Skip root assignments - conflictPackages.add(assignment.variable.package) - - return ResolutionResult( - success: false, - conflict: ConflictReport( - conflictType: Unsatisfiable, - packages: conflictPackages, - details: "No satisfying assignment found: " & $conflict.clause, - suggestions: @[ - "Check for conflicting version requirements", - "Check for incompatible variant flags", - "Try relaxing version constraints", - "Consider using different package sources" - ] - ) - ) - -# --- Simplified Resolution (for testing) --- - -proc resolveSimple*(rootPackage: string, rootVariant: VariantProfile): ResolutionResult = - ## Simplified resolution for a single root package - ## Useful for testing and simple use cases - - # Create a simple graph with just the root - var graph = newDependencyGraph() - let termId = createTermId(rootPackage, rootVariant.hash) - let term = PackageTerm( - id: termId, - packageName: rootPackage, - variantProfile: rootVariant, - optional: false, - source: "test" - ) - graph.addTerm(term) - - # Create resolution request - let request = ResolutionRequest( - rootPackages: @[PackageSpec( - packageName: rootPackage, - versionConstraint: VersionConstraint( - operator: OpAny, - version: SemanticVersion(major: 1, minor: 0, patch: 0) - ), - variantProfile: rootVariant - )], - constraints: @[] - ) - - return resolve(request, graph) diff --git a/src/nip/resolver/serialization.nim b/src/nip/resolver/serialization.nim deleted file mode 100644 index 38e5030..0000000 --- a/src/nip/resolver/serialization.nim +++ /dev/null @@ -1,269 +0,0 @@ -## Binary Serialization Layer for Dependency Resolution Caching -## -## This module provides MessagePack-based serialization for DependencyGraph -## and related structures, ensuring deterministic, compact binary representation -## for cache storage and retrieval. -## -## **Design Principles:** -## - Deterministic: Same graph always produces identical binary output -## - Compact: MessagePack provides efficient binary encoding -## - Fast: Minimal overhead for serialization/deserialization -## - Canonical: Sorted keys and stable ordering guarantee reproducibility -## -## **Cache Invalidation Strategy:** -## The cache key includes a global repository state hash, ensuring that any -## change to package metadata automatically invalidates stale cache entries. - -import msgpack4nim -import tables -import algorithm -import sequtils -import sets -import strutils -import ./variant_types -import ./dependency_graph -import ../utils/hashing -import ../manifest_parser # For SemanticVersion - -# ============================================================================ -# Canonical Serialization Helpers -# ============================================================================ - -proc canonicalizeVariantDemand*(demand: VariantDemand): string = - ## Convert VariantDemand to canonical string representation. - ## Ensures deterministic ordering of flags and settings. - var parts: seq[string] - - # Add package name - parts.add("pkg:" & demand.packageName) - - # Add variant profile (sorted domains and flags) - var sortedDomains: seq[string] = @[] - for k in demand.variantProfile.domains.keys: - sortedDomains.add(k) - sortedDomains.sort() - - for domainName in sortedDomains: - let domain = demand.variantProfile.domains[domainName] - var sortedFlags: seq[string] = @[] - for flag in domain.flags: - sortedFlags.add(flag) - sortedFlags.sort() - - let exclusive = if domain.exclusivity == Exclusive: "!" else: "" - parts.add(domainName & exclusive & ":" & sortedFlags.join(",")) - - # Add optional flag - if demand.optional: - parts.add("optional:true") - - return parts.join("|") - -proc canonicalizePackageTerm*(term: PackageTerm): string = - ## Convert PackageTerm to canonical string representation. - return term.packageName & "@" & $term.version & "#" & term.variantProfile.hash - -proc canonicalizePackageTermId*(id: PackageTermId): string = - ## Convert PackageTermId to canonical string representation. - return $id - -# ============================================================================ -# DependencyGraph Serialization -# ============================================================================ - -type - SerializedTerm = object - ## Intermediate representation for MessagePack encoding - id: string - packageName: string - version: string - variantHash: string - optional: bool - source: string - - SerializedEdge = object - ## Serialized dependency edge - fromId: string - toId: string - depType: string - - SerializedGraph = object - ## Complete serialized dependency graph - terms: seq[SerializedTerm] - edges: seq[SerializedEdge] - -proc toSerializedTerm(term: PackageTerm): SerializedTerm = - ## Convert PackageTerm to serializable form - result.id = $term.id - result.packageName = term.packageName - result.version = $term.version - result.variantHash = term.variantProfile.hash - result.optional = term.optional - result.source = term.source - -proc toSerializedEdge(edge: DependencyEdge): SerializedEdge = - ## Convert DependencyEdge to serializable form - result.fromId = $edge.fromTerm - result.toId = $edge.toTerm - result.depType = $edge.dependencyType - -proc toMessagePack*(graph: DependencyGraph): string = - ## Serialize DependencyGraph to MessagePack binary format. - ## - ## **Guarantees:** - ## - Deterministic: Same graph always produces identical output - ## - Canonical: Terms sorted by ID for stable ordering - ## - Complete: All metadata and relationships preserved - ## - ## **Returns:** Binary MessagePack string - - var sgraph = SerializedGraph() - - # Convert all terms to serialized form - sgraph.terms = newSeq[SerializedTerm]() - for id, term in graph.terms: - sgraph.terms.add(toSerializedTerm(term)) - - # Sort terms by ID for determinism - sgraph.terms.sort(proc(a, b: SerializedTerm): int = - cmp(a.id, b.id) - ) - - # Convert all edges to serialized form - sgraph.edges = newSeq[SerializedEdge]() - for edge in graph.edges: - sgraph.edges.add(toSerializedEdge(edge)) - - # Sort edges for determinism - sgraph.edges.sort(proc(a, b: SerializedEdge): int = - let cmpFrom = cmp(a.fromId, b.fromId) - if cmpFrom != 0: cmpFrom else: cmp(a.toId, b.toId) - ) - - # Pack to MessagePack binary - return pack(sgraph) - -proc fromMessagePack*(data: string): DependencyGraph = - ## Deserialize DependencyGraph from MessagePack binary format. - ## - ## **Parameters:** - ## - data: Binary MessagePack string - ## - ## **Returns:** Reconstructed DependencyGraph - ## - ## **Raises:** UnpackError if data is corrupted or invalid - ## - ## **Note:** This is a simplified reconstruction that may not preserve - ## all graph invariants. Use with caution. - - let sgraph = unpack(data, SerializedGraph) - - result = DependencyGraph( - terms: initTable[PackageTermId, PackageTerm](), - edges: @[], - incomingEdges: initTable[PackageTermId, seq[DependencyEdge]](), - outgoingEdges: initTable[PackageTermId, seq[DependencyEdge]]() - ) - - # Reconstruct terms (simplified - doesn't fully reconstruct VariantProfile) - for sterm in sgraph.terms: - let id = PackageTermId(sterm.id) - let term = PackageTerm( - id: id, - packageName: sterm.packageName, - version: parseSemanticVersion(sterm.version), - variantProfile: VariantProfile( - domains: initTable[string, VariantDomain](), - hash: sterm.variantHash - ), - optional: sterm.optional, - source: sterm.source - ) - result.terms[id] = term - - # Reconstruct edges - for sedge in sgraph.edges: - let edge = DependencyEdge( - fromTerm: PackageTermId(sedge.fromId), - toTerm: PackageTermId(sedge.toId), - dependencyType: Required # Simplified - ) - result.edges.add(edge) - -# ============================================================================ -# Cache Key Calculation -# ============================================================================ - -proc calculateGlobalRepoStateHash*(metadataStrings: seq[string]): string = - ## Calculate deterministic hash of all repository metadata. - ## - ## **Purpose:** This hash serves as the cache invalidation key. Any change - ## to package metadata will produce a different hash, automatically - ## invalidating stale cache entries. - ## - ## **Algorithm:** - ## 1. Sort all metadata strings lexicographically - ## 2. Serialize sorted list to MessagePack - ## 3. Hash the final binary with xxh3_128 - ## - ## **Guarantees:** - ## - Deterministic: Same repo state always produces same hash - ## - Sensitive: Any metadata change produces different hash - ## - Fast: xxh3_128 provides high-speed hashing - ## - ## **Returns:** 128-bit hash as hex string - - var sortedMetadata = metadataStrings - sortedMetadata.sort() - - # Pack sorted metadata and compute final hash - let sortedBinary = pack(sortedMetadata) - return xxh3_128(sortedBinary) - -proc calculateCacheKey*(rootPackage: string, rootConstraint: string, - repoStateHash: string, demand: VariantDemand): string = - ## Calculate deterministic cache key using xxh3_128. - ## - ## **Purpose:** Generate a unique, deterministic identifier for a specific - ## dependency resolution request. The key captures all inputs that affect - ## the resolution result. - ## - ## **Components:** - ## - Root package name and constraint - ## - Global repository state hash (for invalidation) - ## - Variant demand (canonicalized) - ## - ## **Algorithm:** - ## 1. Canonicalize variant demand (sorted flags, stable ordering) - ## 2. Assemble all components in fixed order - ## 3. Serialize to MessagePack binary - ## 4. Hash with xxh3_128 - ## - ## **Guarantees:** - ## - Deterministic: Same inputs always produce same key - ## - Unique: Different inputs produce different keys (with high probability) - ## - Fast: xxh3_128 provides high-speed hashing - ## - ## **Returns:** 128-bit hash as hex string - - # Canonicalize the most complex structure - let canonicalDemand = canonicalizeVariantDemand(demand) - - # Assemble all components in fixed order - let components = @[ - rootPackage, - rootConstraint, - repoStateHash, - canonicalDemand - ] - - # Serialize to canonical binary - let encoded = pack(components) - - # Hash the binary - return xxh3_128(encoded) - -# ============================================================================ -# Serialization Tests (Determinism Verification) -# ============================================================================ -# Note: Tests moved to tests/test_serialization.nim to use proper test fixtures diff --git a/src/nip/resolver/solver_types.nim b/src/nip/resolver/solver_types.nim deleted file mode 100644 index bbc2d7d..0000000 --- a/src/nip/resolver/solver_types.nim +++ /dev/null @@ -1,378 +0,0 @@ -## Solver Data Structures for PubGrub-Style Dependency Resolution -## -## This module defines the core data structures for the PubGrub solver, -## adapted for NexusOS variant system. -## -## Philosophy: -## - Terms represent assertions about packages (positive or negative) -## - Incompatibilities represent mutually exclusive states -## - Assignments track the solver's current decisions -## - Derivations provide human-readable error messages -## -## Key Concepts: -## - A Term is "Package P satisfies Constraint C" -## - An Incompatibility is "¬(Term1 ∧ Term2 ∧ ... ∧ TermN)" -## - The solver finds an Assignment that satisfies all Incompatibilities - -import std/[strutils, hashes, tables, sets, options] -import ../manifest_parser # For SemanticVersion, VersionConstraint -import ./variant_types # For VariantProfile - -type - PackageId* = string - - ## A constraint on a package's version and variants - ## This represents the mathematical "Range" of valid states - Constraint* = object - versionRange*: VersionConstraint - variantReq*: VariantProfile - - # If true, this constraint implies "NOT this range" - isNegative*: bool - - ## A Term is a specific assertion about a package - ## Logic: "Package P satisfies Constraint C" - ## Example: Term(nginx, >=1.20 +wayland) - Term* = object - package*: PackageId - constraint*: Constraint - - ## The cause of an incompatibility (for error reporting) - ## This enables PubGrub's human-readable error messages - IncompatibilityCause* = enum - Root, ## The user requested this - Dependency, ## Package A depends on B - VariantConflict, ## +wayland vs +x11 are mutually exclusive - BuildHashMismatch, ## Different build configurations conflict - NoVersions, ## No versions satisfy the constraint - PackageNotFound ## Package doesn't exist in any source - - ## An Incompatibility is a set of Terms that are mutually exclusive - ## Logic: ¬(Term1 ∧ Term2 ∧ ... ∧ TermN) - ## Or: at least one of the Terms must be false - ## - ## Example: "nginx depends on zlib" becomes: - ## Incompatibility([Term(nginx, >=1.20), Term(zlib, NOT >=1.0)]) - ## Meaning: "It's incompatible to have nginx >=1.20 AND NOT have zlib >=1.0" - Incompatibility* = object - terms*: seq[Term] - cause*: IncompatibilityCause - - # For error reporting (PubGrub's magic) - externalContext*: string ## Human-readable explanation - fromPackage*: Option[PackageId] ## Which package caused this - fromVersion*: Option[SemanticVersion] ## Which version caused this - - ## An Assignment represents a decision made by the solver - ## It maps packages to specific versions/variants - Assignment* = object - package*: PackageId - version*: SemanticVersion - variant*: VariantProfile - - # Decision level (for backtracking) - decisionLevel*: int - - # Why was this assignment made? - cause*: Option[Incompatibility] - - ## The solver's current state - ## Tracks all assignments and incompatibilities - SolverState* = object - assignments*: Table[PackageId, Assignment] - incompatibilities*: seq[Incompatibility] - - # Current decision level (incremented on each choice) - decisionLevel*: int - - # Packages we've already processed - processed*: HashSet[PackageId] - -# --- String Representations --- - -proc `$`*(c: Constraint): string = - ## String representation of a constraint - result = $c.versionRange.operator & " " & $c.versionRange.version - - if c.variantReq.domains.len > 0: - result.add(" ") - for domain, variantDomain in c.variantReq.domains.pairs: - for flag in variantDomain.flags: - result.add("+" & domain & ":" & flag & " ") - - if c.isNegative: - result = "NOT (" & result & ")" - -proc `$`*(t: Term): string = - ## String representation of a term - result = t.package & " " & $t.constraint - -proc `$`*(i: Incompatibility): string = - ## String representation of an incompatibility - result = "Incompatibility(" - for idx, term in i.terms: - if idx > 0: - result.add(" AND ") - result.add($term) - result.add(")") - - if i.externalContext.len > 0: - result.add(" [" & i.externalContext & "]") - -proc `$`*(a: Assignment): string = - ## String representation of an assignment - result = a.package & " = " & $a.version - if a.variant.domains.len > 0: - result.add(" " & a.variant.hash) - -# --- Hash Functions --- - -proc hash*(c: Constraint): Hash = - ## Hash function for Constraint - var h: Hash = 0 - h = h !& hash(c.versionRange.operator) - h = h !& hash($c.versionRange.version) - h = h !& hash(c.variantReq.hash) - h = h !& hash(c.isNegative) - result = !$h - -proc hash*(t: Term): Hash = - ## Hash function for Term - var h: Hash = 0 - h = h !& hash(t.package) - h = h !& hash(t.constraint) - result = !$h - -# --- Equality --- - -proc `==`*(a, b: Constraint): bool = - ## Equality for Constraint - result = a.versionRange.operator == b.versionRange.operator and - a.versionRange.version == b.versionRange.version and - a.variantReq.hash == b.variantReq.hash and - a.isNegative == b.isNegative - -proc `==`*(a, b: Term): bool = - ## Equality for Term - result = a.package == b.package and a.constraint == b.constraint - -# --- Constraint Operations --- - -proc isAny*(c: Constraint): bool = - ## Check if constraint accepts any version - result = c.versionRange.operator == OpAny and not c.isNegative - -proc isEmpty*(c: Constraint): bool = - ## Check if constraint is empty (no versions satisfy it) - result = c.isNegative and c.versionRange.operator == OpAny - -proc satisfies*(version: SemanticVersion, variant: VariantProfile, constraint: Constraint): bool = - ## Check if a specific version/variant satisfies a constraint - - # Check if negated - if constraint.isNegative: - return not satisfies(version, variant, Constraint( - versionRange: constraint.versionRange, - variantReq: constraint.variantReq, - isNegative: false - )) - - # Check version constraint - if not satisfiesConstraint(version, constraint.versionRange): - return false - - # Check variant requirements - # For now, we check if all required domains/flags are present - for domain, variantDomain in constraint.variantReq.domains.pairs: - if not variant.domains.hasKey(domain): - return false - - # Check if all required flags in this domain are present - for flag in variantDomain.flags: - if flag notin variant.domains[domain].flags: - return false - - return true - -proc intersect*(a, b: Constraint): Option[Constraint] = - ## Compute the intersection of two constraints - ## Returns None if the constraints are incompatible - ## - ## This is the heart of constraint solving: - ## - What is the intersection of >=1.0 and <2.0? (1.0 <= v < 2.0) - ## - What is the intersection of +wayland and +x11 (if exclusive)? (Empty/Conflict) - - # TODO: Implement full constraint intersection logic - # For now, return a simple implementation - - # If either is empty, result is empty - if a.isEmpty or b.isEmpty: - return none(Constraint) - - # If either is "any", return the other - if a.isAny: - return some(b) - if b.isAny: - return some(a) - - # For now, if constraints are equal, return one of them - if a == b: - return some(a) - - # Otherwise, we need to compute the actual intersection - # This requires version range intersection logic - # TODO: Implement this properly - return none(Constraint) - -proc union*(a, b: Constraint): Option[Constraint] = - ## Compute the union of two constraints - ## Returns None if the constraints cannot be unified - - # TODO: Implement full constraint union logic - # For now, return a simple implementation - - # If either is "any", result is "any" - if a.isAny or b.isAny: - return some(Constraint( - versionRange: VersionConstraint(operator: OpAny), - variantReq: newVariantProfile(), - isNegative: false - )) - - # If constraints are equal, return one of them - if a == b: - return some(a) - - # Otherwise, we need to compute the actual union - # TODO: Implement this properly - return none(Constraint) - -# --- Term Operations --- - -proc negate*(t: Term): Term = - ## Negate a term - ## NOT (P satisfies C) = P satisfies (NOT C) - result = Term( - package: t.package, - constraint: Constraint( - versionRange: t.constraint.versionRange, - variantReq: t.constraint.variantReq, - isNegative: not t.constraint.isNegative - ) - ) - -proc isPositive*(t: Term): bool = - ## Check if term is positive (not negated) - result = not t.constraint.isNegative - -proc isNegative*(t: Term): bool = - ## Check if term is negative (negated) - result = t.constraint.isNegative - -# --- Incompatibility Operations --- - -proc createDependencyIncompatibility*( - dependent: PackageId, - dependentVersion: SemanticVersion, - dependency: PackageId, - dependencyConstraint: Constraint -): Incompatibility = - ## Create an incompatibility from a dependency - ## "Package A version V depends on B with constraint C" becomes: - ## Incompatibility([Term(A, =V), Term(B, NOT C)]) - ## - ## Meaning: "It's incompatible to have A=V AND NOT have B satisfying C" - - result = Incompatibility( - terms: @[ - Term( - package: dependent, - constraint: Constraint( - versionRange: VersionConstraint( - operator: OpExact, - version: dependentVersion - ), - variantReq: newVariantProfile(), - isNegative: false - ) - ), - Term( - package: dependency, - constraint: Constraint( - versionRange: dependencyConstraint.versionRange, - variantReq: dependencyConstraint.variantReq, - isNegative: true # Negated! - ) - ) - ], - cause: Dependency, - externalContext: dependent & " " & $dependentVersion & " depends on " & dependency, - fromPackage: some(dependent), - fromVersion: some(dependentVersion) - ) - -proc createRootIncompatibility*(package: PackageId, constraint: Constraint): Incompatibility = - ## Create an incompatibility from a root requirement - ## "User requires package P with constraint C" becomes: - ## Incompatibility([Term(P, NOT C)]) - ## - ## Meaning: "It's incompatible to NOT have P satisfying C" - - result = Incompatibility( - terms: @[ - Term( - package: package, - constraint: Constraint( - versionRange: constraint.versionRange, - variantReq: constraint.variantReq, - isNegative: true # Negated! - ) - ) - ], - cause: Root, - externalContext: "User requires " & package & " " & $constraint, - fromPackage: some(package), - fromVersion: none(SemanticVersion) - ) - -# --- Solver State Operations --- - -proc newSolverState*(): SolverState = - ## Create a new solver state - result = SolverState( - assignments: initTable[PackageId, Assignment](), - incompatibilities: @[], - decisionLevel: 0, - processed: initHashSet[PackageId]() - ) - -proc addAssignment*(state: var SolverState, assignment: Assignment) = - ## Add an assignment to the solver state - state.assignments[assignment.package] = assignment - -proc hasAssignment*(state: SolverState, package: PackageId): bool = - ## Check if a package has been assigned - result = state.assignments.hasKey(package) - -proc getAssignment*(state: SolverState, package: PackageId): Option[Assignment] = - ## Get the assignment for a package - if state.assignments.hasKey(package): - return some(state.assignments[package]) - else: - return none(Assignment) - -proc addIncompatibility*(state: var SolverState, incomp: Incompatibility) = - ## Add an incompatibility to the solver state - state.incompatibilities.add(incomp) - -proc incrementDecisionLevel*(state: var SolverState) = - ## Increment the decision level (when making a choice) - state.decisionLevel += 1 - -proc markProcessed*(state: var SolverState, package: PackageId) = - ## Mark a package as processed - state.processed.incl(package) - -proc isProcessed*(state: SolverState, package: PackageId): bool = - ## Check if a package has been processed - result = package in state.processed diff --git a/src/nip/resolver/source_adapter.nim b/src/nip/resolver/source_adapter.nim deleted file mode 100644 index 759e277..0000000 --- a/src/nip/resolver/source_adapter.nim +++ /dev/null @@ -1,232 +0,0 @@ -## Source Adapter Interface -## -## This module defines the abstraction for package sources in NIP's dependency -## resolution system. Different package ecosystems (Nix, AUR, Gentoo, etc.) are -## unified behind this interface. -## -## Philosophy: -## - Source adapters abstract away ecosystem differences -## - Frozen sources provide pre-built binaries (Nix, Arch) -## - Flexible sources build on demand (Gentoo, NPK) -## - Strategy pattern enables intelligent source selection -## -## The adapter system enables NIP to access 100,000+ packages from all ecosystems -## while maintaining a unified interface for the dependency resolver. - -import std/[options, tables, algorithm] -import ./variant_types - -# Result type for operations that can fail -type - Result*[T, E] = object - case isOk*: bool - of true: - value*: T - of false: - error*: E - -proc ok*[T, E](value: T): Result[T, E] = - Result[T, E](isOk: true, value: value) - -proc err*[T, E](error: E): Result[T, E] = - Result[T, E](isOk: false, error: error) - -type - # Source classification determines adapter behavior - SourceClass* = enum - Frozen, ## Pre-built binaries only (Nix, Arch) - Flexible, ## Build on demand (Gentoo, NPK) - FullyFlexible ## Source-only, always build - - # Result of package lookup - PackageAvailability* = enum - Available, ## Package exists and can be provided - Unavailable, ## Package doesn't exist in this source - WrongVariant ## Package exists but variant doesn't match - - # CAS identifier for built packages - CasId* = distinct string - - # Build error information - BuildError* = object - message*: string - exitCode*: int - buildLog*: string - - # Base source adapter interface - SourceAdapter* = ref object of RootObj - name*: string ## Source name (e.g., "nix", "aur", "gentoo") - class*: SourceClass ## Source classification - priority*: int ## Selection priority (higher = preferred) - - # Package metadata from source - PackageMetadata* = object - name*: string - version*: string - availableVariants*: seq[VariantProfile] - dependencies*: seq[VariantDemand] - sourceHash*: string - buildTime*: int ## Estimated build time in seconds (0 for frozen) - -# String conversion for CasId -proc `$`*(id: CasId): string = - string(id) - -proc `==`*(a, b: CasId): bool = - string(a) == string(b) - -# Base methods for source adapters -method canSatisfy*(adapter: SourceAdapter, demand: VariantDemand): PackageAvailability {.base.} = - ## Check if this source can satisfy a variant demand - ## Returns Available, Unavailable, or WrongVariant - ## - ## This is the first step in source selection - quickly determine - ## if this source has the package with the right variant. - - raise newException(CatchableError, "canSatisfy not implemented for " & adapter.name) - -method getVariant*(adapter: SourceAdapter, demand: VariantDemand): Option[PackageMetadata] {.base.} = - ## Get package metadata for a specific variant demand - ## Returns Some(metadata) if available, None if not - ## - ## For frozen sources: returns metadata for exact variant match - ## For flexible sources: returns metadata showing build is possible - - raise newException(CatchableError, "getVariant not implemented for " & adapter.name) - -method synthesize*(adapter: SourceAdapter, demand: VariantDemand): Result[CasId, BuildError] {.base.} = - ## Build a package with the requested variant profile - ## Returns CasId on success, BuildError on failure - ## - ## Only applicable for Flexible and FullyFlexible sources - ## Frozen sources should raise an error if called - - raise newException(CatchableError, "synthesize not implemented for " & adapter.name) - -# Resolution strategy for source selection -type - ResolutionStrategy* = enum - PreferBinary, ## Prefer frozen sources, fall back to flexible - PreferSource, ## Always build from source (flexible) - Balanced ## Consider recency, trust, and availability - - SourceSelection* = object - adapter*: SourceAdapter - reason*: string - estimatedTime*: int - -# Source selection function -proc selectSource*( - adapters: seq[SourceAdapter], - demand: VariantDemand, - strategy: ResolutionStrategy -): Option[SourceSelection] = - ## Select the best source adapter for a given demand - ## Returns Some(selection) if a source can satisfy, None otherwise - ## - ## Strategy determines selection logic: - ## - PreferBinary: Choose frozen first, fall back to flexible - ## - PreferSource: Always choose flexible if available - ## - Balanced: Consider multiple factors (recency, trust, build time) - - var candidates: seq[tuple[adapter: SourceAdapter, availability: PackageAvailability]] = @[] - - # Check all adapters for availability - for adapter in adapters: - let availability = adapter.canSatisfy(demand) - if availability == Available: - candidates.add((adapter, availability)) - - if candidates.len == 0: - return none(SourceSelection) - - # Apply strategy to select best candidate - case strategy: - of PreferBinary: - # Prefer frozen sources (pre-built binaries) - for (adapter, _) in candidates: - if adapter.class == Frozen: - return some(SourceSelection( - adapter: adapter, - reason: "Pre-built binary available", - estimatedTime: 0 - )) - - # Fall back to flexible sources - for (adapter, _) in candidates: - if adapter.class in [Flexible, FullyFlexible]: - let metadata = adapter.getVariant(demand) - if metadata.isSome: - return some(SourceSelection( - adapter: adapter, - reason: "Build from source (no binary available)", - estimatedTime: metadata.get.buildTime - )) - - of PreferSource: - # Always prefer building from source - for (adapter, _) in candidates: - if adapter.class in [Flexible, FullyFlexible]: - let metadata = adapter.getVariant(demand) - if metadata.isSome: - return some(SourceSelection( - adapter: adapter, - reason: "Build from source (user preference)", - estimatedTime: metadata.get.buildTime - )) - - # Fall back to frozen if no flexible source available - for (adapter, _) in candidates: - if adapter.class == Frozen: - return some(SourceSelection( - adapter: adapter, - reason: "Pre-built binary (no source available)", - estimatedTime: 0 - )) - - of Balanced: - # Consider multiple factors: priority, build time, recency - # Sort by priority (higher first) - var sortedCandidates = candidates - sortedCandidates.sort(proc(a, b: auto): int = - b[0].priority - a[0].priority - ) - - # Return highest priority candidate - if sortedCandidates.len > 0: - let adapter = sortedCandidates[0][0] - let metadata = adapter.getVariant(demand) - let estimatedTime = if metadata.isSome: metadata.get.buildTime else: 0 - - return some(SourceSelection( - adapter: adapter, - reason: "Best balance of priority and availability", - estimatedTime: estimatedTime - )) - - return none(SourceSelection) - -# Helper to create CasId from string -proc newCasId*(id: string): CasId = - CasId(id) - -# String representation for debugging -proc `$`*(selection: SourceSelection): string = - result = "SourceSelection(" - result.add("adapter=" & selection.adapter.name) - result.add(", reason=\"" & selection.reason & "\"") - result.add(", estimatedTime=" & $selection.estimatedTime & "s") - result.add(")") - -proc `$`*(availability: PackageAvailability): string = - case availability: - of Available: "Available" - of Unavailable: "Unavailable" - of WrongVariant: "WrongVariant" - -proc `$`*(class: SourceClass): string = - case class: - of Frozen: "Frozen" - of Flexible: "Flexible" - of FullyFlexible: "FullyFlexible" - diff --git a/src/nip/resolver/variant_hash.nim b/src/nip/resolver/variant_hash.nim deleted file mode 100644 index 621239c..0000000 --- a/src/nip/resolver/variant_hash.nim +++ /dev/null @@ -1,150 +0,0 @@ -## Variant Hash Calculation -## -## This module implements deterministic hash calculation for variant profiles -## using xxh4-128 (or xxh3-128 until xxh4 is available). -## -## Philosophy: -## - Same variant profile ALWAYS produces same hash -## - Hash is deterministic across all platforms and runs -## - 128-bit output is collision-safe for any realistic number of variants -## - No cryptographic properties needed (no adversary in variant space) -## -## The hash enables: -## - Unique identification of build configurations -## - Content-addressable storage of builds -## - Reproducible build verification -## - Efficient deduplication - -import std/[strutils, tables, sequtils, algorithm, sets] -import ./variant_types -# import ../xxhash # For xxh3-128 (placeholder for xxh4) - imported via variant_types - -proc calculateVariantHash*(profile: var VariantProfile): string = - ## Calculate deterministic xxh4-128 hash of variant profile - ## Uses lazy evaluation with caching from variant_types - ## - ## The hash is calculated from the canonical string representation, - ## which is sorted alphabetically for determinism. - ## - ## Format: xxh4-<128-bit-hex> or xxh3-<128-bit-hex> - ## - ## Example: - ## Input: init:dinit|graphics:wayland,vulkan|optimization:lto - ## Output: xxh3-8f3c2d1e9a4b5c6d7e8f9a0b1c2d3e4f - - # Use lazy cached calculation from variant_types - profile.calculateHash() - result = profile.hash - -proc updateHash*(profile: var VariantProfile) = - ## Update the hash field of a variant profile - ## Call this after modifying the profile - ## Invalidates cache and recalculates - - profile.hash = "" # Invalidate cache - profile.calculateHash() - -proc verifyHash*(profile: var VariantProfile): bool = - ## Verify that the stored hash matches the calculated hash - ## Returns true if hash is correct, false otherwise - - let storedHash = profile.hash - profile.hash = "" # Invalidate to force recalculation - let calculatedHash = calculateVariantHash(profile) - result = storedHash == calculatedHash - -# Helper to create profile with hash -proc createVariantProfile*(domains: Table[string, VariantDomain]): VariantProfile = - ## Create a variant profile with domains and calculate hash - - result.domains = domains - result.updateHash() - -proc inferDomain(flag: string): string = - ## Infer domain from flag name (simple heuristic) - ## This is a convenience for user-friendly syntax - - case flag: - of "wayland", "x11", "vulkan", "opengl": - "graphics" - of "hardened", "selinux", "apparmor": - "security" - of "ipv6", "ipv4", "bluetooth", "wifi": - "network" - of "lto", "pgo", "native": - "optimization" - of "systemd", "dinit", "openrc", "runit": - "init" - else: - "features" # Default domain - -# Parse variant string to profile -proc parseVariantString*(variantStr: string): VariantProfile = - ## Parse variant string to profile - ## Format: +flag1 +flag2 -flag3 domain:value - ## - ## Examples: - ## "+wayland +vulkan -X" → graphics:wayland,vulkan - ## "init:dinit" → init:dinit (exclusive) - ## "+hardened +ipv6" → security:hardened network:ipv6 - - result = newVariantProfile() - - if variantStr.strip() == "": - return result - - let parts = variantStr.split() - - for part in parts: - if part.startsWith("+"): - # Positive flag: +wayland → graphics:wayland - let flag = part[1..^1] - # Infer domain from flag name (simple heuristic) - let domain = inferDomain(flag) - result.addFlag(domain, flag) - - elif part.startsWith("-"): - # Negative flag: -X → exclude X - # For now, we don't store negative flags - # They're used during resolution to filter - discard - - elif ":" in part: - # Explicit domain:value → init:dinit - let colonPos = part.find(':') - let domain = part[0../// (Installation Root) -## - /Programs//Current (Symlink to active version) -## - /System/Index/bin/ (Symlinks to executables) -## - /System/Index/lib/ (Symlinks to libraries) -## -## Responsibilities: -## 1. File reconstruction from CAS -## 2. Symlink management -## 3. User/Group creation -## 4. Service file generation - -import std/[os, posix, strutils, strformat, options, osproc, logging] -import nip/manifest_parser -import nip/cas -import nip/types # For Multihash if needed - -type - SystemIntegrator* = ref object - casRoot*: string - programsRoot*: string - systemIndexRoot*: string - dryRun*: bool - -proc newSystemIntegrator*(casRoot, programsRoot, systemIndexRoot: string, dryRun: bool = false): SystemIntegrator = - result = SystemIntegrator( - casRoot: casRoot, - programsRoot: programsRoot, - systemIndexRoot: systemIndexRoot, - dryRun: dryRun - ) - -proc log(si: SystemIntegrator, msg: string) = - if si.dryRun: - echo "[DRY-RUN] " & msg - else: - info(msg) - -# ============================================================================ -# File Reconstruction -# ============================================================================ - -proc reconstructFiles(si: SystemIntegrator, manifest: PackageManifest, installDir: string) = - ## Reconstruct files from CAS chunks into the installation directory - si.log(fmt"Reconstructing files for {manifest.name} v{manifest.version} in {installDir}") - - if not si.dryRun: - createDir(installDir) - - for file in manifest.files: - let destPath = installDir / file.path - let destDir = destPath.parentDir - - if not si.dryRun: - createDir(destDir) - - # In a real implementation, we might have multiple chunks per file. - # For now, we assume 1-to-1 mapping or that CAS handles retrieval transparently. - # manifest_parser uses string for hash, cas uses Multihash. - # We assume 'file.hash' is the CAS object hash. - - try: - if not si.dryRun: - # Retrieve content from CAS - # Note: cas.retrieveObject takes Multihash - let content = retrieveObject(Multihash(file.hash), si.casRoot) - writeFile(destPath, content) - - # Set permissions - # Parse octal string e.g. "755" - var perms: set[FilePermission] = {} - if file.permissions.len == 3: - let user = file.permissions[0].ord - '0'.ord - let group = file.permissions[1].ord - '0'.ord - let other = file.permissions[2].ord - '0'.ord - - if (user and 4) != 0: perms.incl(fpUserRead) - if (user and 2) != 0: perms.incl(fpUserWrite) - if (user and 1) != 0: perms.incl(fpUserExec) - - if (group and 4) != 0: perms.incl(fpGroupRead) - if (group and 2) != 0: perms.incl(fpGroupWrite) - if (group and 1) != 0: perms.incl(fpGroupExec) - - if (other and 4) != 0: perms.incl(fpOthersRead) - if (other and 2) != 0: perms.incl(fpOthersWrite) - if (other and 1) != 0: perms.incl(fpOthersExec) - - setFilePermissions(destPath, perms) - - # Add CAS reference - # refId = package:version - let refId = fmt"{manifest.name}:{manifest.version}" - addReference(si.casRoot, Multihash(file.hash), "npk", refId) - - except Exception as e: - error(fmt"Failed to reconstruct file {file.path}: {e.msg}") - raise - -# ============================================================================ -# Symlink Management -# ============================================================================ - -proc createSymlinks(si: SystemIntegrator, manifest: PackageManifest, installDir: string) = - ## Create system links in /System/Index - si.log(fmt"Creating symlinks for {manifest.name}") - - # 1. Update 'Current' link - let packageRoot = si.programsRoot / manifest.name - let currentLink = packageRoot / "Current" - - if not si.dryRun: - createDir(packageRoot) - # Atomic symlink update would be better, but for MVP: - if symlinkExists(currentLink) or fileExists(currentLink): - removeFile(currentLink) - createSymlink(installDir, currentLink) - - # 2. Link binaries to /System/Index/bin - let binDir = installDir / "bin" - let systemBin = si.systemIndexRoot / "bin" - - if dirExists(binDir): - if not si.dryRun: createDir(systemBin) - for kind, path in walkDir(binDir): - if kind == pcFile or kind == pcLinkToFile: - let filename = path.extractFilename - let target = systemBin / filename - si.log(fmt"Linking {filename} -> {target}") - - if not si.dryRun: - if symlinkExists(target) or fileExists(target): - # Conflict resolution strategy: Overwrite? Warn? - # For now, overwrite - removeFile(target) - # Link to the 'Current' path, not the specific version path, - # so upgrades don't break links if 'Current' is updated. - # Target: /Programs//Current/bin/ - let persistentPath = currentLink / "bin" / filename - createSymlink(persistentPath, target) - - # 3. Link libraries to /System/Index/lib - let libDir = installDir / "lib" - let systemLib = si.systemIndexRoot / "lib" - - if dirExists(libDir): - if not si.dryRun: createDir(systemLib) - for kind, path in walkDir(libDir): - if kind == pcFile or kind == pcLinkToFile: - let filename = path.extractFilename - # Only link .so files or similar? Or everything? - # GoboLinux links everything usually. - let target = systemLib / filename - si.log(fmt"Linking {filename} -> {target}") - - if not si.dryRun: - if symlinkExists(target) or fileExists(target): - removeFile(target) - let persistentPath = currentLink / "lib" / filename - createSymlink(persistentPath, target) - - # TODO: Handle share, include, etc. - -# ============================================================================ -# User/Group Management -# ============================================================================ - -proc manageUsersGroups(si: SystemIntegrator, manifest: PackageManifest) = - ## Create users and groups defined in the manifest - - # Groups first - for group in manifest.groups: - si.log(fmt"Ensuring group exists: {group.name}") - if not si.dryRun: - # Check if group exists - let checkCmd = fmt"getent group {group.name}" - if execCmd(checkCmd) != 0: - # Create group - var cmd = fmt"groupadd {group.name}" - if group.gid.isSome: - cmd.add(fmt" -g {group.gid.get()}") - - if execCmd(cmd) != 0: - error(fmt"Failed to create group {group.name}") - - # Users - for user in manifest.users: - si.log(fmt"Ensuring user exists: {user.name}") - if not si.dryRun: - # Check if user exists - let checkCmd = fmt"getent passwd {user.name}" - if execCmd(checkCmd) != 0: - # Create user - var cmd = fmt"useradd -m -s {user.shell} -d {user.home}" - if user.uid.isSome: - cmd.add(fmt" -u {user.uid.get()}") - if user.group != "": - cmd.add(fmt" -g {user.group}") - cmd.add(fmt" {user.name}") - - if execCmd(cmd) != 0: - error(fmt"Failed to create user {user.name}") - -# ============================================================================ -# Service Management -# ============================================================================ - -proc manageServices(si: SystemIntegrator, manifest: PackageManifest) = - ## Generate and install system service files - let systemdDir = si.systemIndexRoot / "lib/systemd/system" - - if manifest.services.len > 0: - if not si.dryRun: createDir(systemdDir) - - for service in manifest.services: - let serviceFile = systemdDir / (service.name & ".service") - si.log(fmt"Installing service: {service.name}") - - if not si.dryRun: - writeFile(serviceFile, service.content) - - if service.enabled: - # Enable service (symlink to multi-user.target.wants usually) - # For MVP, we just run systemctl enable - discard execCmd(fmt"systemctl enable {service.name}") - -# ============================================================================ -# Main Installation Procedure -# ============================================================================ - -proc installPackage*(si: SystemIntegrator, manifest: PackageManifest) = - ## Main entry point for installing a package - info(fmt"Installing {manifest.name} v{manifest.version}") - - # 1. Determine installation path - # /Programs// - # We might want to include hash in path to allow multiple builds of same version? - # Task says: /Programs/App/Version/Hash - let installDir = si.programsRoot / manifest.name / $manifest.version / manifest.artifactHash - - if dirExists(installDir): - warn(fmt"Package version already installed at {installDir}") - # Proceed anyway to repair/update? Or return? - # For now, proceed (idempotent) - - # 2. Reconstruct files - si.reconstructFiles(manifest, installDir) - - # 3. Create Users/Groups - si.manageUsersGroups(manifest) - - # 4. Create Symlinks (activates the package) - si.createSymlinks(manifest, installDir) - - # 5. Manage Services - si.manageServices(manifest) - -# ============================================================================ -# Removal Procedure -# ============================================================================ - -proc removePackage*(si: SystemIntegrator, manifest: PackageManifest) = - ## Remove an installed package - info(fmt"Removing {manifest.name} v{manifest.version}") - - let installDir = si.programsRoot / manifest.name / $manifest.version / manifest.artifactHash - let currentLink = si.programsRoot / manifest.name / "Current" - - # 1. Stop and Disable Services - if manifest.services.len > 0: - for service in manifest.services: - si.log(fmt"Stopping/Disabling service: {service.name}") - if not si.dryRun: - discard execCmd(fmt"systemctl stop {service.name}") - discard execCmd(fmt"systemctl disable {service.name}") - - let serviceFile = si.systemIndexRoot / "lib/systemd/system" / (service.name & ".service") - if fileExists(serviceFile): - removeFile(serviceFile) - - # 2. Remove Symlinks from /System/Index - # We need to know what files were linked. - # Strategy: Check if 'Current' points to the version we are removing. - # If so, we should remove the links. - # If 'Current' points to another version, we should NOT remove links (except maybe cleaning up orphans?) - - var isCurrent = false - if symlinkExists(currentLink): - let target = expandSymlink(currentLink) - if target == installDir: - isCurrent = true - - if isCurrent: - si.log("Removing system symlinks") - # Binaries - if dirExists(installDir / "bin"): - for kind, path in walkDir(installDir / "bin"): - let filename = path.extractFilename - let target = si.systemIndexRoot / "bin" / filename - if not si.dryRun and (symlinkExists(target) or fileExists(target)): - removeFile(target) - - # Libraries - if dirExists(installDir / "lib"): - for kind, path in walkDir(installDir / "lib"): - let filename = path.extractFilename - let target = si.systemIndexRoot / "lib" / filename - if not si.dryRun and (symlinkExists(target) or fileExists(target)): - removeFile(target) - - # Remove 'Current' link - if not si.dryRun: - removeFile(currentLink) - - # 3. Remove Installation Directory - if dirExists(installDir): - si.log(fmt"Removing installation directory: {installDir}") - if not si.dryRun: - removeDir(installDir) - - # Remove version dir if empty - let versionDir = installDir.parentDir - if dirExists(versionDir): - var versionEmpty = true - for _ in walkDir(versionDir): - versionEmpty = false - break - if versionEmpty: - removeDir(versionDir) - - # Remove package dir if empty (no other versions) - let packageDir = si.programsRoot / manifest.name - if dirExists(packageDir): - var isEmpty = true - for _ in walkDir(packageDir): - isEmpty = false - break - if isEmpty: - removeDir(packageDir) - - # 4. Remove CAS References - si.log("Removing CAS references") - if not si.dryRun: - let refId = fmt"{manifest.name}:{manifest.version}" - for file in manifest.files: - removeReference(si.casRoot, Multihash(file.hash), "npk", refId) - - info(fmt"Removal of {manifest.name} complete") - diff --git a/src/nip/types.nim b/src/nip/types.nim deleted file mode 100644 index 1c082d4..0000000 --- a/src/nip/types.nim +++ /dev/null @@ -1,254 +0,0 @@ -import std/[times, json, hashes] - - -# ############################################################################# -# Core Type Primitives -# ############################################################################# - -type - Blake2bHash* = distinct string # Enforce type safety for BLAKE2b-512 hashes - Multihash* = distinct string # For future-proofing hash algorithms (will support BLAKE3 later) - SemVer* = distinct string # Semantic Version string - -proc `==`*(a, b: SemVer): bool = - string(a) == string(b) - -proc `==`*(a, b: Multihash): bool = - string(a) == string(b) - -proc `==`*(a, b: Blake2bHash): bool = - string(a) == string(b) - -# ############################################################################# -# .npk Manifest Types -# ############################################################################# - -type - NpkSource* = object - originPackage*: string - originVersion*: string - - NpkDependency* = object - name*: string - hash*: Blake2bHash - - NpkBuild* = object - timestamp*: Time - buildSystem*: string - compiler*: string - envHash*: Blake2bHash - - NpkFile* = object - path*: string - hash*: Blake2bHash - permissions*: string - - NpkArtifact* = object - name*: string - hash*: Blake2bHash - - NpkService* = object - serviceType*: string # e.g., "systemd" - name*: string - hash*: Blake2bHash - - NpkSignature* = object - keyType*: string # e.g., "ed25519" - keyId*: string - value*: string - - NpkManifest* = object - name*: string - version*: SemVer - description*: string - channels*: seq[string] - source*: NpkSource - dependencies*: seq[NpkDependency] - build*: NpkBuild - files*: seq[NpkFile] - artifacts*: seq[NpkArtifact] - services*: seq[NpkService] - signatures*: seq[NpkSignature] - -# ############################################################################# -# nip.lock (System Generation) Types -# ############################################################################# - -type - LockfileGeneration* = object - id*: Blake2bHash - created*: Time - previous*: Blake2bHash - - LockfilePackage* = object - name*: string - hash*: Blake2bHash - - NipLock* = object - lockfileVersion*: string - generation*: LockfileGeneration - packages*: seq[LockfilePackage] - -# ############################################################################# -# Package Management Types -# ############################################################################# - -type - PackageStream* = enum - Stable, Testing, Dev, LTS, Custom - - SourceMethod* = enum - Git, Http, Local, Grafted - - BuildSystemType* = enum - CMake, Meson, Autotools, Cargo, Nim, Custom - - LibcType* = enum - Musl, Glibc, None - - AllocatorType* = enum - Jemalloc, Tcmalloc, Default - - PackageId* = object - name*: string - version*: string - stream*: PackageStream - - Source* = object - url*: string - hash*: string - hashAlgorithm*: string - sourceMethod*: SourceMethod - timestamp*: DateTime - - PackageMetadata* = object - description*: string - license*: string - maintainer*: string - tags*: seq[string] - runtime*: RuntimeProfile - - RuntimeProfile* = object - libc*: LibcType - allocator*: AllocatorType - systemdAware*: bool - reproducible*: bool - tags*: seq[string] - - AculCompliance* = object - required*: bool - membership*: string - attribution*: string - buildLog*: string - - Fragment* = object - id*: PackageId - source*: Source - dependencies*: seq[PackageId] - buildSystem*: BuildSystemType - metadata*: PackageMetadata - acul*: AculCompliance - -# ############################################################################# -# Error Types -# ############################################################################# - -type - NimPakError* = object of CatchableError - code*: ErrorCode - context*: string - suggestions*: seq[string] - - ErrorCode* = enum - # Package errors - PackageNotFound, DependencyConflict, ChecksumMismatch, - InvalidMetadata, PackageCorrupted, VersionMismatch, - # Permission errors - PermissionDenied, ElevationRequired, ReadOnlyViolation, - # Network errors - NetworkError, DownloadFailed, RepositoryUnavailable, TimeoutError, - # Build errors - BuildFailed, CompilationError, MissingDependency, - # ACUL/Policy errors - AculViolation, PolicyViolation, SignatureInvalid, TrustViolation, - # Storage errors - CellNotFound, ObjectNotFound, FileReadError, FileWriteError, - StorageFull, QuotaExceeded, - # Transaction errors - TransactionFailed, RollbackFailed, LockConflict, - # GC errors - GarbageCollectionFailed, ReferenceIntegrityError, - # Format errors - InvalidFormat, UnsupportedVersion, MigrationRequired, - # Generic errors - InvalidOperation, ConfigurationError, UnknownError - -# ############################################################################# -# Transaction Types -# ############################################################################# - -type - OperationKind* = enum - CreateDir, CreateFile, CreateSymlink, RemoveFile, RemoveDir - - Operation* = object - kind*: OperationKind - target*: string - data*: JsonNode - - RollbackInfo* = object - operation*: Operation - originalState*: JsonNode - - Transaction* = object - id*: string - operations*: seq[Operation] - rollbackData*: seq[RollbackInfo] - -# ############################################################################# -# Filesystem Types -# ############################################################################# - -type - FilesystemManager* = object - programsRoot*: string - indexRoot*: string - - InstallLocation* = object - programDir*: string - indexLinks*: seq[SymlinkPair] - - SymlinkPair* = object - source*: string - target*: string - -# ############################################################################# -# Repository Types (NexusForge) -# ############################################################################# - -type - RepoType* = enum - Native, Git, Graft - - GraftBackend* = enum - Nix, Portage, Pkgsrc, Pacman, Apt, Dnf, Mock - - RepoConfig* = object - name*: string - kind*: RepoType - url*: string - priority*: int - # Native specific - key*: string - # Git specific - branch*: string - token*: string - # Graft specific - backend*: GraftBackend - -# Equality operators for PackageId -proc `==`*(a, b: PackageId): bool = - a.name == b.name and a.version == b.version and a.stream == b.stream - -proc hash*(pkg: PackageId): Hash = - hash((pkg.name, pkg.version, pkg.stream)) \ No newline at end of file diff --git a/src/nip/unified_storage.nim b/src/nip/unified_storage.nim deleted file mode 100644 index 2caae2c..0000000 --- a/src/nip/unified_storage.nim +++ /dev/null @@ -1,206 +0,0 @@ -## Unified Storage Architecture for NexusOS -## -## This module implements the unified storage system that supports all three -## package formats (.npk, .nip, .nexter) with shared Content-Addressable Storage (CAS). -## -## Storage Layout: -## --system level: -## /var/lib/nexus -## OR -## --user level: -## ~/.local/share/nexus/ -## ├── cas/ # Shared CAS (chmod 555) -## │ ├── chunks/ # Compressed chunks -## │ ├── refs/ # Reference tracking -## │ │ ├── npks/ # .npk references -## │ │ ├── nips/ # .nip references -## │ │ └── nexters/ # .nexter references -## │ └── audit.log # Write operation log -## ├── npks/ # System packages -## ├── nips/ # User applications -## └── nexters/ # Containers - -import std/[os, times, strutils, tables] - -type - StorageRoot* = object - ## Root directory for unified storage - basePath*: string - casPath*: string - npksPath*: string - nipsPath*: string - nextersPath*: string - auditLogPath*: string - - ChunkType* = enum - ## Type of chunk stored in CAS - Binary, Library, Runtime, Config, Data, Base, Tools - - ChunkMetadata* = object - ## Metadata for a CAS chunk - hash*: string # xxh3-128 hash - size*: int64 - refCount*: int # Total references across all formats - compression*: string # "zstd" - created*: DateTime - chunkType*: ChunkType - - FormatType* = enum - ## Package format type - NPK, NIP, NEXTER - - CASStore* = object - ## Content-Addressable Storage manager - rootPath*: string # ~/.local/share/nexus/cas - chunksPath*: string # cas/chunks/ - refsPath*: string # cas/refs/ - auditLog*: string # cas/audit.log - index*: Table[string, ChunkMetadata] - - CASError* = object of CatchableError - ## CAS-specific errors - code*: CASErrorCode - context*: string - - CASErrorCode* = enum - CASChunkNotFound, - CASChunkHashMismatch, - CASStorageFull, - CASPermissionDenied, - CASInvalidHash - -const - DefaultStorageRoot* = "~/.local/share/nexus" - CASPermissions* = {fpUserRead, fpUserExec, fpGroupRead, fpGroupExec, - fpOthersRead, fpOthersExec} # 555 - WritePermissions* = {fpUserRead, fpUserWrite, fpUserExec, fpGroupRead, - fpGroupExec, fpOthersRead, fpOthersExec} # 755 - -proc expandPath(path: string): string = - ## Expand ~ to home directory - if path.startsWith("~"): - result = getHomeDir() / path[2..^1] - else: - result = path - -proc initStorageRoot*(basePath: string = DefaultStorageRoot): StorageRoot = - ## Initialize the unified storage root structure - let expandedBase = expandPath(basePath) - - result = StorageRoot( - basePath: expandedBase, - casPath: expandedBase / "Cas", - npksPath: expandedBase / "npks", - nipsPath: expandedBase / "nips", - nextersPath: expandedBase / "nexters", - auditLogPath: expandedBase / "Cas" / "audit.log" - ) - -proc createStorageStructure*(root: StorageRoot): bool = - ## Create the unified storage directory structure - ## Returns true if successful, false otherwise - try: - # Create base directory - createDir(root.basePath) - - # Create CAS structure - createDir(root.casPath) - createDir(root.casPath / "chunks") - createDir(root.casPath / "refs") - createDir(root.casPath / "refs" / "npks") - createDir(root.casPath / "refs" / "nips") - createDir(root.casPath / "refs" / "nexters") - - # Create format-specific directories - createDir(root.npksPath) - createDir(root.nipsPath) - createDir(root.nextersPath) - - # Create audit log file - if not fileExists(root.auditLogPath): - writeFile(root.auditLogPath, "# NexusOS Unified Storage Audit Log\n") - writeFile(root.auditLogPath, "# Created: " & $now() & "\n\n") - - # Set CAS to read-only (555) - setFilePermissions(root.casPath, CASPermissions) - - result = true - except OSError as e: - echo "Error creating storage structure: ", e.msg - result = false - except IOError as e: - echo "Error creating audit log: ", e.msg - result = false - -proc verifyStorageStructure*(root: StorageRoot): bool = - ## Verify that the storage structure exists and is valid - result = dirExists(root.basePath) and - dirExists(root.casPath) and - dirExists(root.casPath / "chunks") and - dirExists(root.casPath / "refs") and - dirExists(root.casPath / "refs" / "npks") and - dirExists(root.casPath / "refs" / "nips") and - dirExists(root.casPath / "refs" / "nexters") and - dirExists(root.npksPath) and - dirExists(root.nipsPath) and - dirExists(root.nextersPath) and - fileExists(root.auditLogPath) - -proc initCASStore*(rootPath: string): CASStore = - ## Initialize a CAS store instance - let expandedRoot = expandPath(rootPath) - - result = CASStore( - rootPath: expandedRoot, - chunksPath: expandedRoot / "chunks", - refsPath: expandedRoot / "refs", - auditLog: expandedRoot / "audit.log", - index: initTable[string, ChunkMetadata]() - ) - -proc logAuditEntry*(store: CASStore, operation: string, details: string) = - ## Log an operation to the audit log - let timestamp = now() - let entry = "[$#] $#: $#\n" % [$timestamp, operation, details] - - try: - # Temporarily enable write access - setFilePermissions(store.rootPath, WritePermissions) - - # Append to audit log - let f = open(store.auditLog, fmAppend) - f.write(entry) - f.close() - - # Restore read-only permissions - setFilePermissions(store.rootPath, CASPermissions) - except: - echo "Warning: Failed to write audit log entry" - -when isMainModule: - echo "Testing Unified Storage Structure..." - - # Test storage initialization - let root = initStorageRoot() - echo "Storage root: ", root.basePath - echo "CAS path: ", root.casPath - - # Create structure - if createStorageStructure(root): - echo "✓ Storage structure created successfully" - else: - echo "✗ Failed to create storage structure" - - # Verify structure - if verifyStorageStructure(root): - echo "✓ Storage structure verified" - else: - echo "✗ Storage structure verification failed" - - # Test CAS store - let store = initCASStore(root.casPath) - echo "CAS store initialized: ", store.rootPath - - # Test audit logging - store.logAuditEntry("TEST", "Testing audit log functionality") - echo "✓ Audit log entry written" diff --git a/src/nip/utils/hashing.nim b/src/nip/utils/hashing.nim deleted file mode 100644 index 05836b1..0000000 --- a/src/nip/utils/hashing.nim +++ /dev/null @@ -1,121 +0,0 @@ -## High-Performance Hashing Utilities -## -## This module provides fast, non-cryptographic hashing for cache keys, -## content addressing, and integrity verification. -## -## **Hash Algorithm:** xxh3_128 -## - Speed: 40-60 GiB/s single-threaded -## - Output: 128-bit (collision-safe for cosmic scale: 2^-100) -## - Portability: Excellent on all architectures -## -## **Use Cases:** -## - Cache key calculation (non-cryptographic) -## - Content-addressable storage (CAS) -## - Merkle tree node hashing -## - Build hash calculation -## -## **NOT for:** -## - Cryptographic signatures (use BLAKE3) -## - Security-critical operations (use BLAKE3) -## - Protocol authentication (use BLAKE3) - -import ../xxhash - -# Re-export xxhash functions for convenience -export calculateXXH3 - -proc xxh3_128*(data: string): string = - ## Calculate xxh3_128 hash of binary data. - ## - ## **Parameters:** - ## - data: Binary string to hash - ## - ## **Returns:** 128-bit hash as hex string with "xxh3-" prefix - ## - ## **Performance:** ~40-60 GiB/s on modern CPUs - ## - ## **Example:** - ## ```nim - ## let hash = xxh3_128("hello world") - ## echo hash # "xxh3-a1b2c3d4e5f6..." - ## ``` - - return $calculateXXH3(data) - -proc xxh3_128*(data: seq[byte]): string = - ## Calculate xxh3_128 hash of byte sequence. - ## - ## **Parameters:** - ## - data: Byte sequence to hash - ## - ## **Returns:** 128-bit hash as hex string with "xxh3-" prefix - - return $calculateXXH3(data) - -# ============================================================================ -# Hash Verification Utilities -# ============================================================================ - -proc verifyHash*(data: string, expectedHash: string): bool = - ## Verify that data matches expected hash. - ## - ## **Parameters:** - ## - data: Binary data to verify - ## - expectedHash: Expected xxh3_128 hash (hex string) - ## - ## **Returns:** true if hash matches, false otherwise - ## - ## **Example:** - ## ```nim - ## let data = "hello world" - ## let hash = xxh3_128(data) - ## assert verifyHash(data, hash) - ## ``` - - let actualHash = xxh3_128(data) - return actualHash == expectedHash - -# ============================================================================ -# Performance Benchmarking -# ============================================================================ - -when isMainModule: - import times - import strformat - - proc benchmarkHashing() = - ## Benchmark xxh3_128 performance - - # Generate test data (1 MB) - let dataSize = 1024 * 1024 - var testData = newString(dataSize) - for i in 0.. [options]") - - target = args[0] - - var i = 1 - while i < args.len: - case args[i]: - of "--no-signatures": - options.checkSignatures = false - of "--verbose", "-v": - options.verbose = true - of "--auto-repair": - options.autoRepair = true - of "--output": - if i + 1 < args.len: - case args[i + 1].toLower(): - of "json": options.outputFormat = OutputJson - of "yaml": options.outputFormat = OutputYaml - of "kdl": options.outputFormat = OutputKdl - else: options.outputFormat = OutputHuman - i += 1 - else: - raise newException(ValueError, fmt"Unknown option: {args[i]}") - i += 1 - - return (target, options) - -proc formatVerificationResults*(results: seq[IntegrityCheckResult], options: VerifyOptions): JsonNode = - ## Format verification results for output - var formattedResults = newJArray() - var summary = %*{ - "total_checks": results.len, - "passed": 0, - "failed": 0, - "total_duration": 0.0 - } - - for result in results: - summary["total_duration"] = summary["total_duration"].getFloat() + result.duration - - if result.success: - summary["passed"] = summary["passed"].getInt() + 1 - else: - summary["failed"] = summary["failed"].getInt() + 1 - - var resultJson = %*{ - "package": result.packageName, - "check_type": $result.checkType, - "success": result.success, - "message": result.message, - "duration": result.duration, - "timestamp": $result.checkTime - } - - if options.verbose: - resultJson["details"] = result.details - - formattedResults.add(resultJson) - - return %*{ - "summary": summary, - "results": formattedResults - } - -proc displayHumanResults*(results: seq[IntegrityCheckResult], options: VerifyOptions) = - ## Display verification results in human-readable format - var passed = 0 - var failed = 0 - var totalDuration = 0.0 - - echo bold("Package Verification Results") - echo "=".repeat(40) - - for result in results: - totalDuration += result.duration - - let symbol = if result.success: success("✅") else: error("❌") - let checkType = case result.checkType: - of CheckFileIntegrity: "Integrity" - of CheckSignatureValidity: "Signature" - of CheckKeyringHealth: "Keyring" - of CheckCRLFreshness: "CRL" - of CheckPackageConsistency: "Consistency" - of CheckSystemGeneration: "Generation" - - echo fmt"{symbol} {checkType}: {result.packageName}" - - if result.success: - inc passed - if options.verbose: - echo fmt" ✓ {result.message}" - echo fmt" ⏱ Duration: {result.duration:.3f}s" - else: - inc failed - echo fmt" ✗ {error(result.message)}" - if options.verbose and result.details != nil: - for key, value in result.details.pairs: - echo fmt" • {key}: {value}" - - if options.verbose: - echo "" - - echo "" - echo bold("Summary:") - echo fmt"Total checks: {results.len}" - echo fmt"Passed: {success($passed)}" - echo fmt"Failed: {if failed > 0: error($failed) else: $failed}" - echo fmt"Total time: {totalDuration:.3f}s" - - if failed > 0: - echo "" - echo warning("⚠️ Some verification checks failed. Run with --verbose for details.") - if not options.autoRepair: - echo info("💡 Use --auto-repair to attempt automatic fixes.") - -proc nipVerifyCommand*(args: seq[string]): CommandResult = - ## Main implementation of nip verify command - try: - let (target, options) = parseVerifyOptions(args) - - if options.verbose: - showInfo(fmt"Starting verification of: {target}") - - # Execute verification using the integrity monitor functions - let results = if target == "--all" or target == "all": - # Verify all packages - let monitor = newIntegrityMonitor(getDefaultIntegrityConfig()) - verifyAllPackages(monitor) - else: - # Verify specific package - let packagePath = fmt"/Programs/{target}/current/{target}.npk" - if fileExists(packagePath): - var singleResult: seq[IntegrityCheckResult] = @[] - singleResult.add(verifyPackageIntegrity(target, packagePath)) - - if options.checkSignatures: - let config = getDefaultKeyringConfig() - var keyringManager = newKeyringManager(config) - keyringManager.loadAllKeyrings() - singleResult.add(verifyPackageSignature(target, packagePath, keyringManager)) - - singleResult - else: - @[IntegrityCheckResult( - checkType: CheckFileIntegrity, - packageName: target, - success: false, - message: fmt"Package not found: {target}", - details: %*{"package_path": packagePath}, - checkTime: now(), - duration: 0.0 - )] - - # Handle auto-repair if requested and there are failures - if options.autoRepair: - for result in results: - if not result.success and result.checkType == CheckFileIntegrity: - showInfo(fmt"Attempting auto-repair for {result.packageName}") - # TODO: Implement auto-repair logic - discard - - # Format and display results - case options.outputFormat: - of OutputHuman: - displayHumanResults(results, options) - else: - let formattedData = formatVerificationResults(results, options) - outputData(formattedData) - - # Determine overall success - let failedCount = results.countIt(not it.success) - if failedCount == 0: - return successResult(fmt"All verification checks passed ({results.len} checks)") - else: - return errorResult(fmt"Verification failed: {failedCount} of {results.len} checks failed", 1) - - except Exception as e: - return errorResult(fmt"Verification error: {e.msg}") - -export nipVerifyCommand, VerifyOptions, parseVerifyOptions \ No newline at end of file diff --git a/src/nip/xxh.nim b/src/nip/xxh.nim deleted file mode 100644 index 719a194..0000000 --- a/src/nip/xxh.nim +++ /dev/null @@ -1,132 +0,0 @@ -## xxHash Integration for NexusOS -## -## This module provides xxh3-128 hashing for Content-Addressable Storage (CAS). -## xxh3 is chosen for its exceptional speed (40-60 GiB/s) and 128-bit collision -## resistance, making it ideal for non-cryptographic CAS operations. -## -## Performance: xxh3-128 is 20-80% faster than BLAKE3 for CAS operations -## Collision Safety: 128-bit output provides < 2^-100 collision probability -## -## Note: This is a wrapper around the xxhash Nim library. -## Install with: nimble install xxhash - -import std/[strutils] - -# We'll use a conditional import to handle the case where xxhash isn't installed yet -when defined(useXXHash): - import xxhash - import nint128 # Required for UInt128 toHex -else: - # Fallback implementation using a simple hash for development - # This will be replaced with actual xxhash once the library is installed - import std/hashes as stdhashes - -type - XXH3Hash* = distinct string - ## xxh3-128 hash value (128-bit) - -proc `==`*(a, b: XXH3Hash): bool = - string(a) == string(b) - -proc `$`*(h: XXH3Hash): string = - string(h) - -when defined(useXXHash): - proc calculateXXH3*(data: string): XXH3Hash = - ## Calculate xxh3-128 hash of a string - ## Returns hash in format: "xxh3-" - let hash128 = XXH3_128bits(data) - let hexDigest = hash128.toHex().toLowerAscii() - result = XXH3Hash("xxh3-" & hexDigest) - - proc calculateXXH3*(data: seq[byte]): XXH3Hash = - ## Calculate xxh3-128 hash of a byte sequence - ## Returns hash in format: "xxh3-" - let hash128 = XXH3_128bits(cast[ptr UncheckedArray[byte]](unsafeAddr data[ - 0]), csize_t(data.len)) - let hexDigest = hash128.toHex().toLowerAscii() - result = XXH3Hash("xxh3-" & hexDigest) - - proc calculateFileXXH3*(path: string): XXH3Hash = - ## Calculate xxh3-128 hash of a file - ## Returns hash in format: "xxh3-" - let data = readFile(path) - result = calculateXXH3(data) - -else: - # Fallback implementation for development/testing - # This uses a simple hash and should NOT be used in production - proc calculateXXH3*(data: string): XXH3Hash = - ## FALLBACK: Simple hash for development (NOT production-ready) - ## Install xxhash library for actual xxh3-128 hashing - let simpleHash = stdhashes.hash(data) - let hexDigest = simpleHash.toHex(16).toLowerAscii() - result = XXH3Hash("xxh3-fallback-" & hexDigest) - - proc calculateXXH3*(data: seq[byte]): XXH3Hash = - ## FALLBACK: Simple hash for development (NOT production-ready) - var str = newString(data.len) - for i, b in data: - str[i] = char(b) - result = calculateXXH3(str) - - proc calculateFileXXH3*(path: string): XXH3Hash = - ## FALLBACK: Simple hash for development (NOT production-ready) - let data = readFile(path) - result = calculateXXH3(data) - -proc verifyXXH3*(data: string, expectedHash: XXH3Hash): bool = - ## Verify that data matches the expected xxh3 hash - let calculatedHash = calculateXXH3(data) - result = calculatedHash == expectedHash - -proc verifyXXH3*(data: seq[byte], expectedHash: XXH3Hash): bool = - ## Verify that data matches the expected xxh3 hash - let calculatedHash = calculateXXH3(data) - result = calculatedHash == expectedHash - -proc parseXXH3Hash*(hashStr: string): XXH3Hash = - ## Parse a hash string into XXH3Hash type - ## Validates that it starts with "xxh3-" prefix - if not hashStr.startsWith("xxh3-"): - raise newException(ValueError, "Invalid xxh3 hash format: must start with 'xxh3-'") - result = XXH3Hash(hashStr) - -proc isValidXXH3Hash*(hashStr: string): bool = - ## Check if a string is a valid xxh3 hash format - result = hashStr.startsWith("xxh3-") and hashStr.len > 5 - -when isMainModule: - echo "Testing xxHash Integration..." - - # Test basic hashing - let testData = "Hello, NexusOS with xxh3-128 hashing!" - let hash = calculateXXH3(testData) - echo "Hash: ", $hash - - # Test verification - if verifyXXH3(testData, hash): - echo "✓ Hash verification passed" - else: - echo "✗ Hash verification failed" - - # Test byte sequence hashing - let testBytes = @[byte(72), byte(101), byte(108), byte(108), byte(111)] # "Hello" - let bytesHash = calculateXXH3(testBytes) - echo "Bytes hash: ", $bytesHash - - # Test hash parsing - try: - let parsed = parseXXH3Hash($hash) - echo "✓ Hash parsing successful" - except ValueError as e: - echo "✗ Hash parsing failed: ", e.msg - - # Test invalid hash - if not isValidXXH3Hash("invalid-hash"): - echo "✓ Invalid hash detection works" - - when defined(useXXHash): - echo "✓ Using actual xxhash library" - else: - echo "⚠ Using fallback implementation (install xxhash for production)"