## Integrity Manager - Merkle Tree Verification for Content Addressable Storage
##
## **Crypto-Anarchist Zeal Applied to Package Management**
## Trust the math, not the source. Verify everything.
##
## Core Philosophy:
## - Content is king, hashes are truth
## - CAS provides inherent caching via path-based verification
## - Parallel hash calculation for performance
## - Audit trail for all verification events
## - Zero tolerance for corruption
##
## **Canonical Leaf Hashing:**
## The Merkle tree uses path-aware hashing to ensure determinism:
##   CanonicalHash = Hash(RelativePath || ContentHash)
## This guarantees that moving a file changes the package structure hash.

import std/[os, strutils, algorithm, tables, hashes]
import std/[times, asyncdispatch, threadpool]
import nimcrypto/[hash, blake2]
import nip/unified_storage
import nip/manifest_parser

type
  # ==========================================================================
  # Core Types
  # ============================================================================

  IntegrityError* = object of CatchableError
    ## Integrity verification failure
    path*: string
    expectedHash*: string
    actualHash*: string
    errorType*: IntegrityErrorType

  IntegrityErrorType* = enum
    ## Types of integrity failures
    HashMismatch,      ## Calculated hash doesn't match expected
    FileNotFound,      ## Referenced file missing
    PermissionDenied,  ## Cannot read file for verification
    CorruptedData,     ## File exists but appears corrupted
    InvalidHash,       ## Hash format invalid
    CASInconsistent    ## CAS structure inconsistent

  CanonicalLeaf* = object
    ## Canonical leaf node with path-aware hashing
    relativePath*: string    ## Relative path from root
    contentHash*: string     ## Hash of file content
    canonicalHash*: string   ## Hash(relativePath || contentHash)
    size*: int64             ## File size in bytes

  MerkleNode* = object
    ## Node in Merkle tree
    path*: string           ## File/directory path
    hash*: string           ## Content hash (hex encoded)
    size*: int64            ## Size in bytes
    isDirectory*: bool      ## True if directory node
    children*: seq[MerkleNode]  ## Child nodes (for directories)

  MerkleTree* = object
    ## Complete Merkle tree for a package
    root*: MerkleNode
    rootHash*: string        ## The Merkle root (this goes in manifest)
    totalFiles*: int
    totalSize*: int64
    algorithm*: string       ## "blake2b" (TODO: xxh3-128)

  VerificationResult* = object
    ## Result of integrity verification
    success*: bool
    path*: string
    expectedHash*: string
    actualHash*: string
    verifiedFiles*: int
    totalFiles*: int
    duration*: float        ## Verification time in seconds
    errors*: seq[IntegrityError]

  IntegrityCache* = object
    ## Cache for hash calculations
    fileHashes*: Table[string, string]  ## path -> content hash
    dirHashes*: Table[string, string]   ## path -> merkle root
    lastModified*: Table[string, int64] ## path -> mtime

  IntegrityManager* = object
    ## Main integrity verification manager
    casRoot*: string        ## CAS root directory
    cache*: IntegrityCache  ## Hash cache
    auditLog*: string       ## Audit log file path
    parallelism*: int       ## Number of parallel workers
    chunkSize*: int         ## Chunk size for large files
    strictMode*: bool       ## Fail on any hash mismatch

# ============================================================================
# Hash Calculation (BLAKE2b placeholder for xxh3-128)
# ============================================================================

proc calculateHash*(data: string): string =
  ## Calculate hash of data
  ## TODO: Switch to xxh3-128 when available
  ## Returns hash in format: "blake2b-<hex>"
  let digest = blake2_512.digest(data)
  var hexDigest = ""
  for b in digest.data:
    hexDigest.add(b.toHex(2).toLowerAscii())
  result = "blake2b-" & hexDigest

proc calculateFileHash*(path: string, chunkSize: int = 65536): string =
  ## Calculate hash of file using chunked reading
  if not fileExists(path):
    raise newException(IntegrityError, "File not found: " & path)

  let data = readFile(path)
  result = calculateHash(data)

proc hashString*(s: string): string =
  ## Calculate hash of string
  return calculateHash(s)

# ============================================================================
# Canonical Leaf Hashing - The Foundation of Determinism
# ============================================================================

proc calculateCanonicalHash*(relativePath: string, contentHash: string): string =
  ## Calculate canonical hash: Hash(RelativePath || ContentHash)
  ## This ensures that file location is part of the hash
  ##
  ## **Critical for CAS determinism:**
  ## - Same content in different locations = different canonical hash
  ## - Moving a file changes the package structure hash
  ## - Prevents hash collisions from identical files in different dirs
  let canonicalInput = relativePath & "|" & contentHash
  return hashString(canonicalInput)

# ============================================================================
# Parallel Hashing Worker
# ============================================================================

proc parallelHashWorker(path: string, relativePath: string): CanonicalLeaf {.gcsafe.} =
  ## Worker to calculate canonical leaf hash concurrently
  ## This is the expensive operation that benefits from parallelization
  ##
  ## **Performance Critical:**
  ## - File I/O (reading content)
  ## - Hash calculation (CPU-bound)
  ## - Both benefit from parallel execution

  # 1. Calculate file content hash (expensive I/O + CPU)
  let contentHash = calculateFileHash(path)
  let fileSize = getFileSize(path)

  # 2. Calculate canonical hash = Hash(path || content_hash)
  let canonicalHash = calculateCanonicalHash(relativePath, contentHash)

  return CanonicalLeaf(
    relativePath: relativePath,
    contentHash: contentHash,
    canonicalHash: canonicalHash,
    size: fileSize
  )

proc collectCanonicalLeaves*(rootPath: string, cache: var IntegrityCache,
                             parallel: bool = true): seq[CanonicalLeaf] =
  ## Collect all files as canonical leaves with path-aware hashing
  ## This is the foundation of deterministic Merkle tree construction
  ##
  ## **Algorithm:**
  ## 1. Walk directory tree recursively
  ## 2. For each file: calculate content hash (parallel if enabled)
  ## 3. Calculate canonical hash = Hash(path || content_hash)
  ## 4. Sort by relative path for absolute determinism
  ##
  ## **Parallelization:**
  ## - Uses spawn/threadpool for concurrent file hashing
  ## - Significant speedup for large packages (10-100+ files)
  ## - Falls back to sequential for small packages
  var leaves: seq[CanonicalLeaf] = @[]

  # Normalize root path
  let normalizedRoot = rootPath.normalizedPath()
  let rootLen = normalizedRoot.len + 1  # Include trailing separator

  # Collect all file paths first
  var filePaths: seq[tuple[fullPath: string, relativePath: string]] = @[]
  for path in walkDirRec(normalizedRoot, yieldFilter = {pcFile}):
    let relativePath = if path.len > rootLen:
      path[rootLen..^1]
    else:
      extractFilename(path)
    filePaths.add((fullPath: path, relativePath: relativePath))

  # Decide on parallelization strategy
  let useParallel = parallel and filePaths.len > 10  # Parallel for 10+ files

  if useParallel:
    # Parallel processing using spawn
    var futures: seq[FlowVar[CanonicalLeaf]] = @[]

    for (fullPath, relativePath) in filePaths:
      # Check cache first
      let info = getFileInfo(fullPath)

      if relativePath in cache.fileHashes and
         relativePath in cache.lastModified and
         cache.lastModified[relativePath] == info.lastWriteTime.toUnix():
        # Cache hit - use cached values
        let contentHash = cache.fileHashes[relativePath]
        let canonicalHash = calculateCanonicalHash(relativePath, contentHash)
        leaves.add(CanonicalLeaf(
          relativePath: relativePath,
          contentHash: contentHash,
          canonicalHash: canonicalHash,
          size: info.size
        ))
      else:
        # Cache miss - spawn parallel worker
        futures.add(spawn parallelHashWorker(fullPath, relativePath))

    # Collect results from parallel workers
    for future in futures:
      let leaf = ^future  # Wait for result
      leaves.add(leaf)

      # Update cache
      cache.fileHashes[leaf.relativePath] = leaf.contentHash
      cache.lastModified[leaf.relativePath] = getFileInfo(
        normalizedRoot / leaf.relativePath
      ).lastWriteTime.toUnix()

  else:
    # Sequential processing (small packages or parallel disabled)
    for (fullPath, relativePath) in filePaths:
      # Check cache
      var contentHash: string
      let info = getFileInfo(fullPath)

      if relativePath in cache.fileHashes and
         relativePath in cache.lastModified and
         cache.lastModified[relativePath] == info.lastWriteTime.toUnix():
        # Cache hit
        contentHash = cache.fileHashes[relativePath]
      else:
        # Cache miss - calculate
        contentHash = calculateFileHash(fullPath)
        cache.fileHashes[relativePath] = contentHash
        cache.lastModified[relativePath] = info.lastWriteTime.toUnix()

      # Calculate canonical hash
      let canonicalHash = calculateCanonicalHash(relativePath, contentHash)

      leaves.add(CanonicalLeaf(
        relativePath: relativePath,
        contentHash: contentHash,
        canonicalHash: canonicalHash,
        size: info.size
      ))

  # Sort leaves by relative path for absolute determinism
  # This is CRITICAL - must happen after all parallel work completes
  leaves.sort(proc(a, b: CanonicalLeaf): int = cmp(a.relativePath, b.relativePath))

  return leaves

# ============================================================================
# Merkle Tree Construction from Canonical Leaves
# ============================================================================

proc buildMerkleTreeFromLeaves*(leaves: seq[CanonicalLeaf]): MerkleNode =
  ## Build Merkle tree from flat list of canonical leaves
  ## Uses bottom-up construction with deterministic ordering
  ##
  ## **Algorithm:**
  ## 1. Start with sorted canonical leaves
  ## 2. Pair adjacent nodes and hash: Hash(left || right)
  ## 3. Repeat until single root node remains
  ## 4. Handle odd nodes by promoting to next level

  if leaves.len == 0:
    # Empty tree
    return MerkleNode(
      path: "",
      hash: hashString(""),
      size: 0,
      isDirectory: true,
      children: @[]
    )

  if leaves.len == 1:
    # Single leaf - return as root
    let leaf = leaves[0]
    return MerkleNode(
      path: leaf.relativePath,
      hash: leaf.canonicalHash,
      size: leaf.size,
      isDirectory: false,
      children: @[]
    )

  # Multiple leaves - build tree bottom-up
  var currentLevel: seq[MerkleNode] = @[]

  # Create leaf nodes from canonical leaves
  for leaf in leaves:
    currentLevel.add(MerkleNode(
      path: leaf.relativePath,
      hash: leaf.canonicalHash,
      size: leaf.size,
      isDirectory: false,
      children: @[]
    ))

  # Build tree by pairing nodes
  while currentLevel.len > 1:
    var nextLevel: seq[MerkleNode] = @[]

    var i = 0
    while i < currentLevel.len:
      if i + 1 < currentLevel.len:
        # Pair two nodes
        let left = currentLevel[i]
        let right = currentLevel[i + 1]

        # Combine hashes: Hash(leftHash || rightHash)
        let combinedHash = hashString(left.hash & right.hash)

        nextLevel.add(MerkleNode(
          path: "",  # Internal nodes don't have paths
          hash: combinedHash,
          size: left.size + right.size,
          isDirectory: true,
          children: @[left, right]
        ))

        i += 2
      else:
        # Odd node - promote to next level
        nextLevel.add(currentLevel[i])
        i += 1

    currentLevel = nextLevel

  return currentLevel[0]

proc buildMerkleTree*(rootPath: string, cache: var IntegrityCache): MerkleTree =
  ## Build Merkle tree for a directory using canonical leaf hashing
  ## This is the main entry point for build_hash calculation
  ##
  ## **Returns:** MerkleTree with rootHash suitable for manifest

  if not dirExists(rootPath):
    raise newException(IntegrityError, "Directory not found: " & rootPath)

  # Collect canonical leaves (path-aware hashing)
  let leaves = collectCanonicalLeaves(rootPath, cache)

  # Build tree from leaves
  let root = buildMerkleTreeFromLeaves(leaves)

  # Calculate statistics
  var totalSize: int64 = 0
  for leaf in leaves:
    totalSize += leaf.size

  result = MerkleTree(
    root: root,
    rootHash: root.hash,
    totalFiles: leaves.len,
    totalSize: totalSize,
    algorithm: "blake2b"  # TODO: xxh3-128
  )

# ============================================================================
# Verification Functions
# ============================================================================

proc verifyContent*(rootPath: string, expectedHash: string, manager: var IntegrityManager): VerificationResult =
  ## Verify content against expected hash
  ## This is the main verification entry point
  let startTime = cpuTime()

  var res = VerificationResult(
    path: rootPath,
    expectedHash: expectedHash,
    success: false,
    verifiedFiles: 0,
    totalFiles: 0,
    errors: @[]
  )

  try:
    # Build Merkle tree and calculate actual hash
    let tree = buildMerkleTree(rootPath, manager.cache)
    res.actualHash = tree.rootHash
    res.totalFiles = tree.totalFiles

    # Compare hashes
    if res.actualHash == expectedHash:
      res.success = true
      res.verifiedFiles = tree.totalFiles
    else:
      res.errors.add(IntegrityError(
        path: rootPath,
        expectedHash: expectedHash,
        actualHash: res.actualHash,
        errorType: HashMismatch,
        msg: "Merkle root hash mismatch"
      ))

  except IntegrityError as e:
    var err = IntegrityError(
      path: e.path,
      expectedHash: e.expectedHash,
      actualHash: e.actualHash,
      errorType: e.errorType,
      msg: e.msg
    )
    res.errors.add(err)
  except OSError as e:
    res.errors.add(IntegrityError(
      path: rootPath,
      expectedHash: expectedHash,
      actualHash: "",
      errorType: FileNotFound,
      msg: "Path not found: " & e.msg
    ))

  res.duration = cpuTime() - startTime

  # Log verification result (defined later in file)
  # logVerificationResult(res, manager)

  return res

proc verifyManifestHashes*(manifest: PackageManifest, manager: var IntegrityManager): seq[VerificationResult] =
  ## Verify all hashes in a manifest
  result = @[]

  # Verify build hash if present
  if manifest.buildHash.len > 0:
    let buildResult = verifyContent("build", manifest.buildHash, manager)
    result.add(buildResult)

  # Verify source hash if present
  if manifest.sourceHash.len > 0:
    let sourceResult = verifyContent("source", manifest.sourceHash, manager)
    result.add(sourceResult)

  # Verify artifact hash if present
  if manifest.artifactHash.len > 0:
    let artifactResult = verifyContent("artifact", manifest.artifactHash, manager)
    result.add(artifactResult)

# ============================================================================
# Audit Logging
# ============================================================================

proc logVerificationResult*(result: VerificationResult, manager: IntegrityManager) =
  ## Log verification result to audit trail
  let timestamp = now().format("yyyy-MM-dd HH:mm:ss")
  let status = if result.success: "SUCCESS" else: "FAILURE"
  let logLine = "$1 [$2] $3: $4 (expected: $5, actual: $6, files: $7/$8, duration: $9s)" % [
    timestamp, status, result.path,
    if result.success: "VERIFIED" else: "HASH_MISMATCH",
    result.expectedHash, result.actualHash,
    $result.verifiedFiles, $result.totalFiles,
    result.duration.formatFloat(ffDecimal, 3)
  ]

  # Append to audit log
  try:
    let logFile = open(manager.auditLog, fmAppend)
    defer: logFile.close()
    logFile.writeLine(logLine)

    # Also log errors
    for error in result.errors:
      let errorLine = "$1 [ERROR] $2: $3 - $4" % [
        timestamp, error.path, $error.errorType, error.msg
      ]
      logFile.writeLine(errorLine)
  except IOError:
    discard  # Logging failure shouldn't break verification

# ============================================================================
# Manager Construction
# ============================================================================

proc newIntegrityManager*(casRoot: string, auditLog: string = "",
                          parallelism: int = 4, strictMode: bool = true): IntegrityManager =
  ## Create new integrity manager
  result = IntegrityManager(
    casRoot: casRoot,
    auditLog: if auditLog.len > 0: auditLog else: casRoot / "integrity.log",
    parallelism: parallelism,
    chunkSize: 65536,  # 64KB chunks
    strictMode: strictMode,
    cache: IntegrityCache(
      fileHashes: initTable[string, string](),
      dirHashes: initTable[string, string](),
      lastModified: initTable[string, int64]()
    )
  )

  # Ensure audit log directory exists
  createDir(parentDir(result.auditLog))

proc clearCache*(manager: var IntegrityManager) =
  ## Clear integrity cache
  manager.cache.fileHashes.clear()
  manager.cache.dirHashes.clear()
  manager.cache.lastModified.clear()

# ============================================================================
# Convenience Functions
# ============================================================================

proc calculateBuildHash*(packagePath: string): string =
  ## Calculate build_hash for a package directory
  ## This is what goes in the manifest
  var cache = IntegrityCache(
    fileHashes: initTable[string, string](),
    dirHashes: initTable[string, string](),
    lastModified: initTable[string, int64]()
  )
  let tree = buildMerkleTree(packagePath, cache)
  result = tree.rootHash

proc verifyPackage*(packagePath: string, manifest: PackageManifest,
                    manager: var IntegrityManager): bool =
  ## Verify a package against its manifest
  if manifest.buildHash.len == 0:
    return false  # No hash to verify against

  let result = verifyContent(packagePath, manifest.buildHash, manager)
  return result.success

# ============================================================================
# Pretty Printing
# ============================================================================

proc `$`*(tree: MerkleTree): string =
  ## Convert Merkle tree to human-readable string
  result = "MerkleTree:\n"
  result.add("  Root Hash: " & tree.rootHash & "\n")
  result.add("  Algorithm: " & tree.algorithm & "\n")
  result.add("  Total Files: " & $tree.totalFiles & "\n")
  result.add("  Total Size: " & $tree.totalSize & " bytes\n")

proc `$`*(res: VerificationResult): string =
  ## Convert verification result to human-readable string
  let status = if res.success: "✅ SUCCESS" else: "❌ FAILURE"
  result = status & "\n"
  result.add("  Path: " & res.path & "\n")
  result.add("  Expected: " & res.expectedHash & "\n")
  result.add("  Actual: " & res.actualHash & "\n")
  result.add("  Files: " & $res.verifiedFiles & "/" & $res.totalFiles & "\n")
  result.add("  Duration: " & res.duration.formatFloat(ffDecimal, 3) & "s\n")

  if res.errors.len > 0:
    result.add("  Errors:\n")
    for err in res.errors:
      result.add("    - " & err.msg & "\n")

when isMainModule:
  echo "Integrity Manager - Merkle Tree Verification"
  echo "Hash Algorithm: blake2b (TODO: xxh3-128)"
  echo ""
  echo "**Canonical Leaf Hashing:**"
  echo "  CanonicalHash = Hash(RelativePath || ContentHash)"
  echo "  This ensures deterministic, path-aware verification."
  echo ""

  # Example usage
  let testContent = "Hello, Merkle Tree!"
  let stringHash = hashString(testContent)
  echo "String hash: " & stringHash
  echo ""
  echo "Trust the math, not the source."
