## Enhanced Garbage Collection System
##
## This module implements an enhanced garbage collection system for the unified
## storage architecture, providing threshold-based triggering, format-aware
## garbage identification, safe deletion mechanisms, and optimized performance
## for large-scale operations across NPK, NIP, and NEXTER formats.

import std/[os, tables, sets, strutils, json, hashes, times]
import std/threadpool  # For parallel operations
import xxhash  # For hashing
import ./cas  # CAS system for storage management

# Import Result types from cas module
type
  Result*[T, E] = object
    case isOk*: bool
    of true:
      value*: T
    of false:
      error*: E

  VoidResult*[E] = object
    case isOk*: bool
    of true:
      discard
    of false:
      errValue*: E

proc ok*[T, E](val: T): Result[T, E] =
  Result[T, E](isOk: true, value: val)

proc err*[T, E](error: E): Result[T, E] =
  Result[T, E](isOk: false, error: error)

proc ok*[E](dummy: typedesc[E]): VoidResult[E] =
  VoidResult[E](isOk: true)

proc isErr*[T, E](r: Result[T, E]): bool = not r.isOk
proc get*[T, E](r: Result[T, E]): T = r.value
proc getError*[T, E](r: Result[T, E]): E = r.error

type
  ## Enhanced garbage collection configuration
  GcConfig* = object
    priorityStrategy*: GcPriority
    batchSize*: int
    maxWorkers*: int
    dryRun*: bool
    verbose*: bool
    adaptiveMode*: bool

  ## Priority strategies for garbage collection
  GcPriority* = enum
    SafetyFirst,     # Prioritize safety over space reclamation
    Balanced,        # Balance between safety and space efficiency
    Aggressive       # Maximize space reclamation

  ## Garbage collection trigger configuration
  GcTriggerConfig* = object
    storageThreshold*: float     # 0.0-1.0, default: 0.8 = 80% usage
    timeIntervalHours*: int      # Time interval for automatic GC in hours (0 = disabled)
    minFreeSpace*: int64         # Minimum free space in MB (default: 1024 = 1GB)
    adaptiveMode*: bool          # Enable adaptive threshold adjustment

  ## Garbage collection result statistics
  GcResult* = object
    deletedCount*: int           # Number of objects deleted
    sizeFreed*: int64            # Total bytes reclaimed
    identificationTime*: float   # Time spent identifying garbage (seconds)
    deletionTime*: float         # Time spent deleting garbage (seconds)
    formatsProcessed*: int       # Number of formats processed
    refCountErrors*: int         # Reference count errors encountered
    lockErrors*: int             # Lock timeout errors

  ## Batch processing result
  GcBatchResult* = object
    deletedCount*: int
    sizeFreed*: int64

  ## Individual deletion result
  GcDeleteResult* = object
    success*: bool
    sizeFreed*: int64
    error*: string

  ## Garbage collection error types
  GcError* = object
    operation*: string
    message*: string

  ## Main garbage collector type
  GarbageCollector* = object
    casManager*: ptr CasManager     # Reference to CAS manager
    config*: GcConfig               # GC configuration
    triggerConfig*: GcTriggerConfig # Trigger configuration
    lastGcTime*: DateTime          # Last garbage collection timestamp
    stats*: GcResult               # Last garbage collection statistics
    auditLog*: string              # Enhanced audit log path

const
  DEFAULT_GC_CONFIG = GcConfig(
    priorityStrategy: Balanced,
    batchSize: 1000,
    maxWorkers: 8,
    dryRun: false,
    verbose: false,
    adaptiveMode: true
  )

  DEFAULT_TRIGGER_CONFIG = GcTriggerConfig(
    storageThreshold: 0.8,      # 80% storage usage
    timeIntervalHours: 24,      # Daily GC
    minFreeSpace: 1024,         # 1GB minimum free space
    adaptiveMode: true
  )

## Utility Functions

proc getDiskSpace*(path: string): int64 =
  ## Get total disk space for a path (simplified implementation)
  # For now, return a reasonable default
  return 1000 * 1024 * 1024 * 1024  # 1TB default

proc getAllPinnedObjects*(cas: CasManager): HashSet[string] =
  ## Get all pinned objects across all pin sets
  result = initHashSet[string]()
  for pinSet in cas.pinSets.values:
    result = result.union(pinSet)

proc getRefCountPath*(cas: CasManager, hash: string): string =
  ## Get path to reference count file for a hash
  result = cas.userCasPath / "refs" / hash.split('-')[1] & ".refcount"

## Garbage Collection Core Logic

proc initGarbageCollector*(casManager: ptr CasManager, config: GcConfig = DEFAULT_GC_CONFIG, 
                          triggerConfig: GcTriggerConfig = DEFAULT_TRIGGER_CONFIG): GarbageCollector =
  ## Initialize enhanced garbage collector
  result = GarbageCollector(
    casManager: casManager,
    config: config,
    triggerConfig: triggerConfig,
    lastGcTime: now(),
    auditLog: casManager[].auditLog
  )

proc shouldTriggerGarbageCollection*(gc: GarbageCollector): bool =
  ## Determine if garbage collection should be triggered based on thresholds
  let stats = gc.casManager[].getStats()
  let totalSpace = getDiskSpace(gc.casManager[].rootPath)
  let freeSpace = totalSpace - stats.compressedSize

  # Check storage threshold
  if stats.compressedSize.float / totalSpace.float > gc.triggerConfig.storageThreshold:
    return true

  # Check minimum free space
  if freeSpace < gc.triggerConfig.minFreeSpace * 1024 * 1024:  # Convert MB to bytes
    return true

  # Check time-based trigger
  if gc.triggerConfig.timeIntervalHours > 0:
    let timeSinceLastGc = now() - gc.lastGcTime
    let hoursSinceLastGc = timeSinceLastGc.inHours
    if hoursSinceLastGc >= gc.triggerConfig.timeIntervalHours:
      return true

  false

proc getReachableHashesFromAllFormats*(cas: CasManager): HashSet[string] =
  ## Get all hashes that are reachable from any format (NPK, NIP, NEXTER)
  result = initHashSet[string]()

  # Check all format reference files
  for formatType in FormatType:
    let formatDir = case formatType
      of NPK: "npks"
      of NIP: "nips"
      of NEXTER: "nexters"

    let refsDir = cas.refsPath / formatDir
    if not dirExists(refsDir):
      continue

    for refFile in walkDir(refsDir):
      if refFile.kind == pcFile and refFile.path.endsWith(".refs"):
        try:
          let content = readFile(refFile.path).strip()
          if content.len > 0:
            let hashes = content.split('\n')
            for hash in hashes:
              if hash.len > 0:
                result.incl(hash)
        except:
          # Skip files that can't be read
          discard

proc identifyFormatGarbage*(cas: ptr CasManager): Table[FormatType, seq[string]] =
  ## Identify format-specific garbage with priority handling
  var formatGarbage = initTable[FormatType, seq[string]]()

  # Get all protected objects (pinned + reachable)
  let pinnedObjects = cas[].getAllPinnedObjects()
  let reachableHashes = cas[].getReachableHashesFromAllFormats()
  let protectedObjects = reachableHashes.union(pinnedObjects)

  # Scan each format for orphaned chunks
  for formatType in FormatType:
    let formatDir = case formatType
      of NPK: "npks"
      of NIP: "nips"
      of NEXTER: "nexters"

    let refsDir = cas[].refsPath / formatDir
    if not dirExists(refsDir):
      continue

    # Get all chunks referenced by this format
    let formatRefs = cas[].formatRefs.getOrDefault(formatType, initTable[string, HashSet[string]]())
    
    # Collect all hashes for this format
    var formatHashes = initHashSet[string]()
    for packageHashes in formatRefs.values:
      formatHashes = formatHashes.union(packageHashes)

    # Identify orphaned chunks (not in protected set and ref count <= 0)
    var garbageHashes = newSeq[string]()
    for hash in formatHashes:
      if hash notin protectedObjects:
        var casRef = cas[]  # Make mutable for getRefCount
        let refCount = casRef.getRefCount(hash)
        if refCount <= 0:
          garbageHashes.add(hash)

    if garbageHashes.len > 0:
      formatGarbage[formatType] = garbageHashes

  formatGarbage

proc safeDeleteObject*(cas: ptr CasManager, hash: string): GcDeleteResult =
  ## Safely delete an object (simplified version without complex locking)
  var delResult = GcDeleteResult(success: false, sizeFreed: 0, error: "")
  
  try:
    # Verify object still exists and is garbage
    let objPath = getObjectPath(cas[].userCasPath, hash)
    if not fileExists(objPath):
      delResult.success = true  # Already deleted
      return delResult

    # Verify ref count is still <= 0 (read directly from disk to avoid side effects)
    let refPath = getRefCountPath(cas[], hash)
    var refCount = 0
    if fileExists(refPath):
      try:
        refCount = parseInt(readFile(refPath).strip())
      except:
        refCount = 0
        
    if refCount > 0:
      delResult.success = true  # No longer garbage
      return delResult

    # Get file size before deletion
    let fileInfo = getFileInfo(objPath)
    delResult.sizeFreed = fileInfo.size

    # Perform atomic deletion
    let tempPath = objPath & ".deleting"
    moveFile(objPath, tempPath)

    # Remove reference count file
    if fileExists(refPath):
      removeFile(refPath)

    # Remove temp file
    removeFile(tempPath)

    delResult.success = true

  except IOError as e:
    delResult.error = "IO Error: " & e.msg
  except Exception as e:
    delResult.error = "Unexpected error: " & e.msg

  return delResult

proc parallelDeleteBatch*(cas: ptr CasManager, batch: seq[string]): GcBatchResult =
  ## Delete a batch of objects in parallel for performance
  var deletedCount = 0
  var sizeFreed = 0'i64
  var futures: seq[FlowVar[GcDeleteResult]] = @[]

  # Spawn parallel deletion tasks
  for hash in batch:
    futures.add(spawn safeDeleteObject(cas, hash))

  # Collect results
  for future in futures:
    let delRes = ^future
    if delRes.success:
      deletedCount.inc
      sizeFreed += delRes.sizeFreed

  GcBatchResult(deletedCount: deletedCount, sizeFreed: sizeFreed)

proc prioritizeGarbage*(formatGarbage: Table[FormatType, seq[string]], priority: GcPriority): seq[string] =
  ## Prioritize garbage collection based on format priority
  result = newSeq[string]()

  case priority:
  of SafetyFirst:
    # Collect in NIP -> NEXTER -> NPK order (user safety first)
    if formatGarbage.hasKey(NIP):
      result.add(formatGarbage[NIP])
    if formatGarbage.hasKey(NEXTER):
      result.add(formatGarbage[NEXTER])
    if formatGarbage.hasKey(NPK):
      result.add(formatGarbage[NPK])

  of Aggressive:
    # Collect in NPK -> NEXTER -> NIP order (maximize space)
    if formatGarbage.hasKey(NPK):
      result.add(formatGarbage[NPK])
    if formatGarbage.hasKey(NEXTER):
      result.add(formatGarbage[NEXTER])
    if formatGarbage.hasKey(NIP):
      result.add(formatGarbage[NIP])

  of Balanced:
    # Collect in NEXTER -> NIP -> NPK order (balanced approach)
    if formatGarbage.hasKey(NEXTER):
      result.add(formatGarbage[NEXTER])
    if formatGarbage.hasKey(NIP):
      result.add(formatGarbage[NIP])
    if formatGarbage.hasKey(NPK):
      result.add(formatGarbage[NPK])

proc garbageCollect*(gc: var GarbageCollector): Result[GcResult, GcError] =
  ## Main garbage collection function with all enhancements
  var totalDeleted = 0
  var totalSizeFreed = 0'i64
  var identificationTime = 0.0
  var deletionTime = 0.0
  var refCountErrors = 0
  var lockErrors = 0

  try:
    # Phase 1: Identification
    let identificationStart = now()
    let formatGarbage = identifyFormatGarbage(gc.casManager)
    identificationTime = (now() - identificationStart).inMilliseconds.float / 1000.0

    if gc.config.verbose:
      echo "Found garbage in ", formatGarbage.len, " formats"

    # Phase 2: Prioritization
    let prioritizedGarbage = prioritizeGarbage(formatGarbage, gc.config.priorityStrategy)

    if gc.config.dryRun:
      # Dry run - just report what would be deleted
      totalDeleted = prioritizedGarbage.len
      if gc.config.verbose:
        echo "DRY RUN: Would delete ", totalDeleted, " objects"
      return ok[GcResult, GcError](GcResult(
        deletedCount: totalDeleted,
        sizeFreed: 0,
        identificationTime: identificationTime,
        deletionTime: 0.0,
        formatsProcessed: formatGarbage.len,
        refCountErrors: 0,
        lockErrors: 0
      ))

    # Phase 3: Batch deletion (parallel)
    let deletionStart = now()
    let batchSize = gc.config.batchSize
    var processed = 0

    # Process in batches for memory efficiency
    for i in countup(0, prioritizedGarbage.len - 1, batchSize):
      let batchEnd = min(i + batchSize - 1, prioritizedGarbage.len - 1)
      let batch = prioritizedGarbage[i..batchEnd]

      if gc.config.verbose:
        echo "Processing batch ", processed div batchSize + 1, " (objects ", i+1, "-", batchEnd+1, ")"

      # Parallel deletion of batch
      let batchResult = parallelDeleteBatch(gc.casManager, batch)
      totalDeleted += batchResult.deletedCount
      totalSizeFreed += batchResult.sizeFreed
      processed += batch.len

      # Check if we should continue (progress reporting)
      if gc.config.verbose and processed mod (batchSize * 5) == 0:
        echo "Progress: ", processed, "/", prioritizedGarbage.len, " objects processed"

    deletionTime = (now() - deletionStart).inMilliseconds.float / 1000.0

    # Update statistics and last GC time
    gc.lastGcTime = now()
    gc.stats = GcResult(
      deletedCount: totalDeleted,
      sizeFreed: totalSizeFreed,
      identificationTime: identificationTime,
      deletionTime: deletionTime,
      formatsProcessed: formatGarbage.len,
      refCountErrors: refCountErrors,
      lockErrors: lockErrors
    )

    # Log completion
    let timestamp = now().format("yyyy-MM-dd'T'HH:mm:ss'Z'")
    let logEntry = "[" & timestamp & "] GC_COMPLETED deleted=" & $totalDeleted & " size_freed=" & $totalSizeFreed & "\n"
    let logFile = open(gc.auditLog, fmAppend)
    logFile.write(logEntry)
    logFile.close()

    return ok[GcResult, GcError](gc.stats)

  except Exception as e:
    return err[GcResult, GcError](GcError(
      operation: "garbageCollect",
      message: "Garbage collection failed: " & e.msg
    ))

## Public API Functions

proc getGcStats*(gc: GarbageCollector): GcResult =
  ## Get last garbage collection statistics
  return gc.stats

proc shouldRunGc*(gc: GarbageCollector): bool =
  ## Convenience function to check if GC should run
  return gc.shouldTriggerGarbageCollection()

proc runGarbageCollection*(cas: ptr CasManager, config: GcConfig = DEFAULT_GC_CONFIG): Result[GcResult, GcError] =
  ## Convenience function to run garbage collection with default settings
  var gc = initGarbageCollector(cas, config)
  return gc.garbageCollect()