Phase 4: Advanced L0 Validation (Deterministic Ordering, Replay Filtering, and Trust Distance Integration)
This commit is contained in:
parent
2276954ba3
commit
995e74dc18
|
|
@ -101,9 +101,9 @@ The Libertaria L0-L1 SDK in Zig is **reaching maturity with 50% scope complete**
|
||||||
- ⏳ Queue manifest generation
|
- ⏳ Queue manifest generation
|
||||||
- ✅ Automatic pruning of expired packets
|
- ✅ Automatic pruning of expired packets
|
||||||
- ⏳ Frame validation pipeline
|
- ⏳ Frame validation pipeline
|
||||||
- ⏳ Deterministic ordering
|
- ✅ Deterministic ordering (Sequencer + Reorder Buffer)
|
||||||
- ⏳ Replay attack detection
|
- ✅ Replay attack detection (Replay Filter)
|
||||||
- ⏳ Trust distance integration
|
- ✅ Trust distance integration (Resolver + Categories)
|
||||||
- **Dependency:** Requires Phase 3 (DONE ✅)
|
- **Dependency:** Requires Phase 3 (DONE ✅)
|
||||||
- **Blocks:** Phase 5 FFI boundary
|
- **Blocks:** Phase 5 FFI boundary
|
||||||
- **Estimated:** 3 weeks
|
- **Estimated:** 3 weeks
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,11 @@
|
||||||
pub const store = @import("opq/store.zig");
|
pub const store = @import("opq/store.zig");
|
||||||
pub const quota = @import("opq/quota.zig");
|
pub const quota = @import("opq/quota.zig");
|
||||||
pub const manager = @import("opq/manager.zig");
|
pub const manager = @import("opq/manager.zig");
|
||||||
|
pub const manifest = @import("opq/manifest.zig");
|
||||||
|
pub const merkle = @import("opq/merkle.zig");
|
||||||
|
pub const sequencer = @import("opq/sequencer.zig");
|
||||||
|
pub const reorder_buffer = @import("opq/reorder_buffer.zig");
|
||||||
|
pub const trust_resolver = @import("opq/trust_resolver.zig");
|
||||||
|
|
||||||
pub const OPQManager = manager.OPQManager;
|
pub const OPQManager = manager.OPQManager;
|
||||||
pub const Policy = quota.Policy;
|
pub const Policy = quota.Policy;
|
||||||
|
|
|
||||||
|
|
@ -5,14 +5,19 @@
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
const store = @import("store.zig");
|
const store = @import("store.zig");
|
||||||
const quota = @import("quota.zig");
|
const quota = @import("quota.zig");
|
||||||
|
const manifest = @import("manifest.zig");
|
||||||
|
const sequencer = @import("sequencer.zig");
|
||||||
|
const trust_resolver = @import("trust_resolver.zig");
|
||||||
const lwf = @import("lwf");
|
const lwf = @import("lwf");
|
||||||
|
|
||||||
pub const OPQManager = struct {
|
pub const OPQManager = struct {
|
||||||
allocator: std.mem.Allocator,
|
allocator: std.mem.Allocator,
|
||||||
policy: quota.Policy,
|
policy: quota.Policy,
|
||||||
store: store.WALStore,
|
store: store.WALStore,
|
||||||
|
index: std.ArrayListUnmanaged(manifest.PacketSummary),
|
||||||
|
trust_resolver: trust_resolver.TrustResolver,
|
||||||
|
|
||||||
pub fn init(allocator: std.mem.Allocator, base_dir: []const u8, persona: quota.Persona) !OPQManager {
|
pub fn init(allocator: std.mem.Allocator, base_dir: []const u8, persona: quota.Persona, resolver: trust_resolver.TrustResolver) !OPQManager {
|
||||||
const policy = quota.Policy.init(persona);
|
const policy = quota.Policy.init(persona);
|
||||||
const wal = try store.WALStore.init(allocator, base_dir, policy.segment_size);
|
const wal = try store.WALStore.init(allocator, base_dir, policy.segment_size);
|
||||||
|
|
||||||
|
|
@ -20,22 +25,69 @@ pub const OPQManager = struct {
|
||||||
.allocator = allocator,
|
.allocator = allocator,
|
||||||
.policy = policy,
|
.policy = policy,
|
||||||
.store = wal,
|
.store = wal,
|
||||||
|
.index = .{},
|
||||||
|
.trust_resolver = resolver,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn deinit(self: *OPQManager) void {
|
pub fn deinit(self: *OPQManager) void {
|
||||||
self.store.deinit();
|
self.store.deinit();
|
||||||
|
self.index.deinit(self.allocator);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ingest a frame into the queue
|
/// Ingest a frame into the queue
|
||||||
pub fn ingestFrame(self: *OPQManager, frame: *const lwf.LWFFrame) !void {
|
pub fn ingestFrame(self: *OPQManager, frame: *const lwf.LWFFrame) !void {
|
||||||
// 1. Append to WAL
|
// 1. Resolve Trust Category
|
||||||
try self.store.appendFrame(frame);
|
const category = self.trust_resolver.resolve(frame.header.source_hint);
|
||||||
|
|
||||||
// 2. Periodic maintenance (could be on a timer, but here we do it after ingest)
|
// 2. Resource Triage (Mechanism: Drop low-trust if busy)
|
||||||
|
// In a real implementation, we'd check current_total_size vs policy.
|
||||||
|
// For now, we allow the ingestion and rely on maintenance to prune.
|
||||||
|
|
||||||
|
// 3. Append to WAL
|
||||||
|
const loc = try self.store.appendFrame(frame);
|
||||||
|
|
||||||
|
// 2. Update In-Memory Index (Summary)
|
||||||
|
// Note: In real scenarios, queue_id should be deterministic or from header.
|
||||||
|
// For now, we use a random ID or part of checksum.
|
||||||
|
var q_id: [16]u8 = undefined;
|
||||||
|
std.crypto.random.bytes(&q_id);
|
||||||
|
|
||||||
|
try self.index.append(self.allocator, .{
|
||||||
|
.queue_id = q_id,
|
||||||
|
.sender_hint = frame.header.source_hint,
|
||||||
|
.size = @intCast(loc.len),
|
||||||
|
.priority = if (frame.header.flags & lwf.LWFFlags.PRIORITY != 0) .high else .normal,
|
||||||
|
.created_at = std.time.timestamp(),
|
||||||
|
.timestamp = frame.header.timestamp,
|
||||||
|
.sequence = frame.header.sequence,
|
||||||
|
.expires_at = std.time.timestamp() + self.policy.max_retention_seconds,
|
||||||
|
.entropy_cost = frame.header.entropy_difficulty,
|
||||||
|
.category = category,
|
||||||
|
});
|
||||||
|
|
||||||
|
// 5. Periodic maintenance
|
||||||
try self.maintenance();
|
try self.maintenance();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn generateManifest(self: *OPQManager, recipient: [24]u8) !manifest.QueueManifest {
|
||||||
|
var qm = manifest.QueueManifest.init(self.allocator, recipient);
|
||||||
|
errdefer qm.deinit();
|
||||||
|
|
||||||
|
for (self.index.items) |item| {
|
||||||
|
// In a real relay, we would filter by recipient!
|
||||||
|
// For now, we just add everything to the manifest.
|
||||||
|
try qm.items.append(self.allocator, item);
|
||||||
|
qm.total_count += 1;
|
||||||
|
qm.total_size += item.size;
|
||||||
|
}
|
||||||
|
|
||||||
|
sequencer.sortDeterministically(qm.items.items);
|
||||||
|
|
||||||
|
try qm.calculateMerkleRoot();
|
||||||
|
return qm;
|
||||||
|
}
|
||||||
|
|
||||||
pub fn maintenance(self: *OPQManager) !void {
|
pub fn maintenance(self: *OPQManager) !void {
|
||||||
// 1. Prune by TTL
|
// 1. Prune by TTL
|
||||||
_ = try self.store.prune(self.policy.max_retention_seconds);
|
_ = try self.store.prune(self.policy.max_retention_seconds);
|
||||||
|
|
@ -53,7 +105,7 @@ test "OPQ Manager: Policy Enforcement" {
|
||||||
defer std.fs.cwd().deleteTree(test_dir) catch {};
|
defer std.fs.cwd().deleteTree(test_dir) catch {};
|
||||||
|
|
||||||
// 1. Client Policy: 5MB limit, 1hr TTL
|
// 1. Client Policy: 5MB limit, 1hr TTL
|
||||||
var manager = try OPQManager.init(allocator, test_dir, .client);
|
var manager = try OPQManager.init(allocator, test_dir, .client, trust_resolver.TrustResolver.noop());
|
||||||
defer manager.deinit();
|
defer manager.deinit();
|
||||||
|
|
||||||
try std.testing.expectEqual(manager.policy.max_storage_bytes, 5 * 1024 * 1024);
|
try std.testing.expectEqual(manager.policy.max_storage_bytes, 5 * 1024 * 1024);
|
||||||
|
|
@ -62,4 +114,50 @@ test "OPQ Manager: Policy Enforcement" {
|
||||||
var frame = try lwf.LWFFrame.init(allocator, 10);
|
var frame = try lwf.LWFFrame.init(allocator, 10);
|
||||||
defer frame.deinit(allocator);
|
defer frame.deinit(allocator);
|
||||||
try manager.ingestFrame(&frame);
|
try manager.ingestFrame(&frame);
|
||||||
|
|
||||||
|
// 3. Generate Manifest
|
||||||
|
const recipient = [_]u8{0} ** 24;
|
||||||
|
var mf = try manager.generateManifest(recipient);
|
||||||
|
defer mf.deinit();
|
||||||
|
|
||||||
|
try std.testing.expectEqual(mf.total_count, 1);
|
||||||
|
try std.testing.expect(mf.total_size > 0);
|
||||||
|
try std.testing.expect(!std.mem.eql(u8, &mf.merkle_root, &[_]u8{0} ** 32));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "OPQ Manager: Deterministic Manifest Ordering" {
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
const test_dir = "test_opq_ordering";
|
||||||
|
|
||||||
|
std.fs.cwd().deleteTree(test_dir) catch {};
|
||||||
|
defer std.fs.cwd().deleteTree(test_dir) catch {};
|
||||||
|
|
||||||
|
var manager = try OPQManager.init(allocator, test_dir, .relay, trust_resolver.TrustResolver.noop());
|
||||||
|
defer manager.deinit();
|
||||||
|
|
||||||
|
// 1. Ingest frames out of order
|
||||||
|
// Frame A: Time 200, Seq 2
|
||||||
|
var f1 = try lwf.LWFFrame.init(allocator, 10);
|
||||||
|
defer f1.deinit(allocator);
|
||||||
|
f1.header.timestamp = 200;
|
||||||
|
f1.header.sequence = 2;
|
||||||
|
f1.updateChecksum();
|
||||||
|
try manager.ingestFrame(&f1);
|
||||||
|
|
||||||
|
// Frame B: Time 100, Seq 1 (Should come first)
|
||||||
|
var f2 = try lwf.LWFFrame.init(allocator, 10);
|
||||||
|
defer f2.deinit(allocator);
|
||||||
|
f2.header.timestamp = 100;
|
||||||
|
f2.header.sequence = 1;
|
||||||
|
f2.updateChecksum();
|
||||||
|
try manager.ingestFrame(&f2);
|
||||||
|
|
||||||
|
// 2. Generate Manifest
|
||||||
|
const recipient = [_]u8{0} ** 24;
|
||||||
|
var mf = try manager.generateManifest(recipient);
|
||||||
|
defer mf.deinit();
|
||||||
|
|
||||||
|
// 3. Verify Order: item[0] should be timestamp 100
|
||||||
|
try std.testing.expectEqual(mf.items.items[0].timestamp, 100);
|
||||||
|
try std.testing.expectEqual(mf.items.items[1].timestamp, 200);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,70 @@
|
||||||
|
//! RFC-0020: OPQ Manifests & Summaries
|
||||||
|
//!
|
||||||
|
//! Provides bandwidth-efficient triage of queued packets.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
const merkle = @import("merkle.zig");
|
||||||
|
const quota = @import("quota.zig");
|
||||||
|
|
||||||
|
pub const Priority = enum(u8) {
|
||||||
|
low = 0,
|
||||||
|
normal = 1,
|
||||||
|
high = 2,
|
||||||
|
critical = 3,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const PacketSummary = struct {
|
||||||
|
queue_id: [16]u8,
|
||||||
|
sender_hint: [24]u8, // DID hint
|
||||||
|
size: u32,
|
||||||
|
priority: Priority,
|
||||||
|
created_at: i64, // WALL time (for expiry)
|
||||||
|
timestamp: u64, // L0 nanoseconds (for ordering)
|
||||||
|
sequence: u32, // L0 sequence (for ordering/replay)
|
||||||
|
expires_at: i64,
|
||||||
|
entropy_cost: u16,
|
||||||
|
category: quota.TrustCategory,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const QueueManifest = struct {
|
||||||
|
allocator: std.mem.Allocator,
|
||||||
|
recipient_hint: [24]u8,
|
||||||
|
total_count: usize,
|
||||||
|
total_size: u64,
|
||||||
|
items: std.ArrayListUnmanaged(PacketSummary),
|
||||||
|
merkle_root: [32]u8,
|
||||||
|
|
||||||
|
pub fn init(allocator: std.mem.Allocator, recipient: [24]u8) QueueManifest {
|
||||||
|
return .{
|
||||||
|
.allocator = allocator,
|
||||||
|
.recipient_hint = recipient,
|
||||||
|
.total_count = 0,
|
||||||
|
.total_size = 0,
|
||||||
|
.items = .{},
|
||||||
|
.merkle_root = [_]u8{0} ** 32,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deinit(self: *QueueManifest) void {
|
||||||
|
self.items.deinit(self.allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn calculateMerkleRoot(self: *QueueManifest) !void {
|
||||||
|
var tree = merkle.MerkleTree.init(self.allocator);
|
||||||
|
defer tree.deinit();
|
||||||
|
|
||||||
|
for (self.items.items) |item| {
|
||||||
|
// Hash the summary to form a leaf
|
||||||
|
var hasher = std.crypto.hash.Blake3.init(.{});
|
||||||
|
hasher.update(&item.queue_id);
|
||||||
|
hasher.update(&item.sender_hint);
|
||||||
|
hasher.update(std.mem.asBytes(&item.size));
|
||||||
|
hasher.update(std.mem.asBytes(&item.created_at));
|
||||||
|
var leaf: [32]u8 = undefined;
|
||||||
|
hasher.final(&leaf);
|
||||||
|
try tree.insert(leaf);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.merkle_root = tree.getRoot();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -0,0 +1,144 @@
|
||||||
|
//! Incremental Merkle Tree implementation for OPQ Manifests.
|
||||||
|
//!
|
||||||
|
//! Provides O(log n) updates and O(log n) inclusion proofs.
|
||||||
|
//! Uses Blake3 for hashing to align with the rest of the SDK.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
|
||||||
|
pub const MerkleTree = struct {
|
||||||
|
allocator: std.mem.Allocator,
|
||||||
|
leaves: std.ArrayListUnmanaged([32]u8),
|
||||||
|
|
||||||
|
pub fn init(allocator: std.mem.Allocator) MerkleTree {
|
||||||
|
return .{
|
||||||
|
.allocator = allocator,
|
||||||
|
.leaves = .{},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deinit(self: *MerkleTree) void {
|
||||||
|
self.leaves.deinit(self.allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn insert(self: *MerkleTree, leaf: [32]u8) !void {
|
||||||
|
try self.leaves.append(self.allocator, leaf);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Calculate the root of the Merkle Tree
|
||||||
|
pub fn getRoot(self: *const MerkleTree) [32]u8 {
|
||||||
|
if (self.leaves.items.len == 0) return [_]u8{0} ** 32;
|
||||||
|
if (self.leaves.items.len == 1) return self.leaves.items[0];
|
||||||
|
|
||||||
|
// This is a naive implementation for now.
|
||||||
|
// For production, we'd want an incremental tree that doesn't recompute everything.
|
||||||
|
var current_level = std.ArrayList([32]u8).empty;
|
||||||
|
defer current_level.deinit(self.allocator);
|
||||||
|
current_level.appendSlice(self.allocator, self.leaves.items) catch return [_]u8{0} ** 32;
|
||||||
|
|
||||||
|
while (current_level.items.len > 1) {
|
||||||
|
var next_level = std.ArrayList([32]u8).empty;
|
||||||
|
var i: usize = 0;
|
||||||
|
while (i < current_level.items.len) : (i += 2) {
|
||||||
|
const left = current_level.items[i];
|
||||||
|
const right = if (i + 1 < current_level.items.len) current_level.items[i + 1] else left;
|
||||||
|
|
||||||
|
var hasher = std.crypto.hash.Blake3.init(.{});
|
||||||
|
hasher.update(&left);
|
||||||
|
hasher.update(&right);
|
||||||
|
var out: [32]u8 = undefined;
|
||||||
|
hasher.final(&out);
|
||||||
|
next_level.append(self.allocator, out) catch break;
|
||||||
|
}
|
||||||
|
current_level.deinit(self.allocator);
|
||||||
|
current_level = next_level;
|
||||||
|
}
|
||||||
|
|
||||||
|
return current_level.items[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generate an inclusion proof for the leaf at index
|
||||||
|
pub fn getProof(self: *const MerkleTree, index: usize) ![][32]u8 {
|
||||||
|
if (index >= self.leaves.items.len) return error.IndexOutOfBounds;
|
||||||
|
|
||||||
|
var proof = std.ArrayList([32]u8).empty;
|
||||||
|
errdefer proof.deinit(self.allocator);
|
||||||
|
|
||||||
|
var current_level = std.ArrayList([32]u8).empty;
|
||||||
|
defer current_level.deinit(self.allocator);
|
||||||
|
try current_level.appendSlice(self.allocator, self.leaves.items);
|
||||||
|
|
||||||
|
var current_index = index;
|
||||||
|
while (current_level.items.len > 1) {
|
||||||
|
const sibling_index = if (current_index % 2 == 0)
|
||||||
|
@min(current_index + 1, current_level.items.len - 1)
|
||||||
|
else
|
||||||
|
current_index - 1;
|
||||||
|
|
||||||
|
try proof.append(self.allocator, current_level.items[sibling_index]);
|
||||||
|
|
||||||
|
var next_level = std.ArrayList([32]u8).empty;
|
||||||
|
var i: usize = 0;
|
||||||
|
while (i < current_level.items.len) : (i += 2) {
|
||||||
|
const left = current_level.items[i];
|
||||||
|
const right = if (i + 1 < current_level.items.len) current_level.items[i + 1] else left;
|
||||||
|
|
||||||
|
var hasher = std.crypto.hash.Blake3.init(.{});
|
||||||
|
hasher.update(&left);
|
||||||
|
hasher.update(&right);
|
||||||
|
var out: [32]u8 = undefined;
|
||||||
|
hasher.final(&out);
|
||||||
|
try next_level.append(self.allocator, out);
|
||||||
|
}
|
||||||
|
current_level.deinit(self.allocator);
|
||||||
|
current_level = next_level;
|
||||||
|
current_index /= 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
return proof.toOwnedSlice(self.allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn verify(root: [32]u8, leaf: [32]u8, index: usize, proof: [][32]u8) bool {
|
||||||
|
var current_hash = leaf;
|
||||||
|
var current_index = index;
|
||||||
|
for (proof) |sibling| {
|
||||||
|
var hasher = std.crypto.hash.Blake3.init(.{});
|
||||||
|
if (current_index % 2 == 0) {
|
||||||
|
hasher.update(¤t_hash);
|
||||||
|
hasher.update(&sibling);
|
||||||
|
} else {
|
||||||
|
hasher.update(&sibling);
|
||||||
|
hasher.update(¤t_hash);
|
||||||
|
}
|
||||||
|
hasher.final(¤t_hash);
|
||||||
|
current_index /= 2;
|
||||||
|
}
|
||||||
|
return std.mem.eql(u8, ¤t_hash, &root);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
test "MerkleTree: root and proof" {
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
var tree = MerkleTree.init(allocator);
|
||||||
|
defer tree.deinit();
|
||||||
|
|
||||||
|
const h1 = [_]u8{1} ** 32;
|
||||||
|
const h2 = [_]u8{2} ** 32;
|
||||||
|
const h3 = [_]u8{3} ** 32;
|
||||||
|
|
||||||
|
try tree.insert(h1);
|
||||||
|
try tree.insert(h2);
|
||||||
|
try tree.insert(h3);
|
||||||
|
|
||||||
|
const root = tree.getRoot();
|
||||||
|
|
||||||
|
// Manual verification of root:
|
||||||
|
// next1 = Blake3(h1, h2)
|
||||||
|
// next2 = Blake3(h3, h3)
|
||||||
|
// root = Blake3(next1, next2)
|
||||||
|
|
||||||
|
const proof = try tree.getProof(0); // Proof for h1
|
||||||
|
defer allocator.free(proof);
|
||||||
|
|
||||||
|
try std.testing.expect(MerkleTree.verify(root, h1, 0, proof));
|
||||||
|
try std.testing.expect(!MerkleTree.verify(root, h2, 0, proof));
|
||||||
|
}
|
||||||
|
|
@ -11,12 +11,20 @@ pub const Persona = enum {
|
||||||
gateway,
|
gateway,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub const TrustCategory = enum(u8) {
|
||||||
|
peer = 0, // Everyone else
|
||||||
|
friend = 1, // In my trust graph
|
||||||
|
infrastructure = 2, // Known relays, bootstrap nodes
|
||||||
|
};
|
||||||
|
|
||||||
pub const Policy = struct {
|
pub const Policy = struct {
|
||||||
persona: Persona,
|
persona: Persona,
|
||||||
max_retention_seconds: i64,
|
max_retention_seconds: i64,
|
||||||
max_storage_bytes: u64,
|
max_storage_bytes: u64,
|
||||||
segment_size: usize,
|
segment_size: usize,
|
||||||
|
|
||||||
|
pub const EVICTION_ORDERING = [_]TrustCategory{ .peer, .friend, .infrastructure };
|
||||||
|
|
||||||
pub fn init(persona: Persona) Policy {
|
pub fn init(persona: Persona) Policy {
|
||||||
return switch (persona) {
|
return switch (persona) {
|
||||||
.client => Policy{
|
.client => Policy{
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,138 @@
|
||||||
|
//! RFC-0120: Reorder Buffer for Deterministic L1 Ingestion
|
||||||
|
//!
|
||||||
|
//! Hand-crafted for maximum efficiency. Ensures that frames are yielded
|
||||||
|
//! to the L1 state machine in a contiguous, monotonic sequence per sender.
|
||||||
|
//! This prevents out-of-order execution which is the primary source of
|
||||||
|
//! race conditions in distributed state machines.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
const manifest = @import("manifest.zig");
|
||||||
|
|
||||||
|
pub const ReorderBuffer = struct {
|
||||||
|
allocator: std.mem.Allocator,
|
||||||
|
next_expected_seq: std.AutoHashMapUnmanaged([24]u8, u32),
|
||||||
|
pending_frames: std.AutoHashMapUnmanaged([24]u8, std.ArrayListUnmanaged(manifest.PacketSummary)),
|
||||||
|
|
||||||
|
pub fn init(allocator: std.mem.Allocator) ReorderBuffer {
|
||||||
|
return .{
|
||||||
|
.allocator = allocator,
|
||||||
|
.next_expected_seq = .{},
|
||||||
|
.pending_frames = .{},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deinit(self: *ReorderBuffer) void {
|
||||||
|
self.next_expected_seq.deinit(self.allocator);
|
||||||
|
var it = self.pending_frames.iterator();
|
||||||
|
while (it.next()) |entry| {
|
||||||
|
entry.value_ptr.deinit(self.allocator);
|
||||||
|
}
|
||||||
|
self.pending_frames.deinit(self.allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add a frame to the buffer.
|
||||||
|
/// Returns a slice of frames that are now ready to be processed in order.
|
||||||
|
/// The caller owns the returned slice.
|
||||||
|
pub fn push(self: *ReorderBuffer, summary: manifest.PacketSummary) ![]manifest.PacketSummary {
|
||||||
|
const sender = summary.sender_hint;
|
||||||
|
const seq = summary.sequence;
|
||||||
|
|
||||||
|
const entry = try self.next_expected_seq.getOrPut(self.allocator, sender);
|
||||||
|
if (!entry.found_existing) entry.value_ptr.* = 0;
|
||||||
|
const next = entry.value_ptr.*;
|
||||||
|
|
||||||
|
if (seq < next) {
|
||||||
|
// Already processed or old, drop it
|
||||||
|
return &[_]manifest.PacketSummary{};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (seq == next) {
|
||||||
|
// Perfect fit! Let's see if we can drain any pending ones too.
|
||||||
|
var ready = std.ArrayList(manifest.PacketSummary).empty;
|
||||||
|
errdefer ready.deinit(self.allocator);
|
||||||
|
|
||||||
|
try ready.append(self.allocator, summary);
|
||||||
|
entry.value_ptr.* += 1;
|
||||||
|
|
||||||
|
// Check if we have the next ones in the pending list
|
||||||
|
if (self.pending_frames.getPtr(sender)) |pending| {
|
||||||
|
while (true) {
|
||||||
|
var found = false;
|
||||||
|
var i: usize = 0;
|
||||||
|
while (i < pending.items.len) {
|
||||||
|
if (pending.items[i].sequence == entry.value_ptr.*) {
|
||||||
|
try ready.append(self.allocator, pending.swapRemove(i));
|
||||||
|
entry.value_ptr.* += 1;
|
||||||
|
found = true;
|
||||||
|
// Reset search since we modified the list
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
if (!found) break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ready.toOwnedSlice(self.allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ahead of sequence, buffer it
|
||||||
|
const pending_entry = try self.pending_frames.getOrPut(self.allocator, sender);
|
||||||
|
if (!pending_entry.found_existing) pending_entry.value_ptr.* = .{};
|
||||||
|
try pending_entry.value_ptr.append(self.allocator, summary);
|
||||||
|
|
||||||
|
return &[_]manifest.PacketSummary{};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Force yield everything for a sender (e.g. on timeout or disconnect)
|
||||||
|
pub fn forceFlush(self: *ReorderBuffer, sender: [24]u8) ![]manifest.PacketSummary {
|
||||||
|
if (self.pending_frames.getPtr(sender)) |pending| {
|
||||||
|
// Sort them first to ensure deterministic yield even in flush
|
||||||
|
const items = pending.items;
|
||||||
|
std.sort.pdq(manifest.PacketSummary, items, {}, compareBySeq);
|
||||||
|
|
||||||
|
const result = try self.allocator.dupe(manifest.PacketSummary, items);
|
||||||
|
pending.clearRetainingCapacity();
|
||||||
|
|
||||||
|
// Update next expected to avoid replaying these
|
||||||
|
if (result.len > 0) {
|
||||||
|
if (self.next_expected_seq.getPtr(sender)) |next| {
|
||||||
|
next.* = result[result.len - 1].sequence + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return &[_]manifest.PacketSummary{};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
fn compareBySeq(_: void, a: manifest.PacketSummary, b: manifest.PacketSummary) bool {
|
||||||
|
return a.sequence < b.sequence;
|
||||||
|
}
|
||||||
|
|
||||||
|
test "ReorderBuffer: contiguous flow" {
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
var rb = ReorderBuffer.init(allocator);
|
||||||
|
defer rb.deinit();
|
||||||
|
|
||||||
|
const sender = [_]u8{0xC} ** 24;
|
||||||
|
|
||||||
|
// Push 0 -> Ready [0]
|
||||||
|
const r1 = try rb.push(.{ .queue_id = [_]u8{0} ** 16, .sender_hint = sender, .size = 0, .priority = .normal, .created_at = 0, .timestamp = 0, .sequence = 0, .expires_at = 0, .entropy_cost = 0, .category = .peer });
|
||||||
|
defer allocator.free(r1);
|
||||||
|
try std.testing.expectEqual(r1.len, 1);
|
||||||
|
try std.testing.expectEqual(r1[0].sequence, 0);
|
||||||
|
|
||||||
|
// Push 2 -> Buffered
|
||||||
|
const r2 = try rb.push(.{ .queue_id = [_]u8{0} ** 16, .sender_hint = sender, .size = 0, .priority = .normal, .created_at = 0, .timestamp = 0, .sequence = 2, .expires_at = 0, .entropy_cost = 0, .category = .peer });
|
||||||
|
defer allocator.free(r2);
|
||||||
|
try std.testing.expectEqual(r2.len, 0);
|
||||||
|
|
||||||
|
// Push 1 -> Ready [1, 2]
|
||||||
|
const r3 = try rb.push(.{ .queue_id = [_]u8{0} ** 16, .sender_hint = sender, .size = 0, .priority = .normal, .created_at = 0, .timestamp = 0, .sequence = 1, .expires_at = 0, .entropy_cost = 0, .category = .peer });
|
||||||
|
defer allocator.free(r3);
|
||||||
|
try std.testing.expectEqual(r3.len, 2);
|
||||||
|
try std.testing.expectEqual(r3[0].sequence, 1);
|
||||||
|
try std.testing.expectEqual(r3[1].sequence, 2);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,130 @@
|
||||||
|
//! RFC-0020 & RFC-0120: Deterministic Frame Sequencer
|
||||||
|
//!
|
||||||
|
//! Ensures that frames are presented to L1 in a stable, deterministic order
|
||||||
|
//! regardless of network arrival order. This is critical for state machine
|
||||||
|
//! consistency and preventing race conditions in the Reality Tunnel.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
const manifest = @import("manifest.zig");
|
||||||
|
|
||||||
|
/// Sorts a slice of PacketSummaries deterministically.
|
||||||
|
/// Ordering Doctrine:
|
||||||
|
/// 1. Primary: Source Hint (truncated DID) - groups frames by sender
|
||||||
|
/// 2. Secondary: Timestamp (L0 nanoseconds) - causal ordering
|
||||||
|
/// 3. Tertiary: Sequence (Tie-breaker for same-nanosecond frames)
|
||||||
|
/// 4. Quaternary: QueueID (Force stability for identical metadata)
|
||||||
|
pub fn sortDeterministically(items: []manifest.PacketSummary) void {
|
||||||
|
std.sort.pdq(manifest.PacketSummary, items, {}, comparePackets);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn comparePackets(_: void, a: manifest.PacketSummary, b: manifest.PacketSummary) bool {
|
||||||
|
// 1. Source Hint
|
||||||
|
const hint_cmp = std.mem.order(u8, &a.sender_hint, &b.sender_hint);
|
||||||
|
if (hint_cmp != .eq) return hint_cmp == .lt;
|
||||||
|
|
||||||
|
// 2. Timestamp
|
||||||
|
if (a.timestamp != b.timestamp) return a.timestamp < b.timestamp;
|
||||||
|
|
||||||
|
// 3. Sequence
|
||||||
|
if (a.sequence != b.sequence) return a.sequence < b.sequence;
|
||||||
|
|
||||||
|
// 4. Queue ID (Total Stability)
|
||||||
|
return std.mem.order(u8, &a.queue_id, &b.queue_id) == .lt;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub const ReplayFilter = struct {
|
||||||
|
last_sequences: std.AutoHashMapUnmanaged([24]u8, u32),
|
||||||
|
allocator: std.mem.Allocator,
|
||||||
|
|
||||||
|
pub fn init(allocator: std.mem.Allocator) ReplayFilter {
|
||||||
|
return .{
|
||||||
|
.last_sequences = .{},
|
||||||
|
.allocator = allocator,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deinit(self: *ReplayFilter) void {
|
||||||
|
self.last_sequences.deinit(self.allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Checks if a packet is a replay.
|
||||||
|
/// Returns true if the packet should be processed (new sequence),
|
||||||
|
/// false if it should be dropped (replay or old).
|
||||||
|
pub fn isNew(self: *ReplayFilter, sender: [24]u8, sequence: u32) !bool {
|
||||||
|
const entry = try self.last_sequences.getOrPut(self.allocator, sender);
|
||||||
|
if (!entry.found_existing) {
|
||||||
|
entry.value_ptr.* = sequence;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sequence > entry.value_ptr.*) {
|
||||||
|
entry.value_ptr.* = sequence;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return false; // Replay or old packet
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
test "Deterministic Sorting" {
|
||||||
|
var summaries = [_]manifest.PacketSummary{
|
||||||
|
.{
|
||||||
|
.queue_id = [_]u8{3} ** 16,
|
||||||
|
.sender_hint = [_]u8{0xA} ** 24,
|
||||||
|
.size = 100,
|
||||||
|
.priority = .normal,
|
||||||
|
.created_at = 0,
|
||||||
|
.timestamp = 200,
|
||||||
|
.sequence = 2,
|
||||||
|
.expires_at = 0,
|
||||||
|
.entropy_cost = 0,
|
||||||
|
.category = .peer,
|
||||||
|
},
|
||||||
|
.{
|
||||||
|
.queue_id = [_]u8{1} ** 16,
|
||||||
|
.sender_hint = [_]u8{0xA} ** 24,
|
||||||
|
.size = 100,
|
||||||
|
.priority = .normal,
|
||||||
|
.created_at = 0,
|
||||||
|
.timestamp = 100,
|
||||||
|
.sequence = 1,
|
||||||
|
.expires_at = 0,
|
||||||
|
.entropy_cost = 0,
|
||||||
|
.category = .peer,
|
||||||
|
},
|
||||||
|
.{
|
||||||
|
.queue_id = [_]u8{2} ** 16,
|
||||||
|
.sender_hint = [_]u8{0xB} ** 24,
|
||||||
|
.size = 100,
|
||||||
|
.priority = .normal,
|
||||||
|
.created_at = 0,
|
||||||
|
.timestamp = 50,
|
||||||
|
.sequence = 5,
|
||||||
|
.expires_at = 0,
|
||||||
|
.entropy_cost = 0,
|
||||||
|
.category = .peer,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
sortDeterministically(&summaries);
|
||||||
|
|
||||||
|
// Source A comes before Source B
|
||||||
|
// Within Source A, timestamp 100 comes before 200
|
||||||
|
try std.testing.expectEqual(summaries[0].sender_hint[0], 0xA);
|
||||||
|
try std.testing.expectEqual(summaries[0].timestamp, 100);
|
||||||
|
try std.testing.expectEqual(summaries[1].timestamp, 200);
|
||||||
|
try std.testing.expectEqual(summaries[2].sender_hint[0], 0xB);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "Replay Filter" {
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
var filter = ReplayFilter.init(allocator);
|
||||||
|
defer filter.deinit();
|
||||||
|
|
||||||
|
const sender = [_]u8{0xC} ** 24;
|
||||||
|
|
||||||
|
try std.testing.expect(try filter.isNew(sender, 10));
|
||||||
|
try std.testing.expect(try filter.isNew(sender, 11));
|
||||||
|
try std.testing.expect(!(try filter.isNew(sender, 10))); // Replay
|
||||||
|
try std.testing.expect(!(try filter.isNew(sender, 5))); // Older
|
||||||
|
}
|
||||||
|
|
@ -27,6 +27,13 @@ pub const SegmentHeader = struct {
|
||||||
pub const SIZE = 4 + 1 + 3 + 8 + 4 + 8; // 28 bytes
|
pub const SIZE = 4 + 1 + 3 + 8 + 4 + 8; // 28 bytes
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub const WALLocation = struct {
|
||||||
|
segment_id: u64,
|
||||||
|
segment_seq: u32,
|
||||||
|
offset: usize,
|
||||||
|
len: usize,
|
||||||
|
};
|
||||||
|
|
||||||
pub const WALStore = struct {
|
pub const WALStore = struct {
|
||||||
allocator: std.mem.Allocator,
|
allocator: std.mem.Allocator,
|
||||||
base_dir_path: []const u8,
|
base_dir_path: []const u8,
|
||||||
|
|
@ -58,7 +65,7 @@ pub const WALStore = struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Append a frame to the active segment
|
/// Append a frame to the active segment
|
||||||
pub fn appendFrame(self: *WALStore, frame: *const lwf.LWFFrame) !void {
|
pub fn appendFrame(self: *WALStore, frame: *const lwf.LWFFrame) !WALLocation {
|
||||||
const frame_size = frame.header.payload_len + lwf.LWFHeader.SIZE + lwf.LWFTrailer.SIZE;
|
const frame_size = frame.header.payload_len + lwf.LWFHeader.SIZE + lwf.LWFTrailer.SIZE;
|
||||||
|
|
||||||
// Check if we need a new segment
|
// Check if we need a new segment
|
||||||
|
|
@ -70,8 +77,16 @@ pub const WALStore = struct {
|
||||||
const encoded = try frame.encode(self.allocator);
|
const encoded = try frame.encode(self.allocator);
|
||||||
defer self.allocator.free(encoded);
|
defer self.allocator.free(encoded);
|
||||||
|
|
||||||
|
const loc = WALLocation{
|
||||||
|
.segment_id = self.active_segment_id,
|
||||||
|
.segment_seq = self.active_segment_seq,
|
||||||
|
.offset = self.current_offset,
|
||||||
|
.len = encoded.len,
|
||||||
|
};
|
||||||
|
|
||||||
try file.writeAll(encoded);
|
try file.writeAll(encoded);
|
||||||
self.current_offset += encoded.len;
|
self.current_offset += encoded.len;
|
||||||
|
return loc;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn rotateSegment(self: *WALStore) !void {
|
fn rotateSegment(self: *WALStore) !void {
|
||||||
|
|
@ -258,7 +273,7 @@ test "OPQ WAL Store: Append and Rotate" {
|
||||||
// 1024 / 208 ≈ 4 frames per segment (plus header)
|
// 1024 / 208 ≈ 4 frames per segment (plus header)
|
||||||
var i: usize = 0;
|
var i: usize = 0;
|
||||||
while (i < 10) : (i += 1) {
|
while (i < 10) : (i += 1) {
|
||||||
try wal.appendFrame(&frame);
|
_ = try wal.appendFrame(&frame);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Verify files created
|
// 3. Verify files created
|
||||||
|
|
@ -288,7 +303,7 @@ test "OPQ WAL Store: Pruning" {
|
||||||
|
|
||||||
var frame = try lwf.LWFFrame.init(allocator, 10);
|
var frame = try lwf.LWFFrame.init(allocator, 10);
|
||||||
defer frame.deinit(allocator);
|
defer frame.deinit(allocator);
|
||||||
try wal.appendFrame(&frame);
|
_ = try wal.appendFrame(&frame);
|
||||||
|
|
||||||
// Manually finalize and wait 2 seconds (for test purposes we could mock time,
|
// Manually finalize and wait 2 seconds (for test purposes we could mock time,
|
||||||
// but here we'll just test the logic with a very small TTL)
|
// but here we'll just test the logic with a very small TTL)
|
||||||
|
|
@ -318,7 +333,7 @@ test "OPQ WAL Store: Space-based Pruning" {
|
||||||
// Append 4 frames (should create multiple segments)
|
// Append 4 frames (should create multiple segments)
|
||||||
var i: usize = 0;
|
var i: usize = 0;
|
||||||
while (i < 4) : (i += 1) {
|
while (i < 4) : (i += 1) {
|
||||||
try wal.appendFrame(&frame);
|
_ = try wal.appendFrame(&frame);
|
||||||
}
|
}
|
||||||
|
|
||||||
const usage_before = try wal.getDiskUsage();
|
const usage_before = try wal.getDiskUsage();
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
//! RFC-0120: L0 Trust Resolver Interface
|
||||||
|
//!
|
||||||
|
//! Provides the mechanism for L1 to inject trust data into the transport layer
|
||||||
|
//! for prioritized resource allocation.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
const quota = @import("quota.zig");
|
||||||
|
|
||||||
|
pub const TrustResolver = struct {
|
||||||
|
context: ?*anyopaque,
|
||||||
|
resolve_fn: *const fn (ctx: ?*anyopaque, hint: [24]u8) quota.TrustCategory,
|
||||||
|
|
||||||
|
/// Resolve a DID hint to a trust category.
|
||||||
|
/// L0 is intentionally dumb; it just calls this function.
|
||||||
|
pub fn resolve(self: TrustResolver, hint: [24]u8) quota.TrustCategory {
|
||||||
|
return self.resolve_fn(self.context, hint);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Default resolver: everything is a peer.
|
||||||
|
pub fn noop() TrustResolver {
|
||||||
|
return .{
|
||||||
|
.context = null,
|
||||||
|
.resolve_fn = struct {
|
||||||
|
fn func(_: ?*anyopaque, _: [24]u8) quota.TrustCategory {
|
||||||
|
return .peer;
|
||||||
|
}
|
||||||
|
}.func,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -13,11 +13,11 @@ pub const L0Service = struct {
|
||||||
opq_manager: opq.OPQManager,
|
opq_manager: opq.OPQManager,
|
||||||
|
|
||||||
/// Initialize the L0 service with a bound socket and storage
|
/// Initialize the L0 service with a bound socket and storage
|
||||||
pub fn init(allocator: std.mem.Allocator, address: std.net.Address, base_dir: []const u8, persona: opq.Persona) !L0Service {
|
pub fn init(allocator: std.mem.Allocator, address: std.net.Address, base_dir: []const u8, persona: opq.Persona, resolver: opq.trust_resolver.TrustResolver) !L0Service {
|
||||||
return L0Service{
|
return L0Service{
|
||||||
.allocator = allocator,
|
.allocator = allocator,
|
||||||
.socket = try utcp.UTCP.init(address),
|
.socket = try utcp.UTCP.init(address),
|
||||||
.opq_manager = try opq.OPQManager.init(allocator, base_dir, persona),
|
.opq_manager = try opq.OPQManager.init(allocator, base_dir, persona, resolver),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -59,7 +59,7 @@ test "L0 Integrated Service: Loopback Ingestion" {
|
||||||
const addr = try std.net.Address.parseIp("127.0.0.1", 0);
|
const addr = try std.net.Address.parseIp("127.0.0.1", 0);
|
||||||
|
|
||||||
// 1. Start Service (Relay persona)
|
// 1. Start Service (Relay persona)
|
||||||
var service = try L0Service.init(allocator, addr, test_dir, .relay);
|
var service = try L0Service.init(allocator, addr, test_dir, .relay, opq.trust_resolver.TrustResolver.noop());
|
||||||
defer service.deinit();
|
defer service.deinit();
|
||||||
|
|
||||||
const service_addr = try service.socket.getLocalAddress();
|
const service_addr = try service.socket.getLocalAddress();
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue