implement durable pkvdb engine

.gitignore

@@ -1,8 +1,12 @@
 main
+pizzakv
+.zig-cache/
+*.pkvdb
+.pizzakv.sock
 .claude
 tools/
 .db
 .dbb
 test_nov.js
 bin/
-.DS_Store
+.DS_Store

README.md

@@ -65,7 +65,15 @@ make clean
 # Start server in Pizzaria mode (\r-delimited protocol, port 8085)
 ./pizzakv
 
-# The server will create a .db file for persistence
+# Custom port
+./pizzakv -port=9000
+
+# Unix socket mode (creates .pizzakv.sock in current directory)
+./pizzakv -unix
+./pizzakv -unix -redis
+
+# The server will create a .db file for persistence.
+# In unix mode, a .pizzakv.sock file is also created and removed on shutdown.
 ```
 
 ## Benchmarking
@@ -158,7 +166,7 @@ pizzakv/
 ├── hashing.zig        # Hash function
 ├── redis.zig          # RESP protocol parser
 ├── command.zig        # Command execution
-├── socket.zig         # TCP socket operations
+├── socket.zig         # TCP and Unix socket operations
 └── benchmark_*.sh     # Benchmark scripts
 ```
 

benchmark.zig

@@ -0,0 +1,158 @@
+const std = @import("std");
+const builtin = @import("builtin");
+const engine_mod = @import("engine.zig");
+const redis = @import("redis.zig");
+const pkbfi = @import("pkbfi.zig");
+
+const allocator = std.heap.smp_allocator;
+
+fn elapsed(start: i128) u64 {
+    return @intCast(@max(@as(i128, 1), std.time.nanoTimestamp() - start));
+}
+
+fn rate(operations: u64, nanoseconds: u64) u64 {
+    return @intCast((@as(u128, operations) * std.time.ns_per_s) / nanoseconds);
+}
+
+fn rss() u64 {
+    const value = std.posix.getrusage(0).maxrss;
+    return switch (builtin.target.os.tag) {
+        .linux => @as(u64, @intCast(value)) * 1024,
+        else => @intCast(value),
+    };
+}
+
+fn percentile(values: []u64, numerator: usize, denominator: usize) u64 {
+    std.sort.heap(u64, values, {}, std.sort.asc(u64));
+    return values[@min(values.len - 1, values.len * numerator / denominator)];
+}
+
+pub fn main() !void {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const directory = try tmp.dir.realpath(".", &path_buffer);
+    const path = try std.fs.path.join(allocator, &.{ directory, "benchmark.pkvdb" });
+    defer allocator.free(path);
+    var engine = try engine_mod.Engine.open(allocator, path);
+    var value: [128]u8 = [_]u8{'v'} ** 128;
+    var latencies: [1000]u64 = undefined;
+    const initial_file = engine.status().file_bytes;
+    var started = std.time.nanoTimestamp();
+    for (0..1000) |index| {
+        var key_buffer: [32]u8 = undefined;
+        const key = try std.fmt.bufPrint(&key_buffer, "bench/{d:0>8}", .{index});
+        const before = std.time.nanoTimestamp();
+        _ = try engine.put(key, &value);
+        latencies[index] = elapsed(before);
+    }
+    const put_ns = elapsed(started);
+    const put_p95 = percentile(&latencies, 95, 100);
+    started = std.time.nanoTimestamp();
+    for (0..10000) |index| {
+        var key_buffer: [32]u8 = undefined;
+        const key = try std.fmt.bufPrint(&key_buffer, "bench/{d:0>8}", .{index % 1000});
+        const before = std.time.nanoTimestamp();
+        const result = (try engine.get(allocator, key)).?;
+        if (index < latencies.len) latencies[index] = elapsed(before);
+        allocator.free(result.bytes);
+    }
+    const get_ns = elapsed(started);
+    const get_p95 = percentile(&latencies, 95, 100);
+    const redis_get = "*2\r\n$3\r\nGET\r\n$14\r\nbench/00000001\r\n";
+    var pipeline = std.ArrayListUnmanaged(u8){};
+    defer pipeline.deinit(allocator);
+    for (0..64) |_| try pipeline.appendSlice(allocator, redis_get);
+    started = std.time.nanoTimestamp();
+    var redis_operations: u64 = 0;
+    for (0..100) |_| {
+        var position: usize = 0;
+        while (position < pipeline.items.len) {
+            const parsed = try redis.parse(pipeline.items[position..]);
+            var response = try redis.execute(&engine, allocator, parsed.command);
+            response.deinit(allocator);
+            position += parsed.consumed;
+            redis_operations += 1;
+        }
+    }
+    const redis_ns = elapsed(started);
+    var session = pkbfi.Session.init(allocator);
+    defer session.deinit();
+    var get_payload: [18]u8 = undefined;
+    std.mem.writeInt(u32, get_payload[0..4], 14, .little);
+    @memcpy(get_payload[4..], "bench/00000001");
+    started = std.time.nanoTimestamp();
+    for (0..5000) |index| {
+        const response = try session.execute(&engine, .{ .opcode = .get, .flags = 0, .request_id = index, .payload = &get_payload, .consumed = 0 });
+        allocator.free(response);
+    }
+    const pkbfi_point_ns = elapsed(started);
+    var batch_payload = std.ArrayListUnmanaged(u8){};
+    defer batch_payload.deinit(allocator);
+    try batch_payload.appendNTimes(allocator, 0, 8);
+    std.mem.writeInt(u32, batch_payload.items[0..4], 10, .little);
+    for (0..10) |index| {
+        var key_buffer: [32]u8 = undefined;
+        const key = try std.fmt.bufPrint(&key_buffer, "batch/{d:0>8}", .{index});
+        const position = batch_payload.items.len;
+        try batch_payload.appendNTimes(allocator, 0, 12);
+        batch_payload.items[position] = 1;
+        std.mem.writeInt(u32, batch_payload.items[position + 4 ..][0..4], @intCast(key.len), .little);
+        std.mem.writeInt(u32, batch_payload.items[position + 8 ..][0..4], 32, .little);
+        try batch_payload.appendSlice(allocator, key);
+        try batch_payload.appendSlice(allocator, value[0..32]);
+    }
+    started = std.time.nanoTimestamp();
+    for (0..100) |index| {
+        const response = try session.execute(&engine, .{ .opcode = .batch_write, .flags = 0, .request_id = index, .payload = batch_payload.items, .consumed = 0 });
+        allocator.free(response);
+    }
+    const pkbfi_batch_ns = elapsed(started);
+    started = std.time.nanoTimestamp();
+    var cursor: []u8 = try allocator.alloc(u8, 0);
+    var scanned: u64 = 0;
+    while (true) {
+        var batch = try engine.scan(allocator, "bench/", cursor, 128, false, 1024 * 1024);
+        allocator.free(cursor);
+        cursor = try allocator.dupe(u8, batch.next_cursor);
+        scanned += batch.entries.len;
+        const done = batch.done;
+        batch.deinit(allocator);
+        if (done) break;
+    }
+    allocator.free(cursor);
+    const scan_ns = elapsed(started);
+    started = std.time.nanoTimestamp();
+    try engine.checkpoint();
+    const checkpoint_ns = elapsed(started);
+    const before_overwrite_rss = rss();
+    for (0..500) |_| _ = try engine.put("overwrite", &value);
+    const overwrite_rss = rss();
+    const before_churn_rss = rss();
+    for (0..250) |index| {
+        var key_buffer: [32]u8 = undefined;
+        const key = try std.fmt.bufPrint(&key_buffer, "churn/{d}", .{index});
+        _ = try engine.put(key, &value);
+        _ = try engine.delete(key);
+    }
+    const churn_rss = rss();
+    const before_recovery = engine.status();
+    engine.close();
+    started = std.time.nanoTimestamp();
+    engine = try engine_mod.Engine.open(allocator, path);
+    const recovery_ns = elapsed(started);
+    defer engine.close();
+    const status = engine.status();
+    std.debug.print("point_put ops_s={d} p95_ns={d}\n", .{ rate(1000, put_ns), put_p95 });
+    std.debug.print("point_get ops_s={d} p95_ns={d}\n", .{ rate(10000, get_ns), get_p95 });
+    std.debug.print("resp_pipeline ops_s={d} pipeline=64\n", .{rate(redis_operations, redis_ns)});
+    std.debug.print("pkbfi_get ops_s={d}\n", .{rate(5000, pkbfi_point_ns)});
+    std.debug.print("pkbfi_batch transactions_s={d} operations_s={d}\n", .{ rate(100, pkbfi_batch_ns), rate(1000, pkbfi_batch_ns) });
+    std.debug.print("prefix_scan keys_s={d} keys={d}\n", .{ rate(scanned, scan_ns), scanned });
+    std.debug.print("checkpoint ns={d}\n", .{checkpoint_ns});
+    std.debug.print("recovery ns={d} keys={d}\n", .{ recovery_ns, status.live_keys });
+    std.debug.print("rss_overwrite before={d} after={d} delta={d}\n", .{ before_overwrite_rss, overwrite_rss, overwrite_rss -| before_overwrite_rss });
+    std.debug.print("rss_churn before={d} after={d} delta={d}\n", .{ before_churn_rss, churn_rss, churn_rss -| before_churn_rss });
+    std.debug.print("directory bytes_per_live_key={d:.2} bytes={d}\n", .{ @as(f64, @floatFromInt(status.keydir_bytes + status.ordered_index_bytes)) / @as(f64, @floatFromInt(status.live_keys)), status.keydir_bytes + status.ordered_index_bytes });
+    std.debug.print("file_growth bytes={d} bytes_per_put={d:.2} total={d}\n", .{ before_recovery.file_bytes - initial_file, @as(f64, @floatFromInt(before_recovery.file_bytes - initial_file)) / 2600.0, status.file_bytes });
+}

command.zig

@@ -1,145 +1,80 @@
 const std = @import("std");
-const storage = @import("storage.zig");
-const index = @import("index.zig");
-
-const FAILURE_RESPONSE = "error";
-const SUCCESS_RESPONSE = "success";
-
-const Command = enum {
-    read,
-    write,
-    delete,
-    status,
-    keys,
-    reads,
-};
-
-fn parseKeyValue(buf: []const u8) ?[2][]const u8 {
-    var kvIterator = std.mem.splitAny(u8, buf, "|");
-    const key = kvIterator.first();
-    return [2][]const u8{ key, kvIterator.rest() };
-}
-
-pub fn parse(msg: []const u8, allocator: std.mem.Allocator) ?[]const u8 {
-    const trimSet = [_]u8{ '\n', ' ', '\r' };
-    const cleanMsg = std.mem.trim(u8, msg, &trimSet);
-    var messageIterator = std.mem.splitAny(u8, cleanMsg, " ");
-
-    const cmdString = messageIterator.first();
-    const cmd = std.meta.stringToEnum(Command, cmdString) orelse {
-        return null;
-    };
-
-    switch (cmd) {
-        .read => {
-            const key = messageIterator.rest();
-
-            const value = storage.readAlloc(key, allocator) orelse {
-                return FAILURE_RESPONSE;
-            };
-
-            return value;
-        },
-        .write => {
-            const kvPair = messageIterator.rest();
-
-            const kv = parseKeyValue(kvPair) orelse {
-                return FAILURE_RESPONSE;
-            };
-
-            if (storage.write(kv[0], kv[1])) {
-                return SUCCESS_RESPONSE;
-            }
-
-            return FAILURE_RESPONSE;
-        },
-        .delete => {
-            const key = messageIterator.rest();
-            if (!storage.delete(key)) {
-                return FAILURE_RESPONSE;
-            }
-
-            return SUCCESS_RESPONSE;
-        },
-        .keys => {
-            return index.getAllKeys(allocator);
-        },
-        .reads => {
-            const prefix = messageIterator.rest();
-            return index.getValuesByPrefix(prefix, allocator);
-        },
-        .status => {
-            return "well going our operation";
-        },
+const Engine = @import("engine.zig").Engine;
+
+const max_text_response = 1024 * 1024;
+
+pub fn execute(engine: *Engine, allocator: std.mem.Allocator, message: []const u8) ![]u8 {
+    const clean = std.mem.trim(u8, message, "\r\n ");
+    const split = std.mem.indexOfScalar(u8, clean, ' ');
+    const name = if (split) |index| clean[0..index] else clean;
+    const arguments = if (split) |index| clean[index + 1 ..] else "";
+    if (std.mem.eql(u8, name, "read")) {
+        const value = try engine.get(allocator, arguments) orelse return allocator.dupe(u8, "error");
+        return value.bytes;
     }
-
-    return null;
-}
-
-// -- Tests --
-
-const test_allocator = std.heap.page_allocator;
-
-test "parse write command" {
-    storage.init();
-    const result = parse("write mykey|myvalue\r\n", test_allocator) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("success", result);
-}
-
-test "parse read command" {
-    storage.init();
-    // Write first, then read
-    _ = parse("write cmd_rk|cmd_rv", test_allocator);
-    const result = parse("read cmd_rk", test_allocator) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("cmd_rv", result);
-}
-
-test "parse read nonexistent" {
-    storage.init();
-    const result = parse("read cmd_nonexistent_key", test_allocator);
-    try std.testing.expectEqualStrings("error", result.?);
-}
-
-test "parse delete command" {
-    storage.init();
-    _ = parse("write cmd_dk|cmd_dv", test_allocator);
-    const result = parse("delete cmd_dk", test_allocator) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("success", result);
-
-    // Verify deleted
-    const after = parse("read cmd_dk", test_allocator);
-    try std.testing.expectEqualStrings("error", after.?);
-}
-
-test "parse delete nonexistent" {
-    storage.init();
-    const result = parse("delete cmd_nonexistent_del", test_allocator);
-    try std.testing.expectEqualStrings("error", result.?);
-}
-
-test "parse status command" {
-    const result = parse("status", test_allocator) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("well going our operation", result);
-}
-
-test "parse unknown command returns null" {
-    try std.testing.expectEqual(@as(?[]const u8, null), parse("foobar", test_allocator));
-    try std.testing.expectEqual(@as(?[]const u8, null), parse("", test_allocator));
+    if (std.mem.eql(u8, name, "write")) {
+        const separator = std.mem.indexOfScalar(u8, arguments, '|') orelse return allocator.dupe(u8, "error");
+        _ = engine.put(arguments[0..separator], arguments[separator + 1 ..]) catch return allocator.dupe(u8, "error");
+        return allocator.dupe(u8, "success");
+    }
+    if (std.mem.eql(u8, name, "delete")) {
+        const deleted = engine.delete(arguments) catch return allocator.dupe(u8, "error");
+        return allocator.dupe(u8, if (deleted) "success" else "error");
+    }
+    if (std.mem.eql(u8, name, "status")) {
+        const status = engine.status();
+        return std.fmt.allocPrint(allocator, "well going our operation keys={d} latest_lsn={d} checkpoint_lsn={d} file_bytes={d} groups={d} transactions={d} largest_group={d}", .{ status.live_keys, status.latest_lsn, status.checkpoint_lsn, status.file_bytes, status.commit_groups, status.committed_transactions, status.largest_commit_group });
+    }
+    if (std.mem.eql(u8, name, "keys")) return scanText(engine, allocator, "", arguments, false);
+    if (std.mem.eql(u8, name, "reads")) return scanText(engine, allocator, arguments, "", true);
+    if (std.mem.eql(u8, name, "scan")) {
+        var fields = std.mem.splitScalar(u8, arguments, '|');
+        const prefix = fields.next() orelse "";
+        const cursor = fields.next() orelse "";
+        const limit_text = fields.next() orelse "256";
+        const mode = fields.next() orelse "keys";
+        const limit = std.fmt.parseInt(u32, limit_text, 10) catch return allocator.dupe(u8, "error");
+        return scanTextLimit(engine, allocator, prefix, cursor, std.mem.eql(u8, mode, "values"), limit);
+    }
+    if (std.mem.eql(u8, name, "checkpoint")) {
+        engine.checkpoint() catch return allocator.dupe(u8, "error");
+        return allocator.dupe(u8, "success");
+    }
+    return allocator.dupe(u8, "error");
 }
 
-test "parse trims whitespace" {
-    const result = parse("  status \r\n", test_allocator) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("well going our operation", result);
+fn scanText(engine: *Engine, allocator: std.mem.Allocator, prefix: []const u8, cursor: []const u8, values: bool) ![]u8 {
+    return scanTextLimit(engine, allocator, prefix, cursor, values, 256);
 }
 
-test "parseKeyValue splits on pipe" {
-    const kv = parseKeyValue("hello|world") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("hello", kv[0]);
-    try std.testing.expectEqualStrings("world", kv[1]);
+fn scanTextLimit(engine: *Engine, allocator: std.mem.Allocator, prefix: []const u8, cursor: []const u8, values: bool, limit: u32) ![]u8 {
+    var batch = engine.scan(allocator, prefix, cursor, @min(limit, 4096), values, max_text_response) catch return allocator.dupe(u8, "error");
+    defer batch.deinit(allocator);
+    var output = std.ArrayListUnmanaged(u8){};
+    errdefer output.deinit(allocator);
+    for (batch.entries, 0..) |entry, index| {
+        if (index != 0) try output.append(allocator, '\n');
+        try output.appendSlice(allocator, if (values) entry.value.? else entry.key);
+    }
+    return output.toOwnedSlice(allocator);
 }
 
-test "parseKeyValue with multiple pipes" {
-    const kv = parseKeyValue("key|val|ue|extra") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("key", kv[0]);
-    try std.testing.expectEqualStrings("val|ue|extra", kv[1]);
+test "Pizzaria point compatibility and bounded scan" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const directory = try tmp.dir.realpath(".", &path_buffer);
+    const path = try std.fmt.allocPrint(std.testing.allocator, "{s}/protocol.pkvdb", .{directory});
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    var response = try execute(&engine, std.testing.allocator, "write p/1|one\r");
+    try std.testing.expectEqualStrings("success", response);
+    std.testing.allocator.free(response);
+    response = try execute(&engine, std.testing.allocator, "read p/1\r");
+    try std.testing.expectEqualStrings("one", response);
+    std.testing.allocator.free(response);
+    response = try execute(&engine, std.testing.allocator, "scan p/||1|keys\r");
+    defer std.testing.allocator.free(response);
+    try std.testing.expectEqualStrings("p/1", response);
 }

engine.zig

@@ -0,0 +1,1421 @@
+const std = @import("std");
+const pkvdb = @import("pkvdb.zig");
+const keydir = @import("keydir.zig");
+const ordered_index = @import("ordered_index.zig");
+
+pub const RecordRef = keydir.RecordRef;
+const KeyReader = keydir.Reader;
+pub const pkvdb_max_frame = pkvdb.max_transaction_size;
+
+pub const Operation = struct {
+    opcode: pkvdb.Opcode,
+    key: []const u8,
+    value: []const u8 = "",
+};
+
+pub const Value = struct {
+    bytes: []u8,
+    lsn: u64,
+};
+
+pub const ScanEntry = struct {
+    key: []u8,
+    value: ?[]u8,
+    lsn: u64,
+};
+
+pub const ScanBatch = struct {
+    entries: []ScanEntry,
+    next_cursor: []u8,
+    done: bool,
+
+    pub fn deinit(self: *ScanBatch, allocator: std.mem.Allocator) void {
+        for (self.entries) |entry| {
+            allocator.free(entry.key);
+            if (entry.value) |value| allocator.free(value);
+        }
+        allocator.free(self.entries);
+        allocator.free(self.next_cursor);
+        self.* = undefined;
+    }
+};
+
+pub const Status = struct {
+    uuid: [16]u8,
+    file_bytes: u64,
+    latest_lsn: u64,
+    oldest_lsn: u64,
+    checkpoint_lsn: u64,
+    journal_bytes_since_checkpoint: u64,
+    live_keys: u64,
+    keydir_bytes: u64,
+    ordered_index_bytes: u64,
+    bytes_written: u64,
+    checksum_failures: u64,
+    partial_tails: u64,
+    recovery_ns: u64,
+    checkpoint_ns: u64,
+    connection_bytes: u64,
+    active_requests: u64,
+    commit_groups: u64,
+    committed_transactions: u64,
+    largest_commit_group: u64,
+};
+
+const Root = struct {
+    block: pkvdb.Superblock,
+    manifest: ?pkvdb.Manifest,
+};
+
+const PendingWrite = struct {
+    operations: []const Operation,
+    metadata: []const u8,
+    next: ?*PendingWrite = null,
+    completion: *WriteCompletion,
+    lsn: u64 = 0,
+    frame_position: usize = 0,
+    changed: bool = false,
+    prepared_position: usize = 0,
+    prepared_count: usize = 0,
+    bytes: usize,
+};
+
+const WriteCompletion = struct {
+    condition: std.Thread.Condition = .{},
+    remaining: usize,
+    failure: ?anyerror = null,
+};
+
+const max_group_transactions = 4096;
+const max_group_bytes = 64 * 1024 * 1024;
+const max_queued_bytes = 128 * 1024 * 1024;
+const group_wait_ns = 250 * std.time.ns_per_us;
+const map_interval = 64 * 1024 * 1024;
+
+const Mapping = struct {
+    bytes: []align(std.heap.page_size_min) u8,
+    file_start: u64,
+    logical_start: u64,
+    logical_end: u64,
+};
+
+pub const Engine = struct {
+    allocator: std.mem.Allocator,
+    file: std.fs.File,
+    directory: keydir.KeyDir,
+    ordered: ordered_index.OrderedIndex,
+    ordered_ready: bool = false,
+    mappings: std.ArrayListUnmanaged(Mapping) = .{},
+    mapping_lock: std.Thread.RwLock = .{},
+    mapped_length: u64 = 0,
+    lock: std.Thread.RwLock = .{},
+    io_mutex: std.Thread.Mutex = .{},
+    checkpoint_mutex: std.Thread.Mutex = .{},
+    ordered_gate: std.Thread.Mutex = .{},
+    queue_mutex: std.Thread.Mutex = .{},
+    queue_condition: std.Thread.Condition = .{},
+    queue_head: ?*PendingWrite = null,
+    queue_tail: ?*PendingWrite = null,
+    queued_bytes: usize = 0,
+    writer_thread: ?std.Thread = null,
+    writer_stopping: bool = false,
+    writer_failed: bool = false,
+    commit_groups: u64 = 0,
+    committed_transactions: u64 = 0,
+    largest_commit_group: u64 = 0,
+    file_length: u64,
+    latest_lsn: u64 = 0,
+    oldest_lsn: u64 = 0,
+    checkpoint_lsn: u64 = 0,
+    journal_bytes_since_checkpoint: u64 = 0,
+    bytes_written: u64 = 0,
+    checksum_failures: u64 = 0,
+    partial_tails: u64 = 0,
+    recovery_ns: u64 = 0,
+    checkpoint_ns: u64 = 0,
+    connection_bytes: std.atomic.Value(u64) = std.atomic.Value(u64).init(0),
+    active_requests: std.atomic.Value(u64) = std.atomic.Value(u64).init(0),
+    uuid: [16]u8,
+    generation: u64,
+    active_superblock: u1,
+    created_ns: i64,
+
+    pub fn open(allocator: std.mem.Allocator, path: []const u8) !Engine {
+        const started = std.time.nanoTimestamp();
+        const file = std.fs.cwd().openFile(path, .{ .mode = .read_write }) catch |err| switch (err) {
+            error.FileNotFound => try std.fs.cwd().createFile(path, .{ .read = true, .truncate = false }),
+            else => return err,
+        };
+        errdefer file.close();
+        var directory = try keydir.KeyDir.init(allocator);
+        errdefer directory.deinit();
+        var engine = Engine{
+            .allocator = allocator,
+            .file = file,
+            .directory = directory,
+            .ordered = ordered_index.OrderedIndex.init(allocator),
+            .file_length = try file.getEndPos(),
+            .uuid = undefined,
+            .generation = 0,
+            .active_superblock = 0,
+            .created_ns = now(),
+        };
+        errdefer engine.ordered.deinit();
+        if (engine.file_length == 0) {
+            try engine.initialize();
+        } else {
+            try engine.recover();
+        }
+        const elapsed = std.time.nanoTimestamp() - started;
+        engine.recovery_ns = if (elapsed > 0) @intCast(elapsed) else 0;
+        engine.mapTail(true);
+        return engine;
+    }
+
+    pub fn close(self: *Engine) void {
+        self.queue_mutex.lock();
+        self.writer_stopping = true;
+        self.queue_condition.broadcast();
+        self.queue_mutex.unlock();
+        if (self.writer_thread) |thread| thread.join();
+        for (self.mappings.items) |mapping| std.posix.munmap(mapping.bytes);
+        self.mappings.deinit(self.allocator);
+        self.ordered.deinit();
+        self.directory.deinit();
+        self.file.close();
+        self.* = undefined;
+    }
+
+    fn now() i64 {
+        const value = std.time.nanoTimestamp();
+        return std.math.cast(i64, value) orelse if (value < 0) std.math.minInt(i64) else std.math.maxInt(i64);
+    }
+
+    fn mapTail(self: *Engine, force: bool) void {
+        if (self.file_length <= self.mapped_length or !force and self.file_length - self.mapped_length < map_interval) return;
+        const page_size = std.heap.pageSize();
+        const file_start = self.mapped_length - self.mapped_length % page_size;
+        const length = std.math.cast(usize, self.file_length - file_start) orelse return;
+        const bytes = std.posix.mmap(null, length, std.posix.PROT.READ, .{ .TYPE = .SHARED }, self.file.handle, file_start) catch return;
+        if (!force) self.mapping_lock.lock();
+        defer if (!force) self.mapping_lock.unlock();
+        self.mappings.append(self.allocator, .{ .bytes = bytes, .file_start = file_start, .logical_start = self.mapped_length, .logical_end = self.file_length }) catch {
+            std.posix.munmap(bytes);
+            return;
+        };
+        self.mapped_length = self.file_length;
+    }
+
+    fn readBytes(self: *Engine, destination: []u8, file_offset: u64) !usize {
+        const read_end = try std.math.add(u64, file_offset, destination.len);
+        self.mapping_lock.lockShared();
+        var index = self.mappings.items.len;
+        while (index != 0) {
+            index -= 1;
+            const mapping = self.mappings.items[index];
+            if (file_offset >= mapping.logical_start and read_end <= mapping.logical_end) {
+                const start: usize = @intCast(file_offset - mapping.file_start);
+                @memcpy(destination, mapping.bytes[start .. start + destination.len]);
+                self.mapping_lock.unlockShared();
+                return destination.len;
+            }
+        }
+        self.mapping_lock.unlockShared();
+        return self.file.preadAll(destination, file_offset);
+    }
+
+    fn readKeyBytes(context: *const anyopaque, destination: []u8, file_offset: u64) anyerror!usize {
+        const self: *Engine = @ptrCast(@alignCast(@constCast(context)));
+        return self.readBytes(destination, file_offset);
+    }
+
+    fn keyReader(self: *Engine) KeyReader {
+        return .{ .context = self, .readFn = readKeyBytes };
+    }
+
+    fn initialize(self: *Engine) !void {
+        std.crypto.random.bytes(&self.uuid);
+        self.generation = 1;
+        self.created_ns = now();
+        var first: [4096]u8 = undefined;
+        var second: [4096]u8 = undefined;
+        pkvdb.encodeSuperblock(.{
+            .generation = 1,
+            .uuid = self.uuid,
+            .manifest_offset = 0,
+            .checkpoint_lsn = 0,
+            .known_lsn = 0,
+            .known_file_length = pkvdb.data_offset,
+            .created_ns = self.created_ns,
+            .updated_ns = self.created_ns,
+        }, &first);
+        pkvdb.encodeSuperblock(.{
+            .generation = 0,
+            .uuid = self.uuid,
+            .manifest_offset = 0,
+            .checkpoint_lsn = 0,
+            .known_lsn = 0,
+            .known_file_length = pkvdb.data_offset,
+            .created_ns = self.created_ns,
+            .updated_ns = self.created_ns,
+        }, &second);
+        try self.file.pwriteAll(&first, 0);
+        try self.file.pwriteAll(&second, pkvdb.superblock_size);
+        try self.file.sync();
+        self.file_length = pkvdb.data_offset;
+        try self.file.seekTo(self.file_length);
+        self.bytes_written = pkvdb.data_offset;
+    }
+
+    fn readExtentHeader(self: *Engine, offset: u64) !pkvdb.ExtentHeader {
+        var bytes: [64]u8 = undefined;
+        if (try self.file.preadAll(&bytes, offset) != bytes.len) return error.Truncated;
+        return pkvdb.decodeExtentHeader(&bytes) catch |err| {
+            if (err == error.ChecksumMismatch) self.checksum_failures += 1;
+            return err;
+        };
+    }
+
+    fn validatePayload(self: *Engine, offset: u64, length: u64, expected: u32) !void {
+        var crc: u32 = 0xffffffff;
+        var buffer: [64 * 1024]u8 = undefined;
+        var done: u64 = 0;
+        while (done < length) {
+            const amount: usize = @intCast(@min(buffer.len, length - done));
+            if (try self.file.preadAll(buffer[0..amount], try std.math.add(u64, offset, done)) != amount) return error.Truncated;
+            crc = pkvdb.crc32cUpdate(crc, buffer[0..amount]);
+            done += amount;
+        }
+        if (~crc != expected) {
+            self.checksum_failures += 1;
+            return error.ChecksumMismatch;
+        }
+    }
+
+    fn readExtentPayload(self: *Engine, offset: u64, header: pkvdb.ExtentHeader, max: u64) ![]u8 {
+        if (header.payload_length > max or header.payload_length > std.math.maxInt(usize)) return error.InvalidLength;
+        const payload_offset = try std.math.add(u64, offset, pkvdb.extent_header_size);
+        const payload = try self.allocator.alloc(u8, @intCast(header.payload_length));
+        errdefer self.allocator.free(payload);
+        if (try self.file.preadAll(payload, payload_offset) != payload.len) return error.Truncated;
+        if (pkvdb.crc32c(payload) != header.payload_crc) {
+            self.checksum_failures += 1;
+            return error.ChecksumMismatch;
+        }
+        return payload;
+    }
+
+    fn extentEnd(offset: u64, payload_length: u64) !u64 {
+        return pkvdb.align8(try std.math.add(u64, try std.math.add(u64, offset, pkvdb.extent_header_size), payload_length));
+    }
+
+    fn loadManifest(self: *Engine, sb: pkvdb.Superblock) !?pkvdb.Manifest {
+        if (sb.manifest_offset == 0) return null;
+        const header = try self.readExtentHeader(sb.manifest_offset);
+        if (header.extent_type != .manifest or header.version != 1 or header.payload_length != pkvdb.manifest_size) return error.InvalidManifest;
+        if (try extentEnd(sb.manifest_offset, header.payload_length) > sb.known_file_length) return error.InvalidManifest;
+        const payload = try self.readExtentPayload(sb.manifest_offset, header, pkvdb.manifest_size);
+        defer self.allocator.free(payload);
+        const manifest = try pkvdb.decodeManifest(payload);
+        if (!std.mem.eql(u8, &manifest.uuid, &sb.uuid) or manifest.generation != sb.generation or manifest.checkpoint_lsn != sb.checkpoint_lsn) return error.InvalidManifest;
+        if (manifest.known_tail != sb.known_file_length or manifest.known_lsn != sb.known_lsn) return error.InvalidManifest;
+        if (manifest.entries_offset == 0 or manifest.entries_offset >= sb.manifest_offset or manifest.replay_offset < pkvdb.data_offset or manifest.replay_offset > manifest.entries_offset) return error.InvalidManifest;
+        try self.validateCheckpointExtent(manifest.entries_offset, manifest.checkpoint_lsn, sb.known_file_length);
+        return manifest;
+    }
+
+    fn validateCheckpointExtent(self: *Engine, offset: u64, lsn: u64, file_length: u64) !void {
+        const header = try self.readExtentHeader(offset);
+        if (header.extent_type != .checkpoint_entries or header.version != 1 or header.first_lsn != lsn or header.last_lsn != lsn) return error.InvalidCheckpoint;
+        const end = try extentEnd(offset, header.payload_length);
+        if (end > file_length) return error.InvalidCheckpoint;
+        try self.validatePayload(offset + pkvdb.extent_header_size, header.payload_length, header.payload_crc);
+        var bytes: [56]u8 = undefined;
+        if (header.payload_length < bytes.len or try self.file.preadAll(&bytes, offset + pkvdb.extent_header_size) != bytes.len) return error.InvalidCheckpoint;
+        const checkpoint_header = try pkvdb.decodeCheckpointHeaderOnly(&bytes, header.payload_length);
+        if (checkpoint_header.lsn != lsn) return error.InvalidCheckpoint;
+    }
+
+    fn recover(self: *Engine) !void {
+        if (self.file_length < pkvdb.data_offset) return error.Truncated;
+        var blocks: [2][4096]u8 = undefined;
+        _ = try self.file.preadAll(&blocks[0], 0);
+        _ = try self.file.preadAll(&blocks[1], pkvdb.superblock_size);
+        var roots: [2]?Root = .{ null, null };
+        for (0..2) |index| {
+            const sb = pkvdb.decodeSuperblock(&blocks[index], self.file_length) catch continue;
+            const manifest = self.loadManifest(sb) catch continue;
+            roots[index] = .{ .block = sb, .manifest = manifest };
+        }
+        var selected_index: usize = 0;
+        const selected = if (roots[0] != null and roots[1] != null) blk: {
+            selected_index = if (roots[1].?.block.generation > roots[0].?.block.generation) 1 else 0;
+            break :blk roots[selected_index].?;
+        } else if (roots[0]) |root| root else if (roots[1]) |root| blk: {
+            selected_index = 1;
+            break :blk root;
+        } else return error.NoUsableSuperblock;
+        self.uuid = selected.block.uuid;
+        self.generation = selected.block.generation;
+        self.active_superblock = @intCast(selected_index);
+        self.created_ns = selected.block.created_ns;
+        self.latest_lsn = selected.block.checkpoint_lsn;
+        self.checkpoint_lsn = selected.block.checkpoint_lsn;
+        var replay_offset = pkvdb.data_offset;
+        if (selected.manifest) |manifest| {
+            try self.loadCheckpoint(manifest.entries_offset, selected.block.known_file_length);
+            replay_offset = manifest.replay_offset;
+            self.oldest_lsn = manifest.history_start_lsn;
+        }
+        try self.scanExtents(replay_offset);
+        try self.file.setEndPos(self.file_length);
+        try self.file.seekTo(self.file_length);
+        self.bytes_written = self.file_length;
+    }
+
+    fn loadCheckpoint(self: *Engine, offset: u64, known_length: u64) !void {
+        const extent = try self.readExtentHeader(offset);
+        var header_bytes: [56]u8 = undefined;
+        const payload_offset = offset + pkvdb.extent_header_size;
+        if (try self.file.preadAll(&header_bytes, payload_offset) != header_bytes.len) return error.Truncated;
+        const header = try pkvdb.decodeCheckpointHeaderOnly(&header_bytes, extent.payload_length);
+        if (header.entry_count > std.math.maxInt(usize)) return error.InvalidLength;
+        try self.directory.ensureAdditional(self.file, @intCast(header.entry_count));
+        var index: u64 = 0;
+        while (index < header.entry_count) : (index += 1) {
+            var bytes: [48]u8 = undefined;
+            const entry_offset = try std.math.add(u64, payload_offset + pkvdb.checkpoint_header_size, try std.math.mul(u64, index, pkvdb.checkpoint_entry_size));
+            if (try self.file.preadAll(&bytes, entry_offset) != bytes.len) return error.Truncated;
+            const entry = try pkvdb.decodeCheckpointEntry(&bytes, known_length);
+            const record = RecordRef{ .hash = entry.hash, .lsn = entry.lsn, .key_offset = entry.key_offset, .value_offset = entry.value_offset, .key_len = entry.key_len, .value_len = entry.value_len, .flags = entry.flags };
+            try self.directory.put(self.file, record);
+        }
+    }
+
+    fn scanExtents(self: *Engine, start: u64) !void {
+        var offset = start;
+        var last_journal_lsn: u64 = 0;
+        while (offset < self.file_length) {
+            if (self.file_length - offset < pkvdb.extent_header_size) {
+                self.partial_tails += 1;
+                self.file_length = offset;
+                break;
+            }
+            const header = self.readExtentHeader(offset) catch |failure| {
+                if (self.file_length - offset == pkvdb.extent_header_size) {
+                    self.partial_tails += 1;
+                    self.file_length = offset;
+                    break;
+                }
+                return failure;
+            };
+            const end = extentEnd(offset, header.payload_length) catch return error.InvalidLength;
+            if (end > self.file_length) {
+                self.partial_tails += 1;
+                self.file_length = offset;
+                break;
+            }
+            try self.validatePayload(offset + pkvdb.extent_header_size, header.payload_length, header.payload_crc);
+            if (header.extent_type == .journal and header.version == 1) {
+                if (header.payload_length > pkvdb.max_transaction_size + pkvdb.group_header_size) return error.InvalidLength;
+                const payload = try self.readExtentPayload(offset, header, pkvdb.max_transaction_size + pkvdb.group_header_size);
+                defer self.allocator.free(payload);
+                last_journal_lsn = try self.replayJournal(offset, payload, last_journal_lsn);
+                self.journal_bytes_since_checkpoint += end - offset;
+            } else if (header.extent_type == .store_metadata and header.version == 1) {
+                if (header.payload_length > pkvdb.max_transaction_size) return error.InvalidLength;
+                const payload = try self.readExtentPayload(offset, header, pkvdb.max_transaction_size);
+                defer self.allocator.free(payload);
+                try self.replayBaseline(offset, payload);
+            }
+            offset = end;
+        }
+    }
+
+    fn replayBaseline(self: *Engine, extent_offset: u64, payload: []const u8) !void {
+        if (payload.len < 24 or readInt(u16, payload, 0) != 1 or readInt(u64, payload, 8) != 1) return error.InvalidBaseline;
+        const count = readInt(u32, payload, 4);
+        if (count > pkvdb.max_operations) return error.InvalidBaseline;
+        try self.directory.ensureAdditional(self.file, count);
+        var position: usize = 24;
+        for (0..count) |_| {
+            if (position > payload.len or payload.len - position < 8) return error.InvalidBaseline;
+            const key_length = readInt(u32, payload, position);
+            const value_length = readInt(u32, payload, position + 4);
+            if (key_length > pkvdb.max_key_size or value_length > pkvdb.max_value_size) return error.InvalidBaseline;
+            const key_start = position + 8;
+            const key_end = try std.math.add(usize, key_start, key_length);
+            const value_end = try std.math.add(usize, key_end, value_length);
+            if (value_end > payload.len) return error.InvalidBaseline;
+            const key = payload[key_start..key_end];
+            const key_offset = extent_offset + pkvdb.extent_header_size + key_start;
+            try self.replaceRecord(key, .{ .hash = keydir.KeyDir.hash(key), .lsn = 1, .key_offset = key_offset, .value_offset = key_offset + key_length, .key_len = key_length, .value_len = value_length });
+            position = @intCast(try pkvdb.align8(value_end));
+        }
+        if (position != payload.len) return error.InvalidBaseline;
+        self.latest_lsn = 1;
+        self.oldest_lsn = 1;
+    }
+
+    fn replayJournal(self: *Engine, extent_offset: u64, payload: []const u8, previous_lsn: u64) !u64 {
+        if (payload.len < pkvdb.group_header_size or readInt(u16, payload, 0) != 1) return error.InvalidJournal;
+        const count = readInt(u32, payload, 4);
+        if (count == 0 or count > pkvdb.max_operations) return error.InvalidJournal;
+        var position: usize = pkvdb.group_header_size;
+        var last = previous_lsn;
+        for (0..count) |_| {
+            if (position > payload.len or payload.len - position < pkvdb.transaction_header_size) return error.InvalidJournal;
+            const tx = try pkvdb.decodeTransactionHeader(payload[position..]);
+            if (tx.total_length > payload.len - position) return error.InvalidJournal;
+            const tx_end = position + tx.total_length;
+            const body = payload[position + pkvdb.transaction_header_size .. tx_end];
+            if (pkvdb.crc32c(body) != tx.payload_crc or tx.metadata_length > body.len) return error.ChecksumMismatch;
+            if (last != 0 and tx.lsn <= last) return error.InvalidLsn;
+            last = tx.lsn;
+            if (tx.lsn > self.latest_lsn) try self.applyTransaction(extent_offset + pkvdb.extent_header_size, position, tx, payload[position..tx_end]);
+            position = tx_end;
+        }
+        if (position != payload.len) return error.InvalidJournal;
+        return last;
+    }
+
+    fn applyTransaction(self: *Engine, extent_payload_offset: u64, tx_position: usize, tx: pkvdb.TransactionHeader, frame: []const u8) !void {
+        var position: usize = pkvdb.transaction_header_size + tx.metadata_length;
+        try self.directory.ensureAdditional(self.file, tx.operation_count);
+        for (0..tx.operation_count) |_| {
+            if (position > frame.len or frame.len - position < pkvdb.operation_header_size) return error.InvalidJournal;
+            const operation = try pkvdb.decodeOperationHeader(frame[position..]);
+            const data_start = try std.math.add(usize, position, pkvdb.operation_header_size);
+            const key_end = try std.math.add(usize, data_start, operation.key_length);
+            const value_end = try std.math.add(usize, key_end, operation.value_length);
+            const extension_end = try std.math.add(usize, value_end, operation.extension_length);
+            if (extension_end > frame.len) return error.InvalidJournal;
+            const key = frame[data_start..key_end];
+            switch (operation.opcode) {
+                .put => {
+                    const key_offset = try std.math.add(u64, extent_payload_offset, tx_position + data_start);
+                    const record = RecordRef{ .hash = keydir.KeyDir.hash(key), .lsn = tx.lsn, .key_offset = key_offset, .value_offset = key_offset + operation.key_length, .key_len = operation.key_length, .value_len = operation.value_length };
+                    try self.replaceRecord(key, record);
+                },
+                .delete => _ = try self.removeRecord(key),
+            }
+            position = @intCast(try pkvdb.align8(extension_end));
+        }
+        if (position != frame.len) return error.InvalidJournal;
+        self.latest_lsn = tx.lsn;
+        if (self.oldest_lsn == 0) self.oldest_lsn = tx.lsn;
+    }
+
+    fn replaceRecord(self: *Engine, key: []const u8, record: RecordRef) !void {
+        if (self.ordered_ready) try self.ordered.put(key, record);
+        try self.directory.putWithKey(self.file, key, record);
+    }
+
+    fn removeRecord(self: *Engine, key: []const u8) !bool {
+        const removed = try self.directory.remove(self.file, key);
+        if (removed and self.ordered_ready and !self.ordered.remove(key)) return error.IndexInconsistent;
+        return removed;
+    }
+
+    fn appendExtent(self: *Engine, extent_type: pkvdb.ExtentType, first_lsn: u64, last_lsn: u64, payload: []const u8) !u64 {
+        const offset = try pkvdb.align8(self.file_length);
+        if (offset != self.file_length) {
+            const padding = [_]u8{0} ** 8;
+            try self.file.pwriteAll(padding[0 .. offset - self.file_length], self.file_length);
+        }
+        var header: [64]u8 = undefined;
+        pkvdb.encodeExtentHeader(.{ .extent_type = extent_type, .payload_length = payload.len, .first_lsn = first_lsn, .last_lsn = last_lsn, .payload_crc = pkvdb.crc32c(payload) }, &header);
+        const old_length = self.file_length;
+        errdefer self.file.setEndPos(old_length) catch {};
+        try self.file.pwriteAll(&header, offset);
+        try self.file.pwriteAll(payload, offset + header.len);
+        const end = try extentEnd(offset, payload.len);
+        if (end > offset + header.len + payload.len) {
+            const padding = [_]u8{0} ** 8;
+            try self.file.pwriteAll(padding[0 .. end - (offset + header.len + payload.len)], offset + header.len + payload.len);
+        }
+        self.file_length = end;
+        self.bytes_written += end - offset;
+        return offset;
+    }
+
+    fn transactionLength(operations: []const Operation, metadata: []const u8) !usize {
+        var frame_length: u64 = pkvdb.transaction_header_size + metadata.len;
+        for (operations) |operation| {
+            const raw = try std.math.add(u64, pkvdb.operation_header_size + operation.key.len + operation.value.len, 0);
+            frame_length = try pkvdb.align8(try std.math.add(u64, frame_length, raw));
+        }
+        if (frame_length > pkvdb.max_transaction_size) return error.TransactionTooLarge;
+        return @intCast(frame_length);
+    }
+
+    fn encodeTransaction(frame: []u8, operations: []const Operation, metadata: []const u8, lsn: u64, transaction_id: u64, timestamp_ns: i64) !void {
+        @memset(frame, 0);
+        var position: usize = pkvdb.transaction_header_size;
+        @memcpy(frame[position .. position + metadata.len], metadata);
+        position += metadata.len;
+        for (operations) |operation| {
+            var header: [16]u8 = undefined;
+            pkvdb.encodeOperationHeader(.{ .opcode = operation.opcode, .key_length = @intCast(operation.key.len), .value_length = @intCast(operation.value.len) }, &header);
+            @memcpy(frame[position .. position + header.len], &header);
+            position += header.len;
+            @memcpy(frame[position .. position + operation.key.len], operation.key);
+            position += operation.key.len;
+            @memcpy(frame[position .. position + operation.value.len], operation.value);
+            position += operation.value.len;
+            position = @intCast(try pkvdb.align8(position));
+        }
+        if (position != frame.len) return error.InvalidLength;
+        const body = frame[pkvdb.transaction_header_size..];
+        var tx_header: [56]u8 = undefined;
+        pkvdb.encodeTransactionHeader(.{ .total_length = @intCast(frame.len), .lsn = lsn, .transaction_id = transaction_id, .timestamp_ns = timestamp_ns, .operation_count = @intCast(operations.len), .metadata_length = @intCast(metadata.len), .payload_crc = pkvdb.crc32c(body) }, &tx_header);
+        @memcpy(frame[0..tx_header.len], &tx_header);
+    }
+
+    pub fn batchWrite(self: *Engine, operations: []const Operation, metadata: []const u8) !u64 {
+        if (operations.len == 0 or operations.len > pkvdb.max_operations or metadata.len > pkvdb.max_transaction_size) return error.InvalidLength;
+        for (operations) |operation| {
+            if (operation.key.len > pkvdb.max_key_size or operation.value.len > pkvdb.max_value_size) return error.InvalidLength;
+            if (operation.opcode == .delete and operation.value.len != 0) return error.InvalidLength;
+        }
+        const frame_length = try transactionLength(operations, metadata);
+        var completion = WriteCompletion{ .remaining = 1 };
+        var pending = PendingWrite{ .operations = operations, .metadata = metadata, .bytes = frame_length, .completion = &completion };
+        try self.enqueueAndWait(&.{&pending}, &completion);
+        if (completion.failure) |failure| return failure;
+        return pending.lsn;
+    }
+
+    pub fn putMany(self: *Engine, operations: []const Operation, lsns: []u64) !void {
+        if (operations.len == 0 or operations.len != lsns.len or operations.len > max_group_transactions) return error.InvalidLength;
+        const pending = try self.allocator.alloc(PendingWrite, operations.len);
+        defer self.allocator.free(pending);
+        const pointers = try self.allocator.alloc(*PendingWrite, operations.len);
+        defer self.allocator.free(pointers);
+        var completion = WriteCompletion{ .remaining = operations.len };
+        for (operations, 0..) |operation, index| {
+            if (operation.opcode != .put or operation.key.len > pkvdb.max_key_size or operation.value.len > pkvdb.max_value_size) return error.InvalidLength;
+            const operation_slice = operations[index .. index + 1];
+            pending[index] = .{ .operations = operation_slice, .metadata = "", .bytes = try transactionLength(operation_slice, ""), .completion = &completion };
+            pointers[index] = &pending[index];
+        }
+        try self.enqueueAndWait(pointers, &completion);
+        for (pending, 0..) |result, index| {
+            if (completion.failure) |failure| return failure;
+            lsns[index] = result.lsn;
+        }
+    }
+
+    fn enqueueAndWait(self: *Engine, pending: []const *PendingWrite, completion: *WriteCompletion) !void {
+        self.queue_mutex.lock();
+        defer self.queue_mutex.unlock();
+        if (self.writer_failed or self.writer_stopping) return error.StorageUnavailable;
+        if (self.writer_thread == null) self.writer_thread = try std.Thread.spawn(.{}, writerMain, .{self});
+        for (pending) |write| {
+            while (self.queued_bytes > max_queued_bytes - write.bytes and !self.writer_failed and !self.writer_stopping) self.queue_condition.wait(&self.queue_mutex);
+            if (self.writer_failed or self.writer_stopping) return error.StorageUnavailable;
+            if (self.queue_tail) |tail| tail.next = write else self.queue_head = write;
+            self.queue_tail = write;
+            self.queued_bytes += write.bytes;
+        }
+        self.queue_condition.signal();
+        while (completion.remaining != 0) completion.condition.wait(&self.queue_mutex);
+    }
+
+    fn writerMain(self: *Engine) void {
+        var group: [max_group_transactions]*PendingWrite = undefined;
+        while (true) {
+            self.queue_mutex.lock();
+            while (self.queue_head == null and !self.writer_stopping) self.queue_condition.wait(&self.queue_mutex);
+            if (self.queue_head == null and self.writer_stopping) {
+                self.queue_mutex.unlock();
+                return;
+            }
+            _ = self.queue_condition.timedWait(&self.queue_mutex, group_wait_ns) catch {};
+            var count: usize = 0;
+            var bytes: usize = pkvdb.group_header_size;
+            while (self.queue_head) |pending| {
+                if (count != 0 and (count == group.len or bytes + pending.bytes > max_group_bytes)) break;
+                self.queue_head = pending.next;
+                if (self.queue_head == null) self.queue_tail = null;
+                pending.next = null;
+                group[count] = pending;
+                count += 1;
+                bytes += pending.bytes;
+                self.queued_bytes -= pending.bytes;
+            }
+            self.queue_condition.broadcast();
+            self.queue_mutex.unlock();
+            self.processGroup(group[0..count], bytes) catch |failure| {
+                self.queue_mutex.lock();
+                self.writer_failed = true;
+                for (group[0..count]) |pending| {
+                    pending.completion.failure = failure;
+                    pending.completion.remaining -= 1;
+                    if (pending.completion.remaining == 0) pending.completion.condition.signal();
+                }
+                while (self.queue_head) |pending| {
+                    self.queue_head = pending.next;
+                    pending.completion.failure = error.StorageUnavailable;
+                    pending.completion.remaining -= 1;
+                    if (pending.completion.remaining == 0) pending.completion.condition.signal();
+                }
+                self.queue_tail = null;
+                self.queued_bytes = 0;
+                self.queue_condition.broadcast();
+                self.queue_mutex.unlock();
+                continue;
+            };
+            self.queue_mutex.lock();
+            for (group[0..count]) |pending| {
+                pending.completion.remaining -= 1;
+                if (pending.completion.remaining == 0) pending.completion.condition.signal();
+            }
+            self.queue_mutex.unlock();
+        }
+    }
+
+    fn processGroup(self: *Engine, group: []*PendingWrite, payload_length: usize) !void {
+        var operation_count: usize = 0;
+        for (group) |pending| operation_count = try std.math.add(usize, operation_count, pending.operations.len);
+        const payload = try self.allocator.alloc(u8, payload_length);
+        defer self.allocator.free(payload);
+        self.ordered_gate.lock();
+        defer self.ordered_gate.unlock();
+        self.lock.lock();
+        self.directory.ensureAdditional(self.file, operation_count) catch |failure| {
+            self.lock.unlock();
+            return failure;
+        };
+        const first_lsn = std.math.add(u64, self.latest_lsn, 1) catch |failure| {
+            self.lock.unlock();
+            return failure;
+        };
+        const prepare_ordered = self.ordered_ready;
+        self.lock.unlock();
+        var prepared_storage: []ordered_index.Prepared = &.{};
+        if (prepare_ordered) prepared_storage = try self.allocator.alloc(ordered_index.Prepared, operation_count);
+        var prepared_count: usize = 0;
+        defer {
+            for (prepared_storage[0..prepared_count]) |*entry| self.ordered.discard(entry);
+            if (prepared_storage.len != 0) self.allocator.free(prepared_storage);
+        }
+        if (prepare_ordered) for (group) |pending| {
+            pending.prepared_position = prepared_count;
+            for (pending.operations) |operation| if (operation.opcode == .put) {
+                prepared_storage[prepared_count] = try self.ordered.prepare(operation.key);
+                prepared_count += 1;
+            };
+            pending.prepared_count = prepared_count - pending.prepared_position;
+        };
+        @memset(payload, 0);
+        const group_timestamp = now();
+        writeInt(u16, payload, 0, 1);
+        writeInt(u32, payload, 4, @intCast(group.len));
+        writeInt(i64, payload, 8, group_timestamp);
+        var position: usize = pkvdb.group_header_size;
+        var lsn = first_lsn;
+        for (group, 0..) |pending, index| {
+            pending.frame_position = position;
+            const timestamp = std.math.add(i64, group_timestamp, @intCast(index)) catch std.math.maxInt(i64);
+            try encodeTransaction(payload[position .. position + pending.bytes], pending.operations, pending.metadata, lsn, lsn, timestamp);
+            pending.lsn = lsn;
+            position += pending.bytes;
+            lsn = try std.math.add(u64, lsn, 1);
+        }
+        const last_lsn = lsn - 1;
+        self.io_mutex.lock();
+        defer self.io_mutex.unlock();
+        const offset = try self.appendExtent(.journal, first_lsn, last_lsn, payload);
+        self.file.sync() catch |failure| {
+            self.file.setEndPos(offset) catch {};
+            self.file_length = offset;
+            return failure;
+        };
+        self.mapTail(false);
+        self.lock.lock();
+        defer self.lock.unlock();
+        for (group) |pending| {
+            try self.publishPending(offset + pkvdb.extent_header_size, pending, prepared_storage[0..prepared_count]);
+        }
+        self.journal_bytes_since_checkpoint += self.file_length - offset;
+        self.commit_groups += 1;
+        self.committed_transactions += group.len;
+        self.largest_commit_group = @max(self.largest_commit_group, group.len);
+    }
+
+    fn publishPending(self: *Engine, extent_payload_offset: u64, pending: *PendingWrite, prepared: []ordered_index.Prepared) !void {
+        var position: usize = pkvdb.transaction_header_size + pending.metadata.len;
+        var prepared_position = pending.prepared_position;
+        for (pending.operations) |operation| {
+            const key_offset = try std.math.add(u64, extent_payload_offset, pending.frame_position + position + pkvdb.operation_header_size);
+            switch (operation.opcode) {
+                .put => {
+                    const record = RecordRef{ .hash = keydir.KeyDir.hash(operation.key), .lsn = pending.lsn, .key_offset = key_offset, .value_offset = key_offset + operation.key.len, .key_len = @intCast(operation.key.len), .value_len = @intCast(operation.value.len) };
+                    if (self.ordered_ready) {
+                        self.ordered.putPrepared(&prepared[prepared_position], record);
+                        prepared_position += 1;
+                    }
+                    try self.directory.putWithKey(self.file, operation.key, record);
+                },
+                .delete => pending.changed = (try self.removeRecord(operation.key)) or pending.changed,
+            }
+            position = @intCast(try pkvdb.align8(position + pkvdb.operation_header_size + operation.key.len + operation.value.len));
+        }
+        if (prepared_position != pending.prepared_position + pending.prepared_count) return error.IndexInconsistent;
+        if (position != pending.bytes) return error.InvalidLength;
+        self.latest_lsn = pending.lsn;
+        if (self.oldest_lsn == 0) self.oldest_lsn = pending.lsn;
+    }
+
+    pub fn importBaseline(self: *Engine, operations: []const Operation) !void {
+        if (operations.len > pkvdb.max_operations) return error.InvalidLength;
+        self.io_mutex.lock();
+        defer self.io_mutex.unlock();
+        self.lock.lock();
+        defer self.lock.unlock();
+        if (self.checkpoint_lsn != 0 or self.latest_lsn > 1 or self.directory.count != 0 and self.latest_lsn == 0) return error.InvalidState;
+        var length: u64 = 24;
+        for (operations) |operation| {
+            if (operation.opcode != .put or operation.key.len > pkvdb.max_key_size or operation.value.len > pkvdb.max_value_size) return error.InvalidLength;
+            length = try pkvdb.align8(try std.math.add(u64, length, 8 + operation.key.len + operation.value.len));
+        }
+        if (length > pkvdb.max_transaction_size) return error.TransactionTooLarge;
+        const payload = try self.allocator.alloc(u8, @intCast(length));
+        defer self.allocator.free(payload);
+        @memset(payload, 0);
+        writeInt(u16, payload, 0, 1);
+        writeInt(u32, payload, 4, @intCast(operations.len));
+        writeInt(u64, payload, 8, 1);
+        writeInt(i64, payload, 16, now());
+        var position: usize = 24;
+        for (operations) |operation| {
+            writeInt(u32, payload, position, @intCast(operation.key.len));
+            writeInt(u32, payload, position + 4, @intCast(operation.value.len));
+            position += 8;
+            @memcpy(payload[position .. position + operation.key.len], operation.key);
+            position += operation.key.len;
+            @memcpy(payload[position .. position + operation.value.len], operation.value);
+            position += operation.value.len;
+            const aligned: usize = @intCast(try pkvdb.align8(position));
+            @memset(payload[position..aligned], 0);
+            position = aligned;
+        }
+        try self.directory.ensureAdditional(self.file, operations.len);
+        const offset = try self.appendExtent(.store_metadata, 1, 1, payload);
+        try self.file.sync();
+        try self.replayBaseline(offset, payload);
+    }
+
+    pub fn put(self: *Engine, key: []const u8, value: []const u8) !u64 {
+        return self.batchWrite(&.{.{ .opcode = .put, .key = key, .value = value }}, "");
+    }
+
+    pub fn delete(self: *Engine, key: []const u8) !bool {
+        if (key.len > pkvdb.max_key_size) return error.InvalidLength;
+        const operations = [_]Operation{.{ .opcode = .delete, .key = key }};
+        var completion = WriteCompletion{ .remaining = 1 };
+        var pending = PendingWrite{ .operations = &operations, .metadata = "", .bytes = try transactionLength(&operations, ""), .completion = &completion };
+        try self.enqueueAndWait(&.{&pending}, &completion);
+        if (completion.failure) |failure| return failure;
+        return pending.changed;
+    }
+
+    pub fn get(self: *Engine, allocator: std.mem.Allocator, key: []const u8) !?Value {
+        const record = try self.getRef(key) orelse return null;
+        const bytes = try allocator.alloc(u8, record.value_len);
+        errdefer allocator.free(bytes);
+        _ = try self.readValue(record, bytes, 0);
+        return .{ .bytes = bytes, .lsn = record.lsn };
+    }
+
+    pub fn getRef(self: *Engine, key: []const u8) !?RecordRef {
+        if (key.len > pkvdb.max_key_size) return error.InvalidLength;
+        self.lock.lockShared();
+        defer self.lock.unlockShared();
+        return try self.directory.getWithReader(self.keyReader(), key);
+    }
+
+    pub fn readValue(self: *Engine, record: RecordRef, destination: []u8, value_position: u32) !usize {
+        if (value_position > record.value_len) return error.InvalidOffset;
+        const amount = @min(destination.len, record.value_len - value_position);
+        const file_offset = record.value_offset + value_position;
+        const got = try self.readBytes(destination[0..amount], file_offset);
+        if (got != amount) return error.Truncated;
+        return amount;
+    }
+
+    pub fn exists(self: *Engine, key: []const u8) !bool {
+        self.lock.lockShared();
+        defer self.lock.unlockShared();
+        return (try self.directory.getWithReader(self.keyReader(), key)) != null;
+    }
+
+    pub fn multiGet(self: *Engine, allocator: std.mem.Allocator, keys: []const []const u8) ![]?Value {
+        if (keys.len > pkvdb.max_operations) return error.InvalidLength;
+        const refs = try allocator.alloc(?RecordRef, keys.len);
+        defer allocator.free(refs);
+        @memset(refs, null);
+        self.lock.lockShared();
+        for (keys, 0..) |key, index| {
+            if (key.len > pkvdb.max_key_size) {
+                self.lock.unlockShared();
+                return error.InvalidLength;
+            }
+            refs[index] = self.directory.getWithReader(self.keyReader(), key) catch |failure| {
+                self.lock.unlockShared();
+                return failure;
+            };
+        }
+        self.lock.unlockShared();
+        const values = try allocator.alloc(?Value, keys.len);
+        errdefer allocator.free(values);
+        @memset(values, null);
+        errdefer for (values) |value| if (value) |present| allocator.free(present.bytes);
+        for (refs, 0..) |entry, index| {
+            const record = entry orelse continue;
+            const bytes = try allocator.alloc(u8, record.value_len);
+            errdefer allocator.free(bytes);
+            _ = try self.readValue(record, bytes, 0);
+            values[index] = .{ .bytes = bytes, .lsn = record.lsn };
+        }
+        return values;
+    }
+
+    pub fn scan(self: *Engine, allocator: std.mem.Allocator, prefix: []const u8, cursor: []const u8, limit: u32, include_values: bool, max_bytes: u32) !ScanBatch {
+        if (prefix.len > pkvdb.max_key_size or cursor.len > pkvdb.max_key_size or limit == 0 or limit > 4096 or max_bytes == 0 or max_bytes > 1024 * 1024) return error.InvalidLength;
+        try self.ensureOrdered();
+        self.lock.lockShared();
+        defer self.lock.unlockShared();
+        var node = self.ordered.lowerBound(if (cursor.len == 0) prefix else cursor);
+        if (cursor.len != 0 and node != null and std.mem.eql(u8, node.?.key, cursor)) node = ordered_index.OrderedIndex.next(node.?);
+        var entries = std.ArrayListUnmanaged(ScanEntry){};
+        errdefer {
+            for (entries.items) |entry| {
+                allocator.free(entry.key);
+                if (entry.value) |value| allocator.free(value);
+            }
+            entries.deinit(allocator);
+        }
+        var bytes_used: usize = 0;
+        while (node) |current| {
+            if (entries.items.len >= limit or !std.mem.startsWith(u8, current.key, prefix)) break;
+            const record = current.record;
+            const next_size = current.key.len + if (include_values) record.value_len else 0;
+            if (next_size > max_bytes) return error.ScanEntryTooLarge;
+            if (bytes_used + next_size > max_bytes) break;
+            const key = try allocator.dupe(u8, current.key);
+            errdefer allocator.free(key);
+            var value: ?[]u8 = null;
+            if (include_values) {
+                value = try allocator.alloc(u8, record.value_len);
+                errdefer allocator.free(value.?);
+                _ = try self.readValue(record, value.?, 0);
+            }
+            try entries.append(allocator, .{ .key = key, .value = value, .lsn = record.lsn });
+            bytes_used += next_size;
+            node = ordered_index.OrderedIndex.next(current);
+        }
+        const next_cursor = if (entries.items.len == 0) try allocator.alloc(u8, 0) else try allocator.dupe(u8, entries.items[entries.items.len - 1].key);
+        errdefer allocator.free(next_cursor);
+        const done = node == null or !std.mem.startsWith(u8, node.?.key, prefix);
+        return .{ .entries = try entries.toOwnedSlice(allocator), .next_cursor = next_cursor, .done = done };
+    }
+
+    fn ensureOrdered(self: *Engine) !void {
+        self.lock.lockShared();
+        const ready = self.ordered_ready;
+        self.lock.unlockShared();
+        if (ready) return;
+        self.ordered_gate.lock();
+        defer self.ordered_gate.unlock();
+        self.lock.lock();
+        defer self.lock.unlock();
+        if (self.ordered_ready) return;
+        const records = try self.directory.records(self.allocator);
+        defer self.allocator.free(records);
+        for (records) |record| {
+            const key = try self.allocator.alloc(u8, record.key_len);
+            defer self.allocator.free(key);
+            if (try self.file.preadAll(key, record.key_offset) != key.len) return error.Truncated;
+            try self.ordered.put(key, record);
+        }
+        self.ordered_ready = true;
+    }
+
+    pub fn checkpoint(self: *Engine) !void {
+        const started = std.time.nanoTimestamp();
+        self.checkpoint_mutex.lock();
+        defer self.checkpoint_mutex.unlock();
+        self.io_mutex.lock();
+        self.lock.lockShared();
+        const records = self.directory.records(self.allocator) catch |err| {
+            self.lock.unlockShared();
+            self.io_mutex.unlock();
+            return err;
+        };
+        const lsn = self.latest_lsn;
+        const replay_offset = self.file_length;
+        self.lock.unlockShared();
+        self.io_mutex.unlock();
+        defer self.allocator.free(records);
+        const payload_length = try std.math.add(usize, pkvdb.checkpoint_header_size, try std.math.mul(usize, records.len, pkvdb.checkpoint_entry_size));
+        const payload = try self.allocator.alloc(u8, payload_length);
+        defer self.allocator.free(payload);
+        var header: [56]u8 = undefined;
+        pkvdb.encodeCheckpointHeader(.{ .lsn = lsn, .timestamp_ns = now(), .entry_count = records.len, .source_start = pkvdb.data_offset, .source_end = replay_offset }, &header);
+        @memcpy(payload[0..header.len], &header);
+        for (records, 0..) |record, index| {
+            if (record.flags > std.math.maxInt(u16)) return error.InvalidFlags;
+            var entry: [48]u8 = undefined;
+            pkvdb.encodeCheckpointEntry(.{ .hash = record.hash, .lsn = record.lsn, .key_offset = record.key_offset, .value_offset = record.value_offset, .key_len = record.key_len, .value_len = record.value_len, .flags = @intCast(record.flags) }, &entry);
+            @memcpy(payload[pkvdb.checkpoint_header_size + index * pkvdb.checkpoint_entry_size ..][0..pkvdb.checkpoint_entry_size], &entry);
+        }
+        self.io_mutex.lock();
+        defer self.io_mutex.unlock();
+        self.lock.lockShared();
+        const known_lsn = self.latest_lsn;
+        const next_generation = std.math.add(u64, self.generation, 1) catch |failure| {
+            self.lock.unlockShared();
+            return failure;
+        };
+        const history_start_lsn = self.oldest_lsn;
+        self.lock.unlockShared();
+        const entries_offset = try self.appendExtent(.checkpoint_entries, lsn, lsn, payload);
+        const manifest_offset = try pkvdb.align8(self.file_length);
+        const known_tail = try extentEnd(manifest_offset, pkvdb.manifest_size);
+        var manifest_bytes: [104]u8 = undefined;
+        pkvdb.encodeManifest(.{ .uuid = self.uuid, .generation = next_generation, .checkpoint_lsn = lsn, .entries_offset = entries_offset, .ordered_offset = 0, .replay_offset = replay_offset, .known_tail = known_tail, .known_lsn = known_lsn, .history_start_lsn = history_start_lsn }, &manifest_bytes);
+        const actual_manifest = try self.appendExtent(.manifest, lsn, known_lsn, &manifest_bytes);
+        if (actual_manifest != manifest_offset or self.file_length != known_tail) return error.InvalidManifest;
+        try self.file.sync();
+        const inactive: u1 = self.active_superblock ^ 1;
+        var block: [4096]u8 = undefined;
+        pkvdb.encodeSuperblock(.{ .generation = next_generation, .uuid = self.uuid, .manifest_offset = manifest_offset, .checkpoint_lsn = lsn, .known_lsn = known_lsn, .known_file_length = known_tail, .created_ns = self.created_ns, .updated_ns = now() }, &block);
+        try self.file.pwriteAll(&block, @as(u64, inactive) * pkvdb.superblock_size);
+        try self.file.sync();
+        self.lock.lock();
+        self.active_superblock = inactive;
+        self.generation = next_generation;
+        self.checkpoint_lsn = lsn;
+        self.journal_bytes_since_checkpoint = self.file_length - replay_offset;
+        const elapsed = std.time.nanoTimestamp() - started;
+        self.checkpoint_ns = if (elapsed > 0) @intCast(elapsed) else 0;
+        self.lock.unlock();
+    }
+
+    pub fn status(self: *Engine) Status {
+        self.io_mutex.lock();
+        defer self.io_mutex.unlock();
+        self.lock.lockShared();
+        defer self.lock.unlockShared();
+        return .{
+            .uuid = self.uuid,
+            .file_bytes = self.file_length,
+            .latest_lsn = self.latest_lsn,
+            .oldest_lsn = self.oldest_lsn,
+            .checkpoint_lsn = self.checkpoint_lsn,
+            .journal_bytes_since_checkpoint = self.journal_bytes_since_checkpoint,
+            .live_keys = self.directory.count,
+            .keydir_bytes = self.directory.bytes(),
+            .ordered_index_bytes = self.ordered.allocated_bytes,
+            .bytes_written = self.bytes_written,
+            .checksum_failures = self.checksum_failures,
+            .partial_tails = self.partial_tails,
+            .recovery_ns = self.recovery_ns,
+            .checkpoint_ns = self.checkpoint_ns,
+            .connection_bytes = self.connection_bytes.load(.monotonic),
+            .active_requests = self.active_requests.load(.monotonic),
+            .commit_groups = self.commit_groups,
+            .committed_transactions = self.committed_transactions,
+            .largest_commit_group = self.largest_commit_group,
+        };
+    }
+
+    pub fn addConnectionBytes(self: *Engine, amount: u64) void {
+        _ = self.connection_bytes.fetchAdd(amount, .monotonic);
+    }
+
+    pub fn removeConnectionBytes(self: *Engine, amount: u64) void {
+        _ = self.connection_bytes.fetchSub(amount, .monotonic);
+    }
+
+    pub fn beginRequest(self: *Engine) void {
+        _ = self.active_requests.fetchAdd(1, .monotonic);
+    }
+
+    pub fn endRequest(self: *Engine) void {
+        _ = self.active_requests.fetchSub(1, .monotonic);
+    }
+};
+
+fn readInt(comptime T: type, bytes: []const u8, offset: usize) T {
+    return std.mem.readInt(T, bytes[offset..][0..@sizeOf(T)], .little);
+}
+
+fn writeInt(comptime T: type, bytes: []u8, offset: usize, value: T) void {
+    std.mem.writeInt(T, bytes[offset..][0..@sizeOf(T)], value, .little);
+}
+
+test "insert overwrite delete recreate and recovery" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const directory = try tmp.dir.realpath(".", &path_buffer);
+    const path = try std.fmt.allocPrint(std.testing.allocator, "{s}/test.pkvdb", .{directory});
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    _ = try engine.put("key", "one");
+    _ = try engine.put("key", "two");
+    try std.testing.expect(try engine.delete("key"));
+    _ = try engine.put("key", "three");
+    var value = (try engine.get(std.testing.allocator, "key")).?;
+    try std.testing.expectEqualStrings("three", value.bytes);
+    std.testing.allocator.free(value.bytes);
+    engine.close();
+    engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    value = (try engine.get(std.testing.allocator, "key")).?;
+    defer std.testing.allocator.free(value.bytes);
+    try std.testing.expectEqualStrings("three", value.bytes);
+    try std.testing.expectEqual(@as(u64, 4), value.lsn);
+}
+
+test "atomic batch checkpoint tail and ordered scan" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const directory = try tmp.dir.realpath(".", &path_buffer);
+    const path = try std.fmt.allocPrint(std.testing.allocator, "{s}/test.pkvdb", .{directory});
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    _ = try engine.batchWrite(&.{ .{ .opcode = .put, .key = "p/2", .value = "b" }, .{ .opcode = .put, .key = "p/1", .value = "a" } }, "meta");
+    try engine.checkpoint();
+    _ = try engine.put("p/3", "c");
+    engine.close();
+    engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    var batch = try engine.scan(std.testing.allocator, "p/", "", 2, true, 1024);
+    try std.testing.expectEqual(@as(usize, 2), batch.entries.len);
+    try std.testing.expectEqualStrings("p/1", batch.entries[0].key);
+    const cursor = try std.testing.allocator.dupe(u8, batch.next_cursor);
+    batch.deinit(std.testing.allocator);
+    defer std.testing.allocator.free(cursor);
+    batch = try engine.scan(std.testing.allocator, "p/", cursor, 2, true, 1024);
+    defer batch.deinit(std.testing.allocator);
+    try std.testing.expectEqual(@as(usize, 1), batch.entries.len);
+    try std.testing.expectEqualStrings("p/3", batch.entries[0].key);
+}
+
+fn testPath(tmp: *std.testing.TmpDir, name: []const u8, buffer: *[std.fs.max_path_bytes]u8) ![]const u8 {
+    const directory = try tmp.dir.realpath(".", buffer);
+    return std.fs.path.join(std.testing.allocator, &.{ directory, name });
+}
+
+test "partial final extent is ignored and removed" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "partial.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    _ = try engine.put("durable", "value");
+    const valid_length = engine.status().file_bytes;
+    engine.close();
+    const file = try std.fs.cwd().openFile(path, .{ .mode = .read_write });
+    var header: [64]u8 = undefined;
+    pkvdb.encodeExtentHeader(.{ .extent_type = .journal, .payload_length = 100, .first_lsn = 2, .last_lsn = 2, .payload_crc = 0 }, &header);
+    try file.pwriteAll(&header, valid_length);
+    try file.pwriteAll("partial transaction", valid_length + header.len);
+    file.close();
+    engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    const value = (try engine.get(std.testing.allocator, "durable")).?;
+    defer std.testing.allocator.free(value.bytes);
+    try std.testing.expectEqualStrings("value", value.bytes);
+    try std.testing.expectEqual(valid_length, engine.status().file_bytes);
+    try std.testing.expectEqual(@as(u64, 1), engine.status().partial_tails);
+}
+
+test "corruption in committed journal is explicit" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "corrupt.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    _ = try engine.put("key", "value");
+    engine.close();
+    const file = try std.fs.cwd().openFile(path, .{ .mode = .read_write });
+    var byte: [1]u8 = undefined;
+    const offset = pkvdb.data_offset + pkvdb.extent_header_size + pkvdb.group_header_size + pkvdb.transaction_header_size + pkvdb.operation_header_size + 3;
+    _ = try file.preadAll(&byte, offset);
+    byte[0] ^= 1;
+    try file.pwriteAll(&byte, offset);
+    file.close();
+    try std.testing.expectError(error.ChecksumMismatch, Engine.open(std.testing.allocator, path));
+}
+
+test "one corrupted superblock falls back and adopts tail" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "root.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    _ = try engine.put("before", "one");
+    try engine.checkpoint();
+    _ = try engine.put("after", "two");
+    engine.close();
+    const file = try std.fs.cwd().openFile(path, .{ .mode = .read_write });
+    var byte: [1]u8 = undefined;
+    _ = try file.preadAll(&byte, pkvdb.superblock_size + 24);
+    byte[0] ^= 1;
+    try file.pwriteAll(&byte, pkvdb.superblock_size + 24);
+    file.close();
+    engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    const value = (try engine.get(std.testing.allocator, "after")).?;
+    defer std.testing.allocator.free(value.bytes);
+    try std.testing.expectEqualStrings("two", value.bytes);
+    try std.testing.expectEqual(@as(u64, 2), engine.status().latest_lsn);
+}
+
+test "unrooted checkpoint and manifest do not hide journal" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "interrupted-checkpoint.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    _ = try engine.put("key", "value");
+    var roots: [8192]u8 = undefined;
+    _ = try engine.file.preadAll(&roots, 0);
+    try engine.checkpoint();
+    engine.close();
+    const file = try std.fs.cwd().openFile(path, .{ .mode = .read_write });
+    try file.pwriteAll(&roots, 0);
+    file.close();
+    engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    const value = (try engine.get(std.testing.allocator, "key")).?;
+    defer std.testing.allocator.free(value.bytes);
+    try std.testing.expectEqualStrings("value", value.bytes);
+    try std.testing.expectEqual(@as(u64, 1), engine.status().latest_lsn);
+}
+
+test "unknown compatible extent is skipped by length" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "unknown.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    _ = try engine.put("key", "value");
+    const offset = engine.status().file_bytes;
+    engine.close();
+    const file = try std.fs.cwd().openFile(path, .{ .mode = .read_write });
+    var header: [64]u8 = undefined;
+    pkvdb.encodeExtentHeader(.{ .extent_type = @enumFromInt(99), .version = 9, .payload_length = 3, .first_lsn = 0, .last_lsn = 0, .payload_crc = pkvdb.crc32c("new") }, &header);
+    try file.pwriteAll(&header, offset);
+    try file.pwriteAll("new", offset + header.len);
+    try file.pwriteAll(&([_]u8{0} ** 5), offset + header.len + 3);
+    file.close();
+    engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    try std.testing.expect(try engine.exists("key"));
+}
+
+test "repeated overwrite and delete keep directory memory bounded" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "memory.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    _ = try engine.put("same", "first");
+    const initial = engine.status().keydir_bytes;
+    for (0..100) |index| {
+        var value: [16]u8 = undefined;
+        const encoded = try std.fmt.bufPrint(&value, "{d}", .{index});
+        _ = try engine.put("same", encoded);
+    }
+    try std.testing.expect(try engine.delete("same"));
+    _ = try engine.put("same", "last");
+    try std.testing.expectEqual(initial, engine.status().keydir_bytes);
+    try std.testing.expectEqual(@as(u64, 1), engine.status().live_keys);
+}
+
+test "concurrent reads and writes preserve complete values" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "concurrent.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    _ = try engine.put("key", "00000000");
+    var failed = std.atomic.Value(bool).init(false);
+    const Writer = struct {
+        fn run(target: *Engine, failure: *std.atomic.Value(bool)) void {
+            for (0..50) |index| {
+                var value: [8]u8 = undefined;
+                _ = std.fmt.bufPrint(&value, "{d:0>8}", .{index}) catch {
+                    failure.store(true, .release);
+                    return;
+                };
+                _ = target.put("key", &value) catch {
+                    failure.store(true, .release);
+                    return;
+                };
+            }
+        }
+    };
+    const thread = try std.Thread.spawn(.{}, Writer.run, .{ &engine, &failed });
+    for (0..50) |_| {
+        const value = try engine.get(std.testing.allocator, "key") orelse return error.TestUnexpectedResult;
+        try std.testing.expectEqual(@as(usize, 8), value.bytes.len);
+        std.testing.allocator.free(value.bytes);
+    }
+    thread.join();
+    try std.testing.expect(!failed.load(.acquire));
+}
+
+fn copyPrefix(source_path: []const u8, destination_path: []const u8) !void {
+    const source = try std.fs.cwd().openFile(source_path, .{ .mode = .read_only });
+    defer source.close();
+    const length = try source.getEndPos();
+    const destination = try std.fs.cwd().createFile(destination_path, .{ .read = true, .truncate = true });
+    defer destination.close();
+    var buffer: [4096]u8 = undefined;
+    var offset: u64 = 0;
+    while (offset < length) {
+        const amount: usize = @intCast(@min(buffer.len, length - offset));
+        const got = try source.preadAll(buffer[0..amount], offset);
+        if (got == 0) break;
+        try destination.writeAll(buffer[0..got]);
+        offset += got;
+    }
+}
+
+test "every live copy recovers while writes continue" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var source_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const source = try testPath(&tmp, "live.pkvdb", &source_buffer);
+    defer std.testing.allocator.free(source);
+    var engine = try Engine.open(std.testing.allocator, source);
+    defer engine.close();
+    _ = try engine.put("seed", "value");
+    var failed = std.atomic.Value(bool).init(false);
+    const Writer = struct {
+        fn run(target: *Engine, failure: *std.atomic.Value(bool)) void {
+            for (0..30) |index| {
+                var key: [16]u8 = undefined;
+                const encoded = std.fmt.bufPrint(&key, "key-{d}", .{index}) catch {
+                    failure.store(true, .release);
+                    return;
+                };
+                _ = target.put(encoded, "value") catch {
+                    failure.store(true, .release);
+                    return;
+                };
+                if (index == 15) target.checkpoint() catch {
+                    failure.store(true, .release);
+                    return;
+                };
+            }
+        }
+    };
+    const thread = try std.Thread.spawn(.{}, Writer.run, .{ &engine, &failed });
+    for (0..8) |index| {
+        var name: [32]u8 = undefined;
+        const filename = try std.fmt.bufPrint(&name, "copy-{d}.pkvdb", .{index});
+        var copy_buffer: [std.fs.max_path_bytes]u8 = undefined;
+        const destination = try testPath(&tmp, filename, &copy_buffer);
+        defer std.testing.allocator.free(destination);
+        try copyPrefix(source, destination);
+        var copy = try Engine.open(std.testing.allocator, destination);
+        try std.testing.expect(try copy.exists("seed"));
+        copy.close();
+    }
+    thread.join();
+    try std.testing.expect(!failed.load(.acquire));
+}
+
+test "group commit preserves independent transaction LSNs" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "groups.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    var operations: [256]Operation = undefined;
+    var keys: [256][8]u8 = undefined;
+    var lsns: [256]u64 = undefined;
+    for (&operations, 0..) |*operation, index| {
+        const key = try std.fmt.bufPrint(&keys[index], "k{d}", .{index});
+        operation.* = .{ .opcode = .put, .key = key, .value = "value" };
+    }
+    try engine.putMany(&operations, &lsns);
+    const status = engine.status();
+    try std.testing.expectEqual(@as(u64, 1), status.commit_groups);
+    try std.testing.expectEqual(@as(u64, 256), status.committed_transactions);
+    for (lsns, 0..) |lsn, index| try std.testing.expectEqual(index + 1, lsn);
+    engine.close();
+    engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    try std.testing.expectEqual(@as(u64, 256), engine.status().latest_lsn);
+    for (operations) |operation| try std.testing.expect(try engine.exists(operation.key));
+}
+
+test "ordered index stays current after lazy construction" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "lazy-order.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    _ = try engine.put("p/a", "one");
+    var batch = try engine.scan(std.testing.allocator, "p/", "", 10, false, 1024);
+    batch.deinit(std.testing.allocator);
+    _ = try engine.put("p/b", "two");
+    _ = try engine.put("p/a", "updated");
+    try std.testing.expect(try engine.delete("p/b"));
+    _ = try engine.put("p/c", "three");
+    batch = try engine.scan(std.testing.allocator, "p/", "", 10, true, 1024);
+    defer batch.deinit(std.testing.allocator);
+    try std.testing.expectEqual(@as(usize, 2), batch.entries.len);
+    try std.testing.expectEqualStrings("p/a", batch.entries[0].key);
+    try std.testing.expectEqualStrings("updated", batch.entries[0].value.?);
+    try std.testing.expectEqualStrings("p/c", batch.entries[1].key);
+}
+
+test "concurrent delete reports one removal" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const path = try testPath(&tmp, "delete-race.pkvdb", &path_buffer);
+    defer std.testing.allocator.free(path);
+    var engine = try Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    _ = try engine.put("key", "value");
+    var results: [2]bool = undefined;
+    var failed = std.atomic.Value(bool).init(false);
+    const Deleter = struct {
+        fn run(target: *Engine, result: *bool, failure: *std.atomic.Value(bool)) void {
+            result.* = target.delete("key") catch {
+                failure.store(true, .release);
+                return;
+            };
+        }
+    };
+    const first = try std.Thread.spawn(.{}, Deleter.run, .{ &engine, &results[0], &failed });
+    const second = try std.Thread.spawn(.{}, Deleter.run, .{ &engine, &results[1], &failed });
+    first.join();
+    second.join();
+    try std.testing.expect(!failed.load(.acquire));
+    try std.testing.expect(results[0] != results[1]);
+}

hashing.zig

@@ -1,91 +0,0 @@
-const std = @import("std");
-
-pub fn hashKey(k: []const u8) u32 {
-    return fnv1a(k);
-}
-
-pub fn fnv1a(key: []const u8) u32 {
-    var hash: u32 = 2166136261;
-
-    for (key) |c| {
-        hash ^= c;
-        hash *%= 16777619;
-    }
-
-    return hash;
-}
-
-pub fn xoramasrosas(k: []const u8) u32 {
-    var hash: u32 = 17 * 22;
-    const x = "xoramasrosas";
-
-    for (k, 0..) |char, i| {
-        hash = hash +% (char ^ x[i % 12]) << 12;
-    }
-
-    return hash;
-}
-
-pub fn djb2(key: []const u8) u32 {
-    var hash: u32 = 5381;
-
-    for (key) |c| {
-        hash = ((hash << 5) +% hash) +% c;
-    }
-
-    return hash;
-}
-
-test "fnv1a known values" {
-    // FNV-1a 32-bit test vectors
-    try std.testing.expectEqual(@as(u32, 2166136261), fnv1a(""));
-    try std.testing.expect(fnv1a("hello") != fnv1a("world"));
-    try std.testing.expect(fnv1a("hello") != fnv1a("Hello"));
-}
-
-test "fnv1a deterministic" {
-    const h1 = fnv1a("test_key");
-    const h2 = fnv1a("test_key");
-    try std.testing.expectEqual(h1, h2);
-}
-
-test "hashKey delegates to fnv1a" {
-    try std.testing.expectEqual(fnv1a("mykey"), hashKey("mykey"));
-}
-
-test "djb2 known values" {
-    try std.testing.expectEqual(@as(u32, 5381), djb2(""));
-    try std.testing.expect(djb2("hello") != djb2("world"));
-}
-
-test "djb2 deterministic" {
-    try std.testing.expectEqual(djb2("abc"), djb2("abc"));
-}
-
-test "xoramasrosas deterministic" {
-    try std.testing.expectEqual(xoramasrosas("key"), xoramasrosas("key"));
-    try std.testing.expect(xoramasrosas("a") != xoramasrosas("b"));
-}
-
-test "different hash functions produce different results" {
-    const key = "pizzakv";
-    const f = fnv1a(key);
-    const d = djb2(key);
-    const x = xoramasrosas(key);
-    // They should generally differ (not a guarantee but practically true)
-    try std.testing.expect(f != d or f != x or d != x);
-}
-
-test "hash distribution - no trivial collisions for short keys" {
-    const keys = [_][]const u8{ "a", "b", "c", "d", "e", "f", "g", "h" };
-    var hashes: [8]u32 = undefined;
-    for (keys, 0..) |k, i| {
-        hashes[i] = fnv1a(k);
-    }
-    // All hashes should be unique for single-char keys
-    for (0..8) |i| {
-        for (i + 1..8) |j| {
-            try std.testing.expect(hashes[i] != hashes[j]);
-        }
-    }
-}

index.zig

@@ -1,405 +0,0 @@
-const std = @import("std");
-const storage = @import("storage.zig");
-
-var tree_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-const tree_allocator = tree_arena.allocator();
-
-var tree_mutex: std.Thread.Mutex = .{};
-
-const RadixNode = struct {
-    edge: []const u8,
-    children: std.StringHashMap(*RadixNode),
-    is_terminal: bool,
-
-    fn init(edge: []const u8) *RadixNode {
-        const node = tree_allocator.create(RadixNode) catch unreachable;
-        node.* = .{
-            .edge = tree_allocator.dupe(u8, edge) catch unreachable,
-            .children = std.StringHashMap(*RadixNode).init(tree_allocator),
-            .is_terminal = false,
-        };
-        return node;
-    }
-
-    fn deinit(self: *RadixNode) void {
-        var it = self.children.iterator();
-        while (it.next()) |entry| {
-            entry.value_ptr.*.deinit();
-        }
-        self.children.deinit();
-        tree_allocator.free(self.edge);
-        tree_allocator.destroy(self);
-    }
-};
-
-var root: *RadixNode = undefined;
-var root_initialized = false;
-
-fn ensureRoot() void {
-    if (!root_initialized) {
-        root = RadixNode.init("");
-        root_initialized = true;
-    }
-}
-
-fn commonPrefixLen(a: []const u8, b: []const u8) usize {
-    var i: usize = 0;
-    while (i < a.len and i < b.len and a[i] == b[i]) {
-        i += 1;
-    }
-    return i;
-}
-
-pub fn insert(key: []const u8) void {
-    tree_mutex.lock();
-    defer tree_mutex.unlock();
-
-    ensureRoot();
-    if (key.len == 0) return;
-
-    var node = root;
-    var remaining = key;
-
-    while (remaining.len > 0) {
-        var found = false;
-
-        var it = node.children.iterator();
-        while (it.next()) |entry| {
-            const child = entry.value_ptr.*;
-            const prefix_len = commonPrefixLen(child.edge, remaining);
-
-            if (prefix_len > 0) {
-                found = true;
-
-                if (prefix_len == child.edge.len) {
-                    if (prefix_len == remaining.len) {
-                        child.is_terminal = true;
-                        return;
-                    }
-                    remaining = remaining[prefix_len..];
-                    node = child;
-                    break;
-                } else {
-                    const old_edge = child.edge;
-                    const key_suffix = remaining[prefix_len..];
-
-                    // Dupe before freeing old_edge
-                    const child_suffix = tree_allocator.dupe(u8, old_edge[prefix_len..]) catch unreachable;
-                    const intermediate = RadixNode.init(old_edge[0..prefix_len]);
-
-                    // Remove old entry before freeing
-                    _ = node.children.remove(old_edge);
-                    tree_allocator.free(old_edge);
-
-                    child.edge = child_suffix;
-                    intermediate.children.put(child_suffix, child) catch unreachable;
-                    node.children.put(intermediate.edge, intermediate) catch unreachable;
-
-                    if (key_suffix.len == 0) {
-                        intermediate.is_terminal = true;
-                        return;
-                    } else {
-                        const new_child = RadixNode.init(key_suffix);
-                        new_child.is_terminal = true;
-                        intermediate.children.put(key_suffix, new_child) catch unreachable;
-                        return;
-                    }
-                }
-            }
-        }
-
-        if (!found) {
-            const new_child = RadixNode.init(remaining);
-            new_child.is_terminal = true;
-            node.children.put(remaining, new_child) catch unreachable;
-            return;
-        }
-    }
-
-    node.is_terminal = true;
-}
-
-pub fn delete(key: []const u8) void {
-    tree_mutex.lock();
-    defer tree_mutex.unlock();
-
-    ensureRoot();
-    if (key.len == 0) return;
-
-    const node = findNode(root, key);
-    if (node) |n| {
-        n.is_terminal = false;
-    }
-}
-
-fn findNodeForPrefix(node: *RadixNode, prefix: []const u8, path_buf: *[MAX_KEY_LENGTH]u8, path_len: *usize) ?*RadixNode {
-    if (prefix.len == 0) {
-        path_len.* = 0;
-        return node;
-    }
-
-    var current = node;
-    var remaining = prefix;
-    path_len.* = 0;
-
-    while (remaining.len > 0) {
-        var found = false;
-
-        var it = current.children.iterator();
-        while (it.next()) |entry| {
-            const child = entry.value_ptr.*;
-            const prefix_match_len = commonPrefixLen(child.edge, remaining);
-
-            if (prefix_match_len > 0) {
-                if (prefix_match_len == remaining.len) {
-                    return child;
-                }
-
-                if (prefix_match_len == child.edge.len) {
-                    if (path_len.* + prefix_match_len > MAX_KEY_LENGTH) return null;
-                    @memcpy(path_buf[path_len.* .. path_len.* + prefix_match_len], child.edge[0..prefix_match_len]);
-                    path_len.* += prefix_match_len;
-                    remaining = remaining[prefix_match_len..];
-                    current = child;
-                    found = true;
-                    break;
-                }
-
-                return null;
-            }
-        }
-
-        if (!found) {
-            return null;
-        }
-    }
-
-    return current;
-}
-
-fn findNode(node: *RadixNode, key: []const u8) ?*RadixNode {
-    var dummy_buf: [MAX_KEY_LENGTH]u8 = undefined;
-    var dummy_len: usize = 0;
-    return findNodeForPrefix(node, key, &dummy_buf, &dummy_len);
-}
-
-pub fn searchByPrefix(prefix: []const u8) ?*RadixNode {
-    ensureRoot();
-    if (prefix.len == 0) return root;
-    return findNode(root, prefix);
-}
-
-fn countKeys(node: *RadixNode) usize {
-    var count: usize = 0;
-    if (node.is_terminal) {
-        count += 1;
-    }
-
-    var it = node.children.iterator();
-    while (it.next()) |entry| {
-        count += countKeys(entry.value_ptr.*);
-    }
-    return count;
-}
-
-const MAX_KEYS_RETURN = 100_000_000;
-const MAX_KEY_LENGTH = 1024;
-
-fn collectKeysWithBuffer(node: *RadixNode, prefix_buffer: []u8, prefix_len: usize, keys: *std.ArrayListUnmanaged([]const u8), max_keys: usize, search_prefix: []const u8, include_node_edge: bool, allocator: std.mem.Allocator) void {
-    if (keys.items.len >= max_keys) return;
-
-    var current_len = prefix_len;
-
-    if (include_node_edge and node.edge.len > 0) {
-        if (current_len + node.edge.len > MAX_KEY_LENGTH) return;
-        @memcpy(prefix_buffer[current_len .. current_len + node.edge.len], node.edge);
-        current_len += node.edge.len;
-    }
-
-    if (node.is_terminal) {
-        const key = prefix_buffer[0..current_len];
-        if (key.len >= search_prefix.len and std.mem.eql(u8, key[0..search_prefix.len], search_prefix)) {
-            const key_copy = allocator.dupe(u8, key) catch return;
-            keys.append(allocator, key_copy) catch return;
-        }
-    }
-
-    var it = node.children.iterator();
-    while (it.next()) |entry| {
-        if (keys.items.len >= max_keys) break;
-        const child = entry.value_ptr.*;
-
-        collectKeysWithBuffer(child, prefix_buffer, current_len, keys, max_keys, search_prefix, true, allocator);
-    }
-}
-
-fn collectKeys(node: *RadixNode, prefix: []const u8, keys: *std.ArrayListUnmanaged([]const u8), max_keys: usize, search_prefix: []const u8, allocator: std.mem.Allocator) void {
-    var prefix_buffer: [MAX_KEY_LENGTH]u8 = undefined;
-    if (prefix.len > MAX_KEY_LENGTH) return;
-    @memcpy(prefix_buffer[0..prefix.len], prefix);
-    collectKeysWithBuffer(node, &prefix_buffer, prefix.len, keys, max_keys, search_prefix, true, allocator);
-}
-
-pub fn getKeysFromNode(node: *RadixNode, prefix: []const u8, allocator: std.mem.Allocator) [][]const u8 {
-    var keys_list = std.ArrayListUnmanaged([]const u8){};
-    collectKeys(node, prefix, &keys_list, MAX_KEYS_RETURN, prefix, allocator);
-    return keys_list.toOwnedSlice(allocator) catch &[_][]const u8{};
-}
-
-pub fn getKeysByPrefix(prefix: []const u8, allocator: std.mem.Allocator) []const u8 {
-    tree_mutex.lock();
-    defer tree_mutex.unlock();
-
-    ensureRoot();
-    const node = searchByPrefix(prefix) orelse return "";
-    const keys = getKeysFromNode(node, prefix, allocator);
-    if (keys.len == 0) return "";
-    return std.mem.join(allocator, "\n", keys) catch "";
-}
-
-pub fn getValuesByPrefix(prefix: []const u8, allocator: std.mem.Allocator) []const u8 {
-    // Phase 1: Collect matching keys under tree_mutex
-    var keys: [][]const u8 = &[_][]const u8{};
-    {
-        tree_mutex.lock();
-        defer tree_mutex.unlock();
-
-        ensureRoot();
-
-        var path_buf: [MAX_KEY_LENGTH]u8 = undefined;
-        var path_len: usize = 0;
-        const node = findNodeForPrefix(root, prefix, &path_buf, &path_len) orelse {
-            return "";
-        };
-
-        var keys_list = std.ArrayListUnmanaged([]const u8){};
-        collectKeys(node, path_buf[0..path_len], &keys_list, MAX_KEYS_RETURN, prefix, allocator);
-        keys = keys_list.toOwnedSlice(allocator) catch &[_][]const u8{};
-    }
-
-    if (keys.len == 0) return "";
-
-    // Phase 2: Read values without tree_mutex to avoid deadlock with write/delete
-    const values = allocator.alloc([]const u8, keys.len) catch return "";
-    for (keys, 0..) |key, i| {
-        const value = storage.readAlloc(key, allocator) orelse "";
-        values[i] = value;
-    }
-
-    return std.mem.join(allocator, "\n", values) catch "";
-}
-
-pub fn getAllKeys(allocator: std.mem.Allocator) []const u8 {
-    tree_mutex.lock();
-    defer tree_mutex.unlock();
-
-    ensureRoot();
-    const keys = getKeysFromNode(root, &[_]u8{}, allocator);
-    if (keys.len == 0) return "";
-    return std.mem.join(allocator, "\n", keys) catch "";
-}
-
-// -- Tests --
-
-const test_allocator = std.heap.page_allocator;
-
-test "commonPrefixLen" {
-    try std.testing.expectEqual(@as(usize, 3), commonPrefixLen("abc", "abcdef"));
-    try std.testing.expectEqual(@as(usize, 3), commonPrefixLen("abcdef", "abc"));
-    try std.testing.expectEqual(@as(usize, 0), commonPrefixLen("abc", "xyz"));
-    try std.testing.expectEqual(@as(usize, 0), commonPrefixLen("", "abc"));
-    try std.testing.expectEqual(@as(usize, 0), commonPrefixLen("abc", ""));
-    try std.testing.expectEqual(@as(usize, 5), commonPrefixLen("hello", "hello"));
-}
-
-test "insert and searchByPrefix" {
-    insert("idx_apple");
-    insert("idx_app");
-    insert("idx_banana");
-
-    try std.testing.expect(searchByPrefix("idx_apple") != null);
-    try std.testing.expect(searchByPrefix("idx_banana") != null);
-    try std.testing.expect(searchByPrefix("idx_xyz") == null);
-}
-
-test "insert duplicate key does not crash" {
-    insert("idx_dup");
-    insert("idx_dup");
-    // Node should exist and be terminal
-    const node = searchByPrefix("idx_dup");
-    try std.testing.expect(node != null);
-    try std.testing.expect(node.?.is_terminal);
-}
-
-test "delete marks non-terminal" {
-    insert("idx_delme");
-    const node_before = searchByPrefix("idx_delme");
-    try std.testing.expect(node_before != null);
-    try std.testing.expect(node_before.?.is_terminal);
-
-    delete("idx_delme");
-
-    const node_after = searchByPrefix("idx_delme");
-    try std.testing.expect(node_after != null);
-    try std.testing.expect(!node_after.?.is_terminal);
-}
-
-test "getAllKeys returns inserted keys" {
-    insert("idx_all_a");
-    insert("idx_all_b");
-
-    const result = getAllKeys(test_allocator);
-
-    try std.testing.expect(result.len > 0);
-    try std.testing.expect(std.mem.indexOf(u8, result, "idx_all_a") != null);
-    try std.testing.expect(std.mem.indexOf(u8, result, "idx_all_b") != null);
-}
-
-test "insert empty key is no-op" {
-    insert("");
-}
-
-test "radix tree prefix splitting" {
-    insert("idx_test");
-    insert("idx_testing");
-    insert("idx_tested");
-    insert("idx_tester");
-
-    // All four keys should be findable
-    try std.testing.expect(searchByPrefix("idx_test") != null);
-    try std.testing.expect(searchByPrefix("idx_testing") != null);
-    try std.testing.expect(searchByPrefix("idx_tested") != null);
-    try std.testing.expect(searchByPrefix("idx_tester") != null);
-
-    // Verify terminals
-    const node_test = searchByPrefix("idx_test");
-    try std.testing.expect(node_test.?.is_terminal);
-    const node_testing = searchByPrefix("idx_testing");
-    try std.testing.expect(node_testing.?.is_terminal);
-}
-
-test "searchByPrefix returns null for missing prefix" {
-    try std.testing.expect(searchByPrefix("zzz_nonexistent") == null);
-}
-
-test "countKeys counts terminal nodes" {
-    ensureRoot();
-    insert("idx_cnt_a");
-    insert("idx_cnt_b");
-    insert("idx_cnt_c");
-
-    const node = searchByPrefix("idx_cnt") orelse return error.TestUnexpectedResult;
-    const count = countKeys(node);
-    try std.testing.expect(count >= 3);
-}
-
-test "getValuesByPrefix with storage" {
-    storage.init();
-    _ = storage.restore("idx_pv_key1", "val1");
-    _ = storage.restore("idx_pv_key2", "val2");
-
-    const result = getValuesByPrefix("idx_pv_key", test_allocator);
-    try std.testing.expect(result.len > 0);
-    try std.testing.expect(std.mem.indexOf(u8, result, "val1") != null);
-    try std.testing.expect(std.mem.indexOf(u8, result, "val2") != null);
-}

keydir.zig

@@ -0,0 +1,233 @@
+const std = @import("std");
+
+pub const RecordRef = struct {
+    hash: u64,
+    lsn: u64,
+    key_offset: u64,
+    value_offset: u64,
+    key_len: u32,
+    value_len: u32,
+    flags: u32 = 0,
+    reserved: u32 = 0,
+};
+
+pub const Reader = struct {
+    context: *const anyopaque,
+    readFn: *const fn (*const anyopaque, []u8, u64) anyerror!usize,
+
+    fn read(self: Reader, destination: []u8, offset: u64) !usize {
+        return self.readFn(self.context, destination, offset);
+    }
+};
+
+const Slot = struct {
+    record: RecordRef = undefined,
+    distance: u32 = 0,
+    used: bool = false,
+};
+
+pub const KeyDir = struct {
+    allocator: std.mem.Allocator,
+    slots: []Slot,
+    count: usize = 0,
+
+    pub fn init(allocator: std.mem.Allocator) !KeyDir {
+        const slots = try allocator.alloc(Slot, 16);
+        @memset(slots, .{});
+        return .{ .allocator = allocator, .slots = slots };
+    }
+
+    pub fn deinit(self: *KeyDir) void {
+        self.allocator.free(self.slots);
+        self.* = undefined;
+    }
+
+    pub fn hash(key: []const u8) u64 {
+        var value: u64 = 14695981039346656037;
+        for (key) |byte| {
+            value ^= byte;
+            value *%= 1099511628211;
+        }
+        return value;
+    }
+
+    fn keysEqual(file: std.fs.File, record: RecordRef, key: []const u8) !bool {
+        if (record.key_len != key.len) return false;
+        var buffer: [4096]u8 = undefined;
+        var done: usize = 0;
+        while (done < key.len) {
+            const amount = @min(buffer.len, key.len - done);
+            const offset = try std.math.add(u64, record.key_offset, done);
+            const got = try file.preadAll(buffer[0..amount], offset);
+            if (got != amount or !std.mem.eql(u8, buffer[0..amount], key[done .. done + amount])) return false;
+            done += amount;
+        }
+        return true;
+    }
+
+    fn keysEqualWithReader(reader: Reader, record: RecordRef, key: []const u8) !bool {
+        if (record.key_len != key.len) return false;
+        var buffer: [4096]u8 = undefined;
+        var done: usize = 0;
+        while (done < key.len) {
+            const amount = @min(buffer.len, key.len - done);
+            const offset = try std.math.add(u64, record.key_offset, done);
+            const got = try reader.read(buffer[0..amount], offset);
+            if (got != amount or !std.mem.eql(u8, buffer[0..amount], key[done .. done + amount])) return false;
+            done += amount;
+        }
+        return true;
+    }
+
+    fn recordsEqual(file: std.fs.File, a: RecordRef, b: RecordRef) !bool {
+        if (a.key_len != b.key_len) return false;
+        var left: [4096]u8 = undefined;
+        var right: [4096]u8 = undefined;
+        var done: usize = 0;
+        while (done < a.key_len) {
+            const amount = @min(left.len, a.key_len - done);
+            const left_offset = try std.math.add(u64, a.key_offset, done);
+            const right_offset = try std.math.add(u64, b.key_offset, done);
+            if (try file.preadAll(left[0..amount], left_offset) != amount) return error.Truncated;
+            if (try file.preadAll(right[0..amount], right_offset) != amount) return error.Truncated;
+            if (!std.mem.eql(u8, left[0..amount], right[0..amount])) return false;
+            done += amount;
+        }
+        return true;
+    }
+
+    fn grow(self: *KeyDir, file: std.fs.File) anyerror!void {
+        const old = self.slots;
+        const old_count = self.count;
+        const slots = try self.allocator.alloc(Slot, try std.math.mul(usize, old.len, 2));
+        @memset(slots, .{});
+        self.slots = slots;
+        self.count = 0;
+        errdefer {
+            self.allocator.free(slots);
+            self.slots = old;
+            self.count = old_count;
+        }
+        for (old) |slot| if (slot.used) try self.put(file, slot.record);
+        self.allocator.free(old);
+    }
+
+    pub fn ensureAdditional(self: *KeyDir, file: std.fs.File, additional: usize) anyerror!void {
+        const desired = try std.math.add(usize, self.count, additional);
+        while (desired >= self.slots.len - self.slots.len / 5) try self.grow(file);
+    }
+
+    pub fn clear(self: *KeyDir) void {
+        @memset(self.slots, .{});
+        self.count = 0;
+    }
+
+    pub fn put(self: *KeyDir, file: std.fs.File, record: RecordRef) anyerror!void {
+        return self.putInternal(file, record, null);
+    }
+
+    pub fn putWithKey(self: *KeyDir, file: std.fs.File, key: []const u8, record: RecordRef) anyerror!void {
+        return self.putInternal(file, record, key);
+    }
+
+    fn putInternal(self: *KeyDir, file: std.fs.File, record: RecordRef, key: ?[]const u8) anyerror!void {
+        if (self.count + 1 >= self.slots.len - self.slots.len / 5) try self.grow(file);
+        var incoming = Slot{ .record = record, .used = true };
+        var index: usize = @intCast(record.hash & (self.slots.len - 1));
+        while (true) {
+            const slot = &self.slots[index];
+            if (!slot.used) {
+                slot.* = incoming;
+                self.count += 1;
+                return;
+            }
+            if (slot.record.hash == record.hash and if (key) |key_bytes| try keysEqual(file, slot.record, key_bytes) else try recordsEqual(file, slot.record, record)) {
+                slot.record = record;
+                return;
+            }
+            if (slot.distance < incoming.distance) std.mem.swap(Slot, slot, &incoming);
+            incoming.distance += 1;
+            index = (index + 1) & (self.slots.len - 1);
+        }
+    }
+
+    fn findIndex(self: *const KeyDir, file: std.fs.File, key: []const u8, key_hash: u64) !?usize {
+        var index: usize = @intCast(key_hash & (self.slots.len - 1));
+        var distance: u32 = 0;
+        while (true) {
+            const slot = self.slots[index];
+            if (!slot.used or slot.distance < distance) return null;
+            if (slot.record.hash == key_hash and try keysEqual(file, slot.record, key)) return index;
+            distance += 1;
+            index = (index + 1) & (self.slots.len - 1);
+        }
+    }
+
+    pub fn get(self: *const KeyDir, file: std.fs.File, key: []const u8) !?RecordRef {
+        const index = try self.findIndex(file, key, hash(key)) orelse return null;
+        return self.slots[index].record;
+    }
+
+    pub fn getWithReader(self: *const KeyDir, reader: Reader, key: []const u8) !?RecordRef {
+        const key_hash = hash(key);
+        var index: usize = @intCast(key_hash & (self.slots.len - 1));
+        var distance: u32 = 0;
+        while (true) {
+            const slot = self.slots[index];
+            if (!slot.used or slot.distance < distance) return null;
+            if (slot.record.hash == key_hash and try keysEqualWithReader(reader, slot.record, key)) return slot.record;
+            distance += 1;
+            index = (index + 1) & (self.slots.len - 1);
+        }
+    }
+
+    pub fn remove(self: *KeyDir, file: std.fs.File, key: []const u8) !bool {
+        var index = try self.findIndex(file, key, hash(key)) orelse return false;
+        while (true) {
+            const next = (index + 1) & (self.slots.len - 1);
+            if (!self.slots[next].used or self.slots[next].distance == 0) {
+                self.slots[index] = .{};
+                break;
+            }
+            self.slots[index] = self.slots[next];
+            self.slots[index].distance -= 1;
+            index = next;
+        }
+        self.count -= 1;
+        return true;
+    }
+
+    pub fn records(self: *const KeyDir, allocator: std.mem.Allocator) ![]RecordRef {
+        const result = try allocator.alloc(RecordRef, self.count);
+        var index: usize = 0;
+        for (self.slots) |slot| if (slot.used) {
+            result[index] = slot.record;
+            index += 1;
+        };
+        return result;
+    }
+
+    pub fn bytes(self: *const KeyDir) usize {
+        return self.slots.len * @sizeOf(Slot);
+    }
+};
+
+test "hash is 64 bit and deterministic" {
+    try std.testing.expectEqual(KeyDir.hash("pizza"), KeyDir.hash("pizza"));
+    try std.testing.expect(KeyDir.hash("pizza") != KeyDir.hash("pizzb"));
+}
+
+test "colliding hashes compare immutable key bytes" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    const file = try tmp.dir.createFile("keys", .{ .read = true });
+    defer file.close();
+    try file.writeAll("alphaomega");
+    var directory = try KeyDir.init(std.testing.allocator);
+    defer directory.deinit();
+    try directory.put(file, .{ .hash = 42, .lsn = 1, .key_offset = 0, .value_offset = 0, .key_len = 5, .value_len = 0 });
+    try directory.put(file, .{ .hash = 42, .lsn = 2, .key_offset = 5, .value_offset = 0, .key_len = 5, .value_len = 0 });
+    try std.testing.expect((try directory.findIndex(file, "alpha", 42)) != null);
+    try std.testing.expect((try directory.findIndex(file, "omega", 42)) != null);
+    try std.testing.expectEqual(@as(usize, 2), directory.count);
+}

main.zig

@@ -1,293 +1,381 @@
 const std = @import("std");
-
-const net = std.net;
+const builtin = @import("builtin");
 const posix = std.posix;
-const fmt = std.fmt;
-
 const socket = @import("socket.zig");
-const command = @import("command.zig");
-const storage = @import("storage.zig");
-const persistence = @import("persistence.zig");
-const redis = @import("redis.zig");
+const Engine = @import("engine.zig").Engine;
+const pizzaria = @import("command.zig");
+const resp = @import("redis.zig");
+const pkbfi = @import("pkbfi.zig");
+const migration = @import("migration.zig");
+
+const initial_buffer = 64 * 1024;
+const max_connection_bytes = 512 * 1024 * 1024;
+const max_connections = 256;
 
-const builtin = @import("builtin");
-const TCP = switch (builtin.target.os.tag) {
-    .linux, .macos => posix.TCP,
-    else => struct {
-        pub const NODELAY: c_int = 1;
-        pub const CORK: c_int = 3;
-        pub const NOPUSH: c_int = 4;
-    },
-};
-//main.zig:23:12: error: variable of type 'comptime_int' must be const or comptime
-// var PORT = 8085;
-var PORT: u16 = 8085;
-var HOST: []const u8 = "127.0.0.1";
 var should_exit = std.atomic.Value(bool).init(false);
 var active_connections = std.atomic.Value(u32).init(0);
-var redis_mode = false;
-var instant_wal_mode = false;
-var unix_mode = false;
+var memory_mutex: std.Thread.Mutex = .{};
+var allocated_connection_bytes: usize = 0;
 
-fn handleSignal(sig: c_int) callconv(.c) void {
-    _ = sig;
-    should_exit.store(true, .seq_cst);
+fn signalHandler(_: c_int) callconv(.c) void {
+    should_exit.store(true, .release);
 }
 
-pub fn main() !void {
-    var args = try std.process.argsWithAllocator(std.heap.page_allocator);
-    defer args.deinit();
-
-    _ = args.skip();
-    while (args.next()) |arg| {
-        if (std.mem.eql(u8, arg, "-redis")) {
-            redis_mode = true;
-        } else if (std.mem.eql(u8, arg, "-unix")) {
-            unix_mode = true;
-        } else if (std.mem.eql(u8, arg, "-iwal")) {
-            instant_wal_mode = true;
-        } else if (arg.len > 6 and std.mem.eql(u8, arg[0..6], "-port=")) {
-            const port_str = arg[6..];
-            const parsed_port = try std.fmt.parseInt(u16, port_str, 10);
-            if (parsed_port != 0) {
-                PORT = parsed_port;
-            } else {
-                std.debug.print("Invalid port number: {any}\n", .{port_str});
-                return;
-            }
-        } else if (arg.len > 6 and std.mem.eql(u8, arg[0..6], "-host=")) {
-            HOST = arg[6..];
-        } else {
-            std.debug.print("Unknown argument: {any}\n", .{arg});
-            return;
-        }
-    }
-
-    const empty_mask = std.mem.zeroes(posix.sigset_t);
-    const act = posix.Sigaction{
-        .handler = .{ .handler = handleSignal },
-        .mask = empty_mask,
-        .flags = 0,
-    };
-
-    _ = posix.sigaction(posix.SIG.TERM, &act, null);
-    _ = posix.sigaction(posix.SIG.INT, &act, null);
-
-    storage.init();
-    try persistence.init();
-
-    if (instant_wal_mode) {
-        persistence.setInstantWal(true);
-        std.debug.print("\nInstant WAL mode enabled\n", .{});
-    }
-
-    const unix_path = ".pizzakv.sock";
-    const listener = if (unix_mode) try socket.initUnix(unix_path) else try socket.init(HOST, PORT);
-    defer posix.close(listener);
-    defer if (unix_mode) posix.unlink(unix_path) catch {};
-
-    if (unix_mode) {
-        std.debug.print("\n2025 pizzakv! Unix socket at {s}\n<danilo@fragoso.dev>\n---------\n", .{unix_path});
-    } else {
-        std.debug.print("\n2025 pizzakv! TCP Listening on {s}:{any}\n<danilo@fragoso.dev>\n---------\n", .{ HOST, PORT });
-    }
-    if (redis_mode) {
-        std.debug.print("Mode: Redis Protocol (RESP)\nCommands: SET, GET, DEL\n", .{});
-    } else {
-        std.debug.print("Commands:\n\nread key\nwrite key|value\ndelete key\nkeys\nreads prefix\nstatus\n", .{});
-    }
-    std.debug.print("---------\n", .{});
-
-    while (!should_exit.load(.seq_cst)) {
-        var poll_fds = [_]posix.pollfd{
-            .{
-                .fd = listener,
-                .events = posix.POLL.IN,
-                .revents = 0,
-            },
-        };
-
-        const ready = posix.poll(&poll_fds, 100) catch |err| {
-            if (should_exit.load(.seq_cst)) break;
-            std.debug.print("poll error: {any}\n", .{err});
-            continue;
-        };
-
-        if (ready == 0) {
-            continue;
-        }
-
-        if (should_exit.load(.seq_cst)) break;
+fn reserveMemory(engine: *Engine, amount: usize) !void {
+    memory_mutex.lock();
+    defer memory_mutex.unlock();
+    if (amount > max_connection_bytes - allocated_connection_bytes) return error.Backpressure;
+    allocated_connection_bytes += amount;
+    engine.addConnectionBytes(amount);
+}
 
-        var client_address: net.Address = undefined;
-        var client_address_len: posix.socklen_t = @sizeOf(net.Address);
+fn releaseMemory(engine: *Engine, amount: usize) void {
+    memory_mutex.lock();
+    std.debug.assert(amount <= allocated_connection_bytes);
+    allocated_connection_bytes -= amount;
+    memory_mutex.unlock();
+    engine.removeConnectionBytes(amount);
+}
 
-        const conn = posix.accept(listener, &client_address.any, &client_address_len, 0) catch |err| {
-            if (should_exit.load(.seq_cst)) break;
-            std.debug.print("error accept: {any}\n", .{err});
-            continue;
+fn sendAll(connection: posix.socket_t, bytes: []const u8) !void {
+    var position: usize = 0;
+    while (position < bytes.len) {
+        const written = posix.send(connection, bytes[position..], posix.MSG.NOSIGNAL) catch |err| switch (err) {
+            error.WouldBlock => return error.WriteTimedOut,
+            else => return err,
         };
-
-        if (should_exit.load(.seq_cst)) {
-            posix.close(conn);
-            break;
-        }
-
-        if (!unix_mode) posix.setsockopt(conn, posix.IPPROTO.TCP, TCP.NODELAY, &std.mem.toBytes(@as(c_int, 1))) catch {};
-        socket.setReadTimeout(conn, 300) catch {}; // 5 minutes
-        socket.setWriteTimeout(conn, 300) catch {}; // 5 minutes
-
-        if (redis_mode) {
-            const thread = try std.Thread.spawn(.{}, handleRedisConnection, .{conn});
-            thread.detach();
-        } else {
-            const thread = try std.Thread.spawn(.{}, handleConnection, .{conn});
-            thread.detach();
-        }
+        if (written == 0) return error.ConnectionClosed;
+        position += written;
     }
+}
 
-    std.debug.print("\nShutdown signal received...\n", .{});
-
-    const max_wait_ms = 5000;
-    const wait_interval_ms = 100;
-    var waited_ms: u32 = 0;
-
-    while (active_connections.load(.seq_cst) > 0 and waited_ms < max_wait_ms) {
-        posix.nanosleep(0, wait_interval_ms * std.time.ns_per_ms);
-        waited_ms += wait_interval_ms;
+fn handleResp(engine: *Engine, connection: posix.socket_t, command: resp.Command) !void {
+    if (command.command_type == .get) return handleRespGet(engine, connection, command.key);
+    var response = try resp.execute(engine, std.heap.smp_allocator, command);
+    defer response.deinit(std.heap.smp_allocator);
+    if (response == .bulk) {
+        try reserveMemory(engine, response.bulk.bytes.len);
+        defer releaseMemory(engine, response.bulk.bytes.len);
     }
-
-    const remaining = active_connections.load(.seq_cst);
-    if (remaining > 0) {
-        std.debug.print("Warning: {d} connections still active after {d}ms, forcing shutdown...\n", .{ remaining, max_wait_ms });
+    var prefix: [64]u8 = undefined;
+    const encoded = try resp.encodePrefix(response, &prefix);
+    try sendAll(connection, encoded);
+    if (response == .bulk) {
+        try sendAll(connection, response.bulk.bytes);
+        try sendAll(connection, "\r\n");
     }
+}
 
-    persistence.flush() catch |err| {
-        std.debug.print("Failed to flush persistence: {any}\n", .{err});
+fn handleRespGet(engine: *Engine, connection: posix.socket_t, key: []const u8) !void {
+    const record = try engine.getRef(key) orelse {
+        try sendAll(connection, "$-1\r\n");
+        return;
     };
+    var header: [32]u8 = undefined;
+    try sendAll(connection, try std.fmt.bufPrint(&header, "${d}\r\n", .{record.value_len}));
+    const buffer_size = @min(@as(usize, 64 * 1024), record.value_len);
+    try reserveMemory(engine, buffer_size);
+    defer releaseMemory(engine, buffer_size);
+    const buffer = try std.heap.smp_allocator.alloc(u8, buffer_size);
+    defer std.heap.smp_allocator.free(buffer);
+    var position: u32 = 0;
+    while (position < record.value_len) {
+        const amount = try engine.readValue(record, buffer, position);
+        try sendAll(connection, buffer[0..amount]);
+        position += @intCast(amount);
+    }
+    try sendAll(connection, "\r\n");
 }
 
-pub fn handleConnection(conn: posix.socket_t) !void {
-    _ = active_connections.fetchAdd(1, .seq_cst);
-    defer _ = active_connections.fetchSub(1, .seq_cst);
-    defer posix.close(conn);
-
-    var requestBuffer: [1024 * 1024]u8 = undefined;
-    var response_arena = std.heap.ArenaAllocator.init(std.heap.c_allocator);
-    defer response_arena.deinit();
-
-    while (true) {
-        const n = socket.readUntilCR(conn, &requestBuffer) catch |err| {
-            if (err == error.ConnectionClosed) break;
-            if (err == error.WouldBlock) continue;
-            return err;
-        };
-        if (n == 0) {
-            break;
-        }
-
-        const cmdResponse = command.parse(requestBuffer[0..n], response_arena.allocator()) orelse {
-            socket.write(conn, "error\r") catch |err| {
-                std.debug.print("error writing: {any}", .{err});
-            };
+fn handleRespGets(engine: *Engine, connection: posix.socket_t, keys: []const []const u8) !void {
+    const capacity = 1024 * 1024;
+    try reserveMemory(engine, capacity);
+    defer releaseMemory(engine, capacity);
+    const output = try std.heap.smp_allocator.alloc(u8, capacity);
+    defer std.heap.smp_allocator.free(output);
+    var position: usize = 0;
+    for (keys) |key| {
+        const record = try engine.getRef(key) orelse {
+            if (position + 5 > output.len) {
+                try sendAll(connection, output[0..position]);
+                position = 0;
+            }
+            @memcpy(output[position .. position + 5], "$-1\r\n");
+            position += 5;
             continue;
         };
-
-        const terminator = "\r";
-        const iovecs = [_]posix.iovec_const{
-            .{ .base = cmdResponse.ptr, .len = cmdResponse.len },
-            .{ .base = terminator.ptr, .len = 1 },
-        };
-        socket.writev(conn, &iovecs) catch |err| {
-            std.debug.print("error writing: {any}", .{err});
-        };
-
-        // Free temporary allocations from this request
-        _ = response_arena.reset(.retain_capacity);
+        var header: [32]u8 = undefined;
+        const encoded_header = try std.fmt.bufPrint(&header, "${d}\r\n", .{record.value_len});
+        const needed = encoded_header.len + record.value_len + 2;
+        if (needed > output.len) {
+            if (position != 0) {
+                try sendAll(connection, output[0..position]);
+                position = 0;
+            }
+            try handleRespGet(engine, connection, key);
+            continue;
+        }
+        if (position + needed > output.len) {
+            try sendAll(connection, output[0..position]);
+            position = 0;
+        }
+        @memcpy(output[position .. position + encoded_header.len], encoded_header);
+        position += encoded_header.len;
+        const amount = try engine.readValue(record, output[position .. position + record.value_len], 0);
+        position += amount;
+        @memcpy(output[position .. position + 2], "\r\n");
+        position += 2;
     }
+    if (position != 0) try sendAll(connection, output[0..position]);
 }
 
-fn sendAll(conn: posix.socket_t, buf: []const u8) !void {
-    var offset: usize = 0;
-    while (offset < buf.len) {
-        const sent = try posix.send(conn, buf[offset..], posix.MSG.NOSIGNAL);
-        if (sent == 0) return error.ConnectionClosed;
-        offset += sent;
+fn handleConnection(engine: *Engine, connection: posix.socket_t) void {
+    defer _ = active_connections.fetchSub(1, .monotonic);
+    defer posix.close(connection);
+    reserveMemory(engine, initial_buffer) catch return;
+    var buffer = std.heap.smp_allocator.alloc(u8, initial_buffer) catch {
+        releaseMemory(engine, initial_buffer);
+        return;
+    };
+    defer {
+        std.heap.smp_allocator.free(buffer);
+        releaseMemory(engine, buffer.len);
     }
-}
-
-pub fn handleRedisConnection(conn: posix.socket_t) !void {
-    _ = active_connections.fetchAdd(1, .seq_cst);
-    defer _ = active_connections.fetchSub(1, .seq_cst);
-    defer posix.close(conn);
-
-    var requestBuffer: [2 * 1024 * 1024]u8 = undefined;
-    var responseBuffer: [2 * 1024 * 1024]u8 = undefined;
-    var buffered_len: usize = 0;
-
-    const is_darwin = builtin.target.os.tag == .macos;
-    const cork_option = if (is_darwin) TCP.NOPUSH else TCP.CORK;
-
-    while (true) {
-        const n = posix.read(conn, requestBuffer[buffered_len..]) catch |err| {
-            if (err == error.ConnectionResetByPeer) break;
-            return err;
-        };
-        if (n == 0) break;
-
-        const total_len = buffered_len + n;
-        var offset: usize = 0;
-        var response_offset: usize = 0;
-
-        posix.setsockopt(conn, posix.IPPROTO.TCP, cork_option, &std.mem.toBytes(@as(c_int, 1))) catch {};
-
-        while (offset < total_len) {
-            const result = redis.parseCommand(requestBuffer[offset..total_len]) orelse {
-                break;
+    var session = pkbfi.Session.init(std.heap.smp_allocator);
+    defer session.deinit();
+    var buffered: usize = 0;
+    while (!should_exit.load(.acquire)) {
+        if (buffered == buffer.len) {
+            const maximum: usize = if (buffered >= 4 and std.mem.eql(u8, buffer[0..4], "PKBF")) pkbfi.max_frame_size + pkbfi.header_size else if (buffered > 0 and buffer[0] == '*') pkbfi.max_frame_size + 1024 else 1024 * 1024;
+            if (buffer.len >= maximum) return;
+            const next = @min(maximum, buffer.len * 2);
+            reserveMemory(engine, next - buffer.len) catch return;
+            buffer = std.heap.smp_allocator.realloc(buffer, next) catch {
+                releaseMemory(engine, next - buffer.len);
+                return;
             };
-
-            // Mutating commands have small responses. Flush first when the
-            // remaining output slice is small so SET/DEL are never retried
-            // after their side effect has already happened.
-            if (response_offset > 0 and responseBuffer.len - response_offset < 64 and result.cmd.cmd_type != .GET) {
-                try sendAll(conn, responseBuffer[0..response_offset]);
-                response_offset = 0;
-            }
-
-            var response = redis.executeCommand(result.cmd, responseBuffer[response_offset..]);
-            if (response == null) {
-                if (response_offset > 0) {
-                    try sendAll(conn, responseBuffer[0..response_offset]);
-                    response_offset = 0;
+        }
+        const amount = posix.read(connection, buffer[buffered..]) catch |err| switch (err) {
+            error.WouldBlock => return,
+            error.ConnectionResetByPeer => return,
+            else => return,
+        };
+        if (amount == 0) return;
+        buffered += amount;
+        var consumed: usize = 0;
+        while (consumed < buffered) {
+            const input = buffer[consumed..buffered];
+            if (input.len >= 4 and std.mem.eql(u8, input[0..4], "PKBF")) {
+                const frame = pkbfi.parse(input) catch |err| switch (err) {
+                    error.Incomplete => break,
+                    else => return,
+                };
+                if (frame.opcode == .put) {
+                    var operations: [256]@import("engine.zig").Operation = undefined;
+                    var request_ids: [256]u64 = undefined;
+                    var lsns: [256]u64 = undefined;
+                    var count: usize = 0;
+                    var pipeline_consumed: usize = 0;
+                    while (count < operations.len and pipeline_consumed < input.len) {
+                        const next = pkbfi.parse(input[pipeline_consumed..]) catch |err| switch (err) {
+                            error.Incomplete => break,
+                            else => return,
+                        };
+                        if (next.opcode != .put) break;
+                        operations[count] = pkbfi.putOperation(next) catch return;
+                        request_ids[count] = next.request_id;
+                        count += 1;
+                        pipeline_consumed += next.consumed;
+                    }
+                    engine.beginRequest();
+                    engine.putMany(operations[0..count], lsns[0..count]) catch {
+                        engine.endRequest();
+                        return;
+                    };
+                    engine.endRequest();
+                    var responses = std.ArrayListUnmanaged(u8){};
+                    defer responses.deinit(std.heap.smp_allocator);
+                    for (0..count) |index| {
+                        var body: [10]u8 = [_]u8{0} ** 10;
+                        std.mem.writeInt(u64, body[2..10], lsns[index], .little);
+                        const response = pkbfi.encode(std.heap.smp_allocator, @intFromEnum(pkbfi.Opcode.put) | 0x8000, 1, request_ids[index], &body) catch return;
+                        defer std.heap.smp_allocator.free(response);
+                        responses.appendSlice(std.heap.smp_allocator, response) catch return;
+                    }
+                    reserveMemory(engine, responses.items.len) catch return;
+                    sendAll(connection, responses.items) catch {
+                        releaseMemory(engine, responses.items.len);
+                        return;
+                    };
+                    releaseMemory(engine, responses.items.len);
+                    consumed += pipeline_consumed;
+                } else {
+                    engine.beginRequest();
+                    const response = session.execute(engine, frame) catch {
+                        engine.endRequest();
+                        return;
+                    };
+                    engine.endRequest();
+                    reserveMemory(engine, response.len) catch {
+                        std.heap.smp_allocator.free(response);
+                        return;
+                    };
+                    sendAll(connection, response) catch {
+                        std.heap.smp_allocator.free(response);
+                        releaseMemory(engine, response.len);
+                        return;
+                    };
+                    std.heap.smp_allocator.free(response);
+                    releaseMemory(engine, response.len);
+                    consumed += frame.consumed;
                 }
-                response = redis.executeCommand(result.cmd, responseBuffer[0..]) orelse
-                    redis.formatError(responseBuffer[0..], "ERR response too large");
+            } else if (input[0] == '*') {
+                const parsed = resp.parse(input) catch |err| switch (err) {
+                    error.Incomplete => break,
+                    else => return,
+                };
+                if (parsed.command.command_type == .set) {
+                    var operations: [256]@import("engine.zig").Operation = undefined;
+                    var lsns: [256]u64 = undefined;
+                    var count: usize = 0;
+                    var pipeline_consumed: usize = 0;
+                    while (count < operations.len and pipeline_consumed < input.len) {
+                        const next = resp.parse(input[pipeline_consumed..]) catch |err| switch (err) {
+                            error.Incomplete => break,
+                            else => return,
+                        };
+                        if (next.command.command_type != .set) break;
+                        operations[count] = .{ .opcode = .put, .key = next.command.key, .value = next.command.value };
+                        count += 1;
+                        pipeline_consumed += next.consumed;
+                    }
+                    engine.beginRequest();
+                    engine.putMany(operations[0..count], lsns[0..count]) catch {
+                        engine.endRequest();
+                        return;
+                    };
+                    engine.endRequest();
+                    var responses: [256 * 5]u8 = undefined;
+                    for (0..count) |index| @memcpy(responses[index * 5 ..][0..5], "+OK\r\n");
+                    sendAll(connection, responses[0 .. count * 5]) catch return;
+                    consumed += pipeline_consumed;
+                } else if (parsed.command.command_type == .get) {
+                    var keys: [256][]const u8 = undefined;
+                    var count: usize = 0;
+                    var pipeline_consumed: usize = 0;
+                    while (count < keys.len and pipeline_consumed < input.len) {
+                        const next = resp.parse(input[pipeline_consumed..]) catch |err| switch (err) {
+                            error.Incomplete => break,
+                            else => return,
+                        };
+                        if (next.command.command_type != .get) break;
+                        keys[count] = next.command.key;
+                        count += 1;
+                        pipeline_consumed += next.consumed;
+                    }
+                    engine.beginRequest();
+                    handleRespGets(engine, connection, keys[0..count]) catch {
+                        engine.endRequest();
+                        return;
+                    };
+                    engine.endRequest();
+                    consumed += pipeline_consumed;
+                } else {
+                    engine.beginRequest();
+                    handleResp(engine, connection, parsed.command) catch {
+                        engine.endRequest();
+                        return;
+                    };
+                    engine.endRequest();
+                    consumed += parsed.consumed;
+                }
+            } else {
+                const end = std.mem.indexOfScalar(u8, input, '\r') orelse break;
+                engine.beginRequest();
+                const response = pizzaria.execute(engine, std.heap.smp_allocator, input[0..end]) catch {
+                    engine.endRequest();
+                    return;
+                };
+                engine.endRequest();
+                reserveMemory(engine, response.len) catch {
+                    std.heap.smp_allocator.free(response);
+                    return;
+                };
+                sendAll(connection, response) catch {
+                    std.heap.smp_allocator.free(response);
+                    releaseMemory(engine, response.len);
+                    return;
+                };
+                sendAll(connection, "\r") catch {
+                    std.heap.smp_allocator.free(response);
+                    releaseMemory(engine, response.len);
+                    return;
+                };
+                std.heap.smp_allocator.free(response);
+                releaseMemory(engine, response.len);
+                consumed += end + 1 + @intFromBool(input.len > end + 1 and input[end + 1] == '\n');
             }
-
-            const final_response = response orelse {
-                offset += result.bytes_consumed;
-                continue;
-            };
-            response_offset += final_response.len;
-            offset += result.bytes_consumed;
         }
-
-        posix.setsockopt(conn, posix.IPPROTO.TCP, cork_option, &std.mem.toBytes(@as(c_int, 0))) catch {};
-
-        if (response_offset > 0) {
-            try sendAll(conn, responseBuffer[0..response_offset]);
+        if (consumed != 0) {
+            const remaining = buffered - consumed;
+            std.mem.copyForwards(u8, buffer[0..remaining], buffer[consumed..buffered]);
+            buffered = remaining;
         }
+        if (buffer.len > initial_buffer and buffered <= initial_buffer) {
+            var replacement = std.heap.smp_allocator.alloc(u8, initial_buffer) catch return;
+            @memcpy(replacement[0..buffered], buffer[0..buffered]);
+            const released = buffer.len - initial_buffer;
+            std.heap.smp_allocator.free(buffer);
+            buffer = replacement;
+            releaseMemory(engine, released);
+        }
+    }
+}
 
-        if (offset < total_len) {
-            const remaining = total_len - offset;
-            if (remaining > 0 and remaining < requestBuffer.len / 2) {
-                @memcpy(requestBuffer[0..remaining], requestBuffer[offset..total_len]);
-                buffered_len = remaining;
-            } else {
-                buffered_len = 0;
-            }
-        } else {
-            buffered_len = 0;
+pub fn main() !void {
+    var host: []const u8 = "127.0.0.1";
+    var port: u16 = 8085;
+    var path: []const u8 = ".pkvdb";
+    var unix_path: ?[]const u8 = null;
+    var migration_source: ?[]const u8 = null;
+    var args = try std.process.argsWithAllocator(std.heap.page_allocator);
+    defer args.deinit();
+    _ = args.skip();
+    while (args.next()) |argument| {
+        if (std.mem.startsWith(u8, argument, "-host=")) host = argument[6..] else if (std.mem.startsWith(u8, argument, "-port=")) port = try std.fmt.parseInt(u16, argument[6..], 10) else if (std.mem.startsWith(u8, argument, "-path=")) path = argument[6..] else if (std.mem.startsWith(u8, argument, "-migrate=")) migration_source = argument[9..] else if (std.mem.eql(u8, argument, "-unix")) unix_path = ".pizzakv.sock" else if (std.mem.startsWith(u8, argument, "-unix=")) unix_path = argument[6..] else if (std.mem.eql(u8, argument, "-redis") or std.mem.eql(u8, argument, "-pkbfi") or std.mem.eql(u8, argument, "-iwal")) {} else return error.InvalidArgument;
+    }
+    if (migration_source) |source| {
+        const result = try migration.migrate(std.heap.smp_allocator, source, path);
+        std.debug.print("Migrated {d} keys from {d} records checksum={x}\n", .{ result.keys, result.records, result.checksum });
+        return;
+    }
+    const action = posix.Sigaction{ .handler = .{ .handler = signalHandler }, .mask = std.mem.zeroes(posix.sigset_t), .flags = 0 };
+    _ = posix.sigaction(posix.SIG.TERM, &action, null);
+    _ = posix.sigaction(posix.SIG.INT, &action, null);
+    var engine = try Engine.open(std.heap.smp_allocator, path);
+    defer engine.close();
+    const listener = if (unix_path) |name| try socket.initUnix(name) else try socket.init(host, port);
+    defer posix.close(listener);
+    defer if (unix_path) |name| posix.unlink(name) catch {};
+    std.debug.print("PizzaKV {s} Pizzaria/RESP/PKBFI\n", .{path});
+    while (!should_exit.load(.acquire)) {
+        var descriptors = [_]posix.pollfd{.{ .fd = listener, .events = posix.POLL.IN, .revents = 0 }};
+        if ((posix.poll(&descriptors, 100) catch continue) == 0) continue;
+        const connection = posix.accept(listener, null, null, 0) catch continue;
+        if (active_connections.fetchAdd(1, .monotonic) >= max_connections) {
+            _ = active_connections.fetchSub(1, .monotonic);
+            posix.close(connection);
+            continue;
         }
+        if (unix_path == null and (builtin.target.os.tag == .linux or builtin.target.os.tag == .macos)) posix.setsockopt(connection, posix.IPPROTO.TCP, posix.TCP.NODELAY, &std.mem.toBytes(@as(c_int, 1))) catch {};
+        socket.setReadTimeout(connection, 30) catch {};
+        socket.setWriteTimeout(connection, 30) catch {};
+        const thread = std.Thread.spawn(.{}, handleConnection, .{ &engine, connection }) catch {
+            _ = active_connections.fetchSub(1, .monotonic);
+            posix.close(connection);
+            continue;
+        };
+        thread.detach();
     }
+    while (active_connections.load(.monotonic) != 0) std.Thread.sleep(10 * std.time.ns_per_ms);
 }

makefile

@@ -26,18 +26,14 @@ clean:
 	rm -f pizzakv
 
 test:
-	zig test hashing.zig
-	zig test redis.zig
-	zig test storage.zig
-	zig test index.zig
+	zig test pkvdb.zig
+	zig test keydir.zig
+	zig test ordered_index.zig
+	zig test engine.zig
 	zig test command.zig
-	zig test persistence.zig
+	zig test redis.zig
+	zig test pkbfi.zig
+	zig test migration.zig
 
 bench:
-	node tools/test_nov.js
-	node tools/test_accuracy.js
-	node tools/test_comprehensive.js
-	node tools/test_concurrent.js
-	node tools/test_concurrent_reads.js
-	node tools/test_concurrent_reads_quick.js
-	node tools/test_reads_keys.js
+	zig run benchmark.zig -O ReleaseFast

migration.zig

@@ -0,0 +1,146 @@
+const std = @import("std");
+const engine_mod = @import("engine.zig");
+
+pub const Result = struct {
+    keys: u64,
+    records: u64,
+    checksum: u64,
+};
+
+pub fn migrate(allocator: std.mem.Allocator, source_path: []const u8, destination_path: []const u8) !Result {
+    std.fs.cwd().access(destination_path, .{}) catch |err| switch (err) {
+        error.FileNotFound => {},
+        else => return err,
+    };
+    if (std.fs.cwd().openFile(destination_path, .{})) |file| {
+        file.close();
+        return error.DestinationExists;
+    } else |_| {}
+    const source = try std.fs.cwd().openFile(source_path, .{ .mode = .read_only });
+    defer source.close();
+    var state = std.StringHashMap([]u8).init(allocator);
+    defer {
+        var iterator = state.iterator();
+        while (iterator.next()) |entry| {
+            allocator.free(entry.key_ptr.*);
+            allocator.free(entry.value_ptr.*);
+        }
+        state.deinit();
+    }
+    var record = std.ArrayListUnmanaged(u8){};
+    defer record.deinit(allocator);
+    var buffer: [64 * 1024]u8 = undefined;
+    var records: u64 = 0;
+    while (true) {
+        const amount = try source.read(&buffer);
+        if (amount == 0) break;
+        var start: usize = 0;
+        while (std.mem.indexOfScalarPos(u8, buffer[0..amount], start, '\r')) |end| {
+            try record.appendSlice(allocator, buffer[start..end]);
+            if (record.items.len != 0) {
+                try applyRecord(allocator, &state, record.items);
+                records += 1;
+            }
+            record.clearRetainingCapacity();
+            start = end + 1;
+        }
+        if (start < amount) {
+            if (record.items.len + amount - start > 65 * 1024 * 1024) return error.RecordTooLarge;
+            try record.appendSlice(allocator, buffer[start..amount]);
+        }
+    }
+    if (record.items.len != 0) {
+        try applyRecord(allocator, &state, record.items);
+        records += 1;
+    }
+    var engine = try engine_mod.Engine.open(allocator, destination_path);
+    defer engine.close();
+    var operations = std.ArrayListUnmanaged(engine_mod.Operation){};
+    defer operations.deinit(allocator);
+    var bytes: usize = 24;
+    var iterator = state.iterator();
+    while (iterator.next()) |entry| {
+        const needed = 8 + entry.key_ptr.*.len + entry.value_ptr.*.len + 7;
+        if (operations.items.len == 65535 or bytes + needed > 60 * 1024 * 1024) {
+            try engine.importBaseline(operations.items);
+            operations.clearRetainingCapacity();
+            bytes = 24;
+        }
+        try operations.append(allocator, .{ .opcode = .put, .key = entry.key_ptr.*, .value = entry.value_ptr.* });
+        bytes += needed;
+    }
+    if (operations.items.len != 0) {
+        try engine.importBaseline(operations.items);
+    } else if (state.count() == 0) {
+        try engine.importBaseline(&.{});
+    }
+    try engine.checkpoint();
+    const status = engine.status();
+    if (status.live_keys != state.count() or status.latest_lsn != 1 or status.checkpoint_lsn != 1) return error.VerificationFailed;
+    var checksum: u64 = 14695981039346656037;
+    iterator = state.iterator();
+    while (iterator.next()) |entry| {
+        const value = try engine.get(allocator, entry.key_ptr.*) orelse return error.VerificationFailed;
+        defer allocator.free(value.bytes);
+        if (!std.mem.eql(u8, value.bytes, entry.value_ptr.*)) return error.VerificationFailed;
+        for (entry.key_ptr.*) |byte| {
+            checksum ^= byte;
+            checksum *%= 1099511628211;
+        }
+        for (value.bytes) |byte| {
+            checksum ^= byte;
+            checksum *%= 1099511628211;
+        }
+    }
+    return .{ .keys = status.live_keys, .records = records, .checksum = checksum };
+}
+
+fn applyRecord(allocator: std.mem.Allocator, state: *std.StringHashMap([]u8), record: []const u8) !void {
+    const first = std.mem.indexOfScalar(u8, record, '|') orelse return error.InvalidLegacyRecord;
+    const second_relative = std.mem.indexOfScalar(u8, record[first + 1 ..], '|') orelse return error.InvalidLegacyRecord;
+    const second = first + 1 + second_relative;
+    const opcode = record[0..first];
+    const key = record[first + 1 .. second];
+    const value = record[second + 1 ..];
+    if (std.mem.eql(u8, opcode, "W")) {
+        if (state.getPtr(key)) |current| {
+            const replacement = try allocator.dupe(u8, value);
+            allocator.free(current.*);
+            current.* = replacement;
+        } else {
+            const owned_key = try allocator.dupe(u8, key);
+            errdefer allocator.free(owned_key);
+            const owned_value = try allocator.dupe(u8, value);
+            errdefer allocator.free(owned_value);
+            try state.put(owned_key, owned_value);
+        }
+    } else if (std.mem.eql(u8, opcode, "D")) {
+        if (state.fetchRemove(key)) |removed| {
+            allocator.free(removed.key);
+            allocator.free(removed.value);
+        }
+    } else return error.InvalidLegacyRecord;
+}
+
+test "legacy migration creates LSN one checkpoint and leaves source" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    try tmp.dir.writeFile(.{ .sub_path = "old.db", .data = "W|a|one\rW|b|two\rW|a|three\rD|b|\r" });
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const directory = try tmp.dir.realpath(".", &path_buffer);
+    const source = try std.fmt.allocPrint(std.testing.allocator, "{s}/old.db", .{directory});
+    defer std.testing.allocator.free(source);
+    const destination = try std.fmt.allocPrint(std.testing.allocator, "{s}/new.pkvdb", .{directory});
+    defer std.testing.allocator.free(destination);
+    const result = try migrate(std.testing.allocator, source, destination);
+    try std.testing.expectEqual(@as(u64, 1), result.keys);
+    const original = try tmp.dir.readFileAlloc(std.testing.allocator, "old.db", 1024);
+    defer std.testing.allocator.free(original);
+    try std.testing.expectEqualStrings("W|a|one\rW|b|two\rW|a|three\rD|b|\r", original);
+    var engine = try engine_mod.Engine.open(std.testing.allocator, destination);
+    defer engine.close();
+    const value = (try engine.get(std.testing.allocator, "a")).?;
+    defer std.testing.allocator.free(value.bytes);
+    try std.testing.expectEqualStrings("three", value.bytes);
+    try std.testing.expectEqual(@as(u64, 1), engine.status().checkpoint_lsn);
+}

ordered_index.zig

@@ -0,0 +1,236 @@
+const std = @import("std");
+const RecordRef = @import("keydir.zig").RecordRef;
+
+pub const Node = struct {
+    key: []u8,
+    inline_key: [32]u8 = undefined,
+    external_key: bool,
+    record: RecordRef,
+    priority: u64,
+    parent: ?*Node = null,
+    left: ?*Node = null,
+    right: ?*Node = null,
+};
+
+pub const Prepared = struct {
+    node: ?*Node,
+};
+
+pub const OrderedIndex = struct {
+    allocator: std.mem.Allocator,
+    root: ?*Node = null,
+    count: usize = 0,
+    allocated_bytes: usize = 0,
+
+    pub fn init(allocator: std.mem.Allocator) OrderedIndex {
+        return .{ .allocator = allocator };
+    }
+
+    pub fn deinit(self: *OrderedIndex) void {
+        while (self.root) |node| _ = self.remove(node.key);
+        self.* = undefined;
+    }
+
+    fn priority(key: []const u8) u64 {
+        var value: u64 = 14695981039346656037;
+        for (key) |byte| {
+            value ^= byte;
+            value *%= 1099511628211;
+        }
+        value ^= value >> 30;
+        value *%= 0xbf58476d1ce4e5b9;
+        value ^= value >> 27;
+        value *%= 0x94d049bb133111eb;
+        return value ^ (value >> 31);
+    }
+
+    fn rotateLeft(self: *OrderedIndex, node: *Node) void {
+        const child = node.right.?;
+        node.right = child.left;
+        if (child.left) |left| left.parent = node;
+        child.parent = node.parent;
+        if (node.parent) |parent| {
+            if (parent.left == node) parent.left = child else parent.right = child;
+        } else self.root = child;
+        child.left = node;
+        node.parent = child;
+    }
+
+    fn rotateRight(self: *OrderedIndex, node: *Node) void {
+        const child = node.left.?;
+        node.left = child.right;
+        if (child.right) |right| right.parent = node;
+        child.parent = node.parent;
+        if (node.parent) |parent| {
+            if (parent.left == node) parent.left = child else parent.right = child;
+        } else self.root = child;
+        child.right = node;
+        node.parent = child;
+    }
+
+    pub fn put(self: *OrderedIndex, key: []const u8, record: RecordRef) !void {
+        var parent: ?*Node = null;
+        var current = self.root;
+        var order: std.math.Order = .eq;
+        while (current) |node| {
+            order = std.mem.order(u8, key, node.key);
+            if (order == .eq) {
+                node.record = record;
+                return;
+            }
+            parent = node;
+            current = if (order == .lt) node.left else node.right;
+        }
+        var prepared = try self.prepare(key);
+        const node = prepared.node.?;
+        prepared.node = null;
+        node.record = record;
+        node.parent = parent;
+        if (parent) |value| {
+            if (order == .lt) value.left = node else value.right = node;
+        } else self.root = node;
+        self.count += 1;
+        self.allocated_bytes += @sizeOf(Node) + if (node.external_key) key.len else 0;
+        while (node.parent) |value| {
+            if (value.priority <= node.priority) break;
+            if (value.left == node) self.rotateRight(value) else self.rotateLeft(value);
+        }
+    }
+
+    pub fn prepare(self: *OrderedIndex, key: []const u8) !Prepared {
+        const node = try self.allocator.create(Node);
+        errdefer self.allocator.destroy(node);
+        node.* = .{ .key = undefined, .external_key = key.len > 32, .record = undefined, .priority = priority(key) };
+        if (node.external_key) {
+            node.key = try self.allocator.dupe(u8, key);
+        } else {
+            @memcpy(node.inline_key[0..key.len], key);
+            node.key = node.inline_key[0..key.len];
+        }
+        return .{ .node = node };
+    }
+
+    pub fn discard(self: *OrderedIndex, prepared: *Prepared) void {
+        const node = prepared.node orelse return;
+        if (node.external_key) self.allocator.free(node.key);
+        self.allocator.destroy(node);
+        prepared.node = null;
+    }
+
+    pub fn putPrepared(self: *OrderedIndex, prepared: *Prepared, record: RecordRef) void {
+        const node = prepared.node.?;
+        var parent: ?*Node = null;
+        var current = self.root;
+        var order: std.math.Order = .eq;
+        while (current) |existing| {
+            order = std.mem.order(u8, node.key, existing.key);
+            if (order == .eq) {
+                existing.record = record;
+                self.discard(prepared);
+                return;
+            }
+            parent = existing;
+            current = if (order == .lt) existing.left else existing.right;
+        }
+        prepared.node = null;
+        node.record = record;
+        node.parent = parent;
+        if (parent) |value| {
+            if (order == .lt) value.left = node else value.right = node;
+        } else self.root = node;
+        self.count += 1;
+        self.allocated_bytes += @sizeOf(Node) + if (node.external_key) node.key.len else 0;
+        while (node.parent) |value| {
+            if (value.priority <= node.priority) break;
+            if (value.left == node) self.rotateRight(value) else self.rotateLeft(value);
+        }
+    }
+
+    pub fn remove(self: *OrderedIndex, key: []const u8) bool {
+        const node = self.find(key) orelse return false;
+        while (node.left != null or node.right != null) {
+            if (node.left == null) {
+                self.rotateLeft(node);
+            } else if (node.right == null) {
+                self.rotateRight(node);
+            } else if (node.left.?.priority < node.right.?.priority) {
+                self.rotateRight(node);
+            } else {
+                self.rotateLeft(node);
+            }
+        }
+        if (node.parent) |parent| {
+            if (parent.left == node) parent.left = null else parent.right = null;
+        } else self.root = null;
+        self.count -= 1;
+        self.allocated_bytes -= @sizeOf(Node) + if (node.external_key) node.key.len else 0;
+        if (node.external_key) self.allocator.free(node.key);
+        self.allocator.destroy(node);
+        return true;
+    }
+
+    pub fn find(self: *const OrderedIndex, key: []const u8) ?*Node {
+        var current = self.root;
+        while (current) |node| switch (std.mem.order(u8, key, node.key)) {
+            .eq => return node,
+            .lt => current = node.left,
+            .gt => current = node.right,
+        };
+        return null;
+    }
+
+    pub fn lowerBound(self: *const OrderedIndex, key: []const u8) ?*Node {
+        var current = self.root;
+        var result: ?*Node = null;
+        while (current) |node| {
+            if (std.mem.order(u8, node.key, key) == .lt) {
+                current = node.right;
+            } else {
+                result = node;
+                current = node.left;
+            }
+        }
+        return result;
+    }
+
+    pub fn next(node: *Node) ?*Node {
+        if (node.right) |right| {
+            var current = right;
+            while (current.left) |left| current = left;
+            return current;
+        }
+        var current = node;
+        while (current.parent) |parent| {
+            if (parent.left == current) return parent;
+            current = parent;
+        }
+        return null;
+    }
+
+    pub fn records(self: *const OrderedIndex, allocator: std.mem.Allocator) ![]RecordRef {
+        const result = try allocator.alloc(RecordRef, self.count);
+        var node = self.lowerBound("");
+        var index: usize = 0;
+        while (node) |value| {
+            result[index] = value.record;
+            index += 1;
+            node = next(value);
+        }
+        return result;
+    }
+};
+
+test "ordered insert update remove and lower bound" {
+    var index = OrderedIndex.init(std.testing.allocator);
+    defer index.deinit();
+    try index.put("b", .{ .hash = 2, .lsn = 1, .key_offset = 0, .value_offset = 0, .key_len = 1, .value_len = 0 });
+    try index.put("a", .{ .hash = 1, .lsn = 1, .key_offset = 0, .value_offset = 0, .key_len = 1, .value_len = 0 });
+    try index.put("c", .{ .hash = 3, .lsn = 1, .key_offset = 0, .value_offset = 0, .key_len = 1, .value_len = 0 });
+    try index.put("b", .{ .hash = 2, .lsn = 2, .key_offset = 0, .value_offset = 0, .key_len = 1, .value_len = 0 });
+    try std.testing.expectEqual(@as(usize, 3), index.count);
+    try std.testing.expectEqualStrings("b", index.lowerBound("az").?.key);
+    try std.testing.expectEqual(@as(u64, 2), index.find("b").?.record.lsn);
+    try std.testing.expect(index.remove("b"));
+    try std.testing.expectEqualStrings("c", index.lowerBound("b").?.key);
+    try std.testing.expectEqual(@as(usize, 2), index.count);
+}

persistence.zig

@@ -1,182 +0,0 @@
-const std = @import("std");
-const storage = @import("storage.zig");
-
-const BUFFER_SIZE = 1024 * 1024 * 8;
-const FLUSH_THRESHOLD = (BUFFER_SIZE * 3) / 4;
-
-var storage_file: ?std.fs.File = null;
-const c_allocator = std.heap.c_allocator;
-
-var mutex: std.Thread.Mutex = .{};
-
-var write_buffer: [BUFFER_SIZE]u8 = undefined;
-var buffer_position: usize = 0;
-var instant_wal: bool = false;
-
-const OPCode = enum {
-    W,
-    D,
-};
-
-pub fn init() !void {
-    const cwd = std.fs.cwd();
-    storage_file = cwd.openFile(".db", .{ .mode = .read_write }) catch |err| blk: {
-        if (err == std.fs.File.OpenError.FileNotFound) {
-            std.debug.print("No persisted data found, starting fresh...\n", .{});
-            const file = try cwd.createFile(".db", .{ .read = true });
-            std.debug.print("Created new storage file .db\n", .{});
-            break :blk file;
-        } else {
-            return err;
-        }
-    };
-
-    var record_count: usize = 0;
-    try restoreFromFile(storage_file.?, &record_count);
-    std.debug.print("Restored {d} records from persistence", .{record_count});
-
-    storage_file.?.close();
-    storage_file = try cwd.openFile(".db", .{ .mode = .write_only });
-    try storage_file.?.seekFromEnd(0);
-}
-
-fn restoreFromFile(file: std.fs.File, record_count: *usize) !void {
-    var read_buffer: [BUFFER_SIZE]u8 = undefined;
-    var record_buffer = std.ArrayListUnmanaged(u8){};
-    defer record_buffer.deinit(c_allocator);
-
-    while (true) {
-        const n = try file.read(&read_buffer);
-        if (n == 0) break;
-
-        var start: usize = 0;
-        while (std.mem.indexOfScalarPos(u8, read_buffer[0..n], start, '\r')) |end| {
-            try record_buffer.appendSlice(c_allocator, read_buffer[start..end]);
-            restoreRecord(record_buffer.items, record_count);
-            record_buffer.clearRetainingCapacity();
-            start = end + 1;
-        }
-
-        if (start < n) {
-            try record_buffer.appendSlice(c_allocator, read_buffer[start..n]);
-        }
-    }
-
-    if (record_buffer.items.len > 0) {
-        restoreRecord(record_buffer.items, record_count);
-    }
-}
-
-fn restoreRecord(record: []const u8, record_count: *usize) void {
-    if (record.len == 0) {
-        return;
-    }
-
-    record_count.* += 1;
-    std.debug.print("Restoring record N:{d}\r", .{record_count.*});
-
-    const first_pipe = std.mem.indexOfScalar(u8, record, '|') orelse return;
-    const opcode = record[0..first_pipe];
-
-    const remaining = record[first_pipe + 1 ..];
-    const second_pipe = std.mem.indexOfScalar(u8, remaining, '|') orelse return;
-    const key = remaining[0..second_pipe];
-    const value = remaining[second_pipe + 1 ..];
-
-    const opcodeEnum = std.meta.stringToEnum(OPCode, opcode) orelse return;
-
-    switch (opcodeEnum) {
-        .W => _ = storage.restore(key, value),
-        .D => _ = storage.restoreDelete(key),
-    }
-}
-
-pub fn setInstantWal(enabled: bool) void {
-    instant_wal = enabled;
-}
-
-fn recordLen(key: []const u8, value: []const u8) usize {
-    return 1 + 1 + key.len + 1 + value.len + 1;
-}
-
-fn encodeRecord(buf: []u8, opcode: u8, key: []const u8, value: []const u8) usize {
-    var pos: usize = 0;
-    buf[pos] = opcode;
-    pos += 1;
-    buf[pos] = '|';
-    pos += 1;
-    @memcpy(buf[pos .. pos + key.len], key);
-    pos += key.len;
-    buf[pos] = '|';
-    pos += 1;
-    @memcpy(buf[pos .. pos + value.len], value);
-    pos += value.len;
-    buf[pos] = '\r';
-    pos += 1;
-    return pos;
-}
-
-fn syncFile() !void {
-    if (storage_file) |f| {
-        try f.sync();
-    }
-}
-
-pub fn persist(opcode: u8, key: []const u8, value: []const u8) !void {
-    const record_len = recordLen(key, value);
-
-    mutex.lock();
-    defer mutex.unlock();
-
-    if (record_len > BUFFER_SIZE) {
-        return error.RecordTooLarge;
-    }
-
-    if (buffer_position + record_len > FLUSH_THRESHOLD) {
-        try flushBuffer();
-    }
-
-    buffer_position += encodeRecord(write_buffer[buffer_position..], opcode, key, value);
-
-    if (instant_wal) {
-        try flushBuffer();
-        try syncFile();
-    }
-}
-
-pub fn flush() !void {
-    mutex.lock();
-    defer mutex.unlock();
-    try flushBuffer();
-    try syncFile();
-}
-
-fn flushBuffer() !void {
-    if (buffer_position == 0) {
-        return;
-    }
-
-    const f = storage_file orelse return error.StorageFileNotOpen;
-    try f.writeAll(write_buffer[0..buffer_position]);
-    buffer_position = 0;
-}
-
-// -- Tests --
-
-test "recordLen matches encoded record length" {
-    try std.testing.expectEqual(@as(usize, 4 + 3 + 5), recordLen("key", "value"));
-    try std.testing.expectEqual(@as(usize, 4), recordLen("", ""));
-}
-
-test "encodeRecord produces WAL framing" {
-    var buf: [64]u8 = undefined;
-    const written = encodeRecord(&buf, 'W', "key1", "value1");
-    try std.testing.expectEqualStrings("W|key1|value1\r", buf[0..written]);
-    try std.testing.expectEqual(@as(usize, recordLen("key1", "value1")), written);
-}
-
-test "encodeRecord delete framing" {
-    var buf: [64]u8 = undefined;
-    const written = encodeRecord(&buf, 'D', "key2", "");
-    try std.testing.expectEqualStrings("D|key2|\r", buf[0..written]);
-}

pkbfi.zig

@@ -0,0 +1,342 @@
+const std = @import("std");
+const engine_mod = @import("engine.zig");
+const pkvdb = @import("pkvdb.zig");
+
+pub const header_size = 32;
+pub const max_frame_size = pkvdb.max_key_size + pkvdb.max_value_size + 1024;
+
+pub const Opcode = enum(u16) {
+    ping = 1,
+    status = 2,
+    get = 3,
+    put = 4,
+    delete = 5,
+    exists = 6,
+    multi_get = 7,
+    batch_write = 8,
+    scan_open = 9,
+    scan_next = 10,
+    scan_close = 11,
+};
+
+pub const Frame = struct {
+    opcode: Opcode,
+    flags: u16,
+    request_id: u64,
+    payload: []const u8,
+    consumed: usize,
+};
+
+const Scan = struct {
+    id: u64,
+    prefix: []u8,
+    cursor: []u8,
+    include_values: bool,
+    limit: u32,
+};
+
+pub const Session = struct {
+    allocator: std.mem.Allocator,
+    scans: std.ArrayListUnmanaged(Scan) = .{},
+    next_scan_id: u64 = 1,
+
+    pub fn init(allocator: std.mem.Allocator) Session {
+        return .{ .allocator = allocator };
+    }
+
+    pub fn deinit(self: *Session) void {
+        for (self.scans.items) |scan| {
+            self.allocator.free(scan.prefix);
+            self.allocator.free(scan.cursor);
+        }
+        self.scans.deinit(self.allocator);
+        self.* = undefined;
+    }
+
+    fn findScan(self: *Session, id: u64) ?usize {
+        for (self.scans.items, 0..) |scan, index| if (scan.id == id) return index;
+        return null;
+    }
+
+    pub fn execute(self: *Session, engine: *engine_mod.Engine, frame: Frame) ![]u8 {
+        var body = std.ArrayListUnmanaged(u8){};
+        defer body.deinit(self.allocator);
+        self.executeBody(engine, frame, &body) catch |err| {
+            body.clearRetainingCapacity();
+            try appendInt(u16, &body, self.allocator, 2);
+            try body.appendSlice(self.allocator, @errorName(err));
+        };
+        return encode(self.allocator, @intFromEnum(frame.opcode) | 0x8000, 1, frame.request_id, body.items);
+    }
+
+    fn executeBody(self: *Session, engine: *engine_mod.Engine, frame: Frame, body: *std.ArrayListUnmanaged(u8)) !void {
+        try appendInt(u16, body, self.allocator, 0);
+        switch (frame.opcode) {
+            .ping => try body.appendSlice(self.allocator, frame.payload),
+            .status => {
+                if (frame.payload.len != 0) return error.InvalidPayload;
+                const status = engine.status();
+                try body.appendSlice(self.allocator, &status.uuid);
+                inline for (.{ status.file_bytes, status.latest_lsn, status.oldest_lsn, status.checkpoint_lsn, status.journal_bytes_since_checkpoint, status.live_keys, status.keydir_bytes, status.ordered_index_bytes, status.bytes_written, status.checksum_failures, status.partial_tails, status.recovery_ns, status.checkpoint_ns, status.connection_bytes, status.active_requests, status.commit_groups, status.committed_transactions, status.largest_commit_group }) |value| try appendInt(u64, body, self.allocator, value);
+            },
+            .get => {
+                const key = try oneKey(frame.payload);
+                if (try engine.get(self.allocator, key)) |value| {
+                    defer self.allocator.free(value.bytes);
+                    try appendInt(u64, body, self.allocator, value.lsn);
+                    try appendInt(u32, body, self.allocator, @intCast(value.bytes.len));
+                    try body.appendSlice(self.allocator, value.bytes);
+                } else {
+                    body.clearRetainingCapacity();
+                    try appendInt(u16, body, self.allocator, 1);
+                }
+            },
+            .put => {
+                const operation = try putOperation(frame);
+                const lsn = try engine.put(operation.key, operation.value);
+                try appendInt(u64, body, self.allocator, lsn);
+            },
+            .delete => {
+                const deleted = try engine.delete(try oneKey(frame.payload));
+                try body.append(self.allocator, @intFromBool(deleted));
+            },
+            .exists => {
+                const present = try engine.exists(try oneKey(frame.payload));
+                try body.append(self.allocator, @intFromBool(present));
+            },
+            .multi_get => try self.multiGet(engine, frame.payload, body),
+            .batch_write => try self.batchWrite(engine, frame.payload, body),
+            .scan_open => try self.scanOpen(frame.payload, body),
+            .scan_next => try self.scanNext(engine, frame.payload, body),
+            .scan_close => try self.scanClose(frame.payload, body),
+        }
+    }
+
+    fn multiGet(self: *Session, engine: *engine_mod.Engine, payload: []const u8, body: *std.ArrayListUnmanaged(u8)) !void {
+        if (payload.len < 4) return error.InvalidPayload;
+        const count = readInt(u32, payload, 0);
+        if (count > pkvdb.max_operations) return error.InvalidPayload;
+        const keys = try self.allocator.alloc([]const u8, count);
+        defer self.allocator.free(keys);
+        var position: usize = 4;
+        for (keys) |*key| {
+            if (position > payload.len or payload.len - position < 4) return error.InvalidPayload;
+            const length = readInt(u32, payload, position);
+            position = try std.math.add(usize, position, 4);
+            const end = try std.math.add(usize, position, length);
+            if (end > payload.len or length > pkvdb.max_key_size) return error.InvalidPayload;
+            key.* = payload[position..end];
+            position = end;
+        }
+        if (position != payload.len) return error.InvalidPayload;
+        const values = try engine.multiGet(self.allocator, keys);
+        defer {
+            for (values) |value| if (value) |present| self.allocator.free(present.bytes);
+            self.allocator.free(values);
+        }
+        try appendInt(u32, body, self.allocator, count);
+        for (values) |value| if (value) |present| {
+            try body.append(self.allocator, 1);
+            try body.appendNTimes(self.allocator, 0, 3);
+            try appendInt(u32, body, self.allocator, @intCast(present.bytes.len));
+            try appendInt(u64, body, self.allocator, present.lsn);
+            try body.appendSlice(self.allocator, present.bytes);
+        } else {
+            try body.appendNTimes(self.allocator, 0, 16);
+        };
+    }
+
+    fn batchWrite(self: *Session, engine: *engine_mod.Engine, payload: []const u8, body: *std.ArrayListUnmanaged(u8)) !void {
+        if (payload.len < 8) return error.InvalidPayload;
+        const count = readInt(u32, payload, 0);
+        const metadata_length = readInt(u32, payload, 4);
+        if (count == 0 or count > pkvdb.max_operations or metadata_length > payload.len - 8) return error.InvalidPayload;
+        const metadata_end = 8 + metadata_length;
+        const metadata = payload[8..metadata_end];
+        const operations = try self.allocator.alloc(engine_mod.Operation, count);
+        defer self.allocator.free(operations);
+        var position: usize = metadata_end;
+        for (operations) |*operation| {
+            if (position > payload.len or payload.len - position < 12) return error.InvalidPayload;
+            const opcode: pkvdb.Opcode = std.meta.intToEnum(pkvdb.Opcode, payload[position]) catch return error.InvalidPayload;
+            const key_length = readInt(u32, payload, position + 4);
+            const value_length = readInt(u32, payload, position + 8);
+            position += 12;
+            const key_end = try std.math.add(usize, position, key_length);
+            const value_end = try std.math.add(usize, key_end, value_length);
+            if (value_end > payload.len or key_length > pkvdb.max_key_size or value_length > pkvdb.max_value_size or (opcode == .delete and value_length != 0)) return error.InvalidPayload;
+            operation.* = .{ .opcode = opcode, .key = payload[position..key_end], .value = payload[key_end..value_end] };
+            position = value_end;
+        }
+        if (position != payload.len) return error.InvalidPayload;
+        try appendInt(u64, body, self.allocator, try engine.batchWrite(operations, metadata));
+    }
+
+    fn scanOpen(self: *Session, payload: []const u8, body: *std.ArrayListUnmanaged(u8)) !void {
+        if (payload.len < 12 or self.scans.items.len >= 64) return error.InvalidPayload;
+        const include_values = payload[0] != 0;
+        const limit = readInt(u32, payload, 4);
+        const prefix_length = readInt(u32, payload, 8);
+        if (limit == 0 or limit > 4096 or prefix_length > pkvdb.max_key_size or 12 + prefix_length != payload.len) return error.InvalidPayload;
+        const prefix = try self.allocator.dupe(u8, payload[12..]);
+        errdefer self.allocator.free(prefix);
+        const cursor = try self.allocator.alloc(u8, 0);
+        errdefer self.allocator.free(cursor);
+        const id = self.next_scan_id;
+        self.next_scan_id +%= 1;
+        if (self.next_scan_id == 0) self.next_scan_id = 1;
+        try self.scans.append(self.allocator, .{ .id = id, .prefix = prefix, .cursor = cursor, .include_values = include_values, .limit = limit });
+        try appendInt(u64, body, self.allocator, id);
+    }
+
+    fn scanNext(self: *Session, engine: *engine_mod.Engine, payload: []const u8, body: *std.ArrayListUnmanaged(u8)) !void {
+        if (payload.len != 12) return error.InvalidPayload;
+        const index = self.findScan(readInt(u64, payload, 0)) orelse return error.ScanNotFound;
+        const requested = readInt(u32, payload, 8);
+        const scan = &self.scans.items[index];
+        const limit = if (requested == 0) scan.limit else @min(requested, scan.limit);
+        var batch = try engine.scan(self.allocator, scan.prefix, scan.cursor, limit, scan.include_values, 1024 * 1024);
+        defer batch.deinit(self.allocator);
+        try body.append(self.allocator, @intFromBool(batch.done));
+        try body.appendNTimes(self.allocator, 0, 3);
+        try appendInt(u32, body, self.allocator, @intCast(batch.entries.len));
+        for (batch.entries) |entry| {
+            try appendInt(u32, body, self.allocator, @intCast(entry.key.len));
+            try appendInt(u32, body, self.allocator, @intCast(if (entry.value) |value| value.len else 0));
+            try appendInt(u64, body, self.allocator, entry.lsn);
+            try body.appendSlice(self.allocator, entry.key);
+            if (entry.value) |value| try body.appendSlice(self.allocator, value);
+        }
+        const cursor = try self.allocator.dupe(u8, batch.next_cursor);
+        self.allocator.free(scan.cursor);
+        scan.cursor = cursor;
+    }
+
+    fn scanClose(self: *Session, payload: []const u8, body: *std.ArrayListUnmanaged(u8)) !void {
+        if (payload.len != 8) return error.InvalidPayload;
+        const index = self.findScan(readInt(u64, payload, 0)) orelse return error.ScanNotFound;
+        const scan = self.scans.orderedRemove(index);
+        self.allocator.free(scan.prefix);
+        self.allocator.free(scan.cursor);
+        try body.append(self.allocator, 1);
+    }
+};
+
+pub fn putOperation(frame: Frame) !engine_mod.Operation {
+    if (frame.opcode != .put or frame.payload.len < 8) return error.InvalidPayload;
+    const key_length = readInt(u32, frame.payload, 0);
+    const value_length = readInt(u32, frame.payload, 4);
+    const total = try std.math.add(usize, 8, try std.math.add(usize, key_length, value_length));
+    if (total != frame.payload.len or key_length > pkvdb.max_key_size or value_length > pkvdb.max_value_size) return error.InvalidPayload;
+    return .{ .opcode = .put, .key = frame.payload[8 .. 8 + key_length], .value = frame.payload[8 + key_length ..] };
+}
+
+fn oneKey(payload: []const u8) ![]const u8 {
+    if (payload.len < 4) return error.InvalidPayload;
+    const length = readInt(u32, payload, 0);
+    if (length > pkvdb.max_key_size or 4 + length != payload.len) return error.InvalidPayload;
+    return payload[4..];
+}
+
+pub fn parse(bytes: []const u8) !Frame {
+    if (bytes.len < header_size) return error.Incomplete;
+    if (!std.mem.eql(u8, bytes[0..4], "PKBF")) return error.InvalidMagic;
+    if (readInt(u16, bytes, 4) != 1) return error.IncompatibleVersion;
+    const payload_length = readInt(u32, bytes, 20);
+    if (payload_length > max_frame_size) return error.FrameTooLarge;
+    const total = try std.math.add(usize, header_size, payload_length);
+    if (total > bytes.len) return error.Incomplete;
+    var header: [32]u8 = undefined;
+    @memcpy(&header, bytes[0..32]);
+    const header_crc = readInt(u32, &header, 28);
+    writeInt(u32, &header, 28, 0);
+    if (pkvdb.crc32c(&header) != header_crc) return error.ChecksumMismatch;
+    const payload = bytes[header_size..total];
+    if (pkvdb.crc32c(payload) != readInt(u32, bytes, 24)) return error.ChecksumMismatch;
+    const opcode = std.meta.intToEnum(Opcode, readInt(u16, bytes, 8) & 0x7fff) catch return error.UnknownOpcode;
+    return .{ .opcode = opcode, .flags = readInt(u16, bytes, 10), .request_id = readInt(u64, bytes, 12), .payload = payload, .consumed = total };
+}
+
+pub fn encode(allocator: std.mem.Allocator, opcode: u16, flags: u16, request_id: u64, payload: []const u8) ![]u8 {
+    if (payload.len > max_frame_size) return error.FrameTooLarge;
+    const output = try allocator.alloc(u8, header_size + payload.len);
+    errdefer allocator.free(output);
+    @memset(output[0..header_size], 0);
+    @memcpy(output[0..4], "PKBF");
+    writeInt(u16, output, 4, 1);
+    writeInt(u16, output, 6, 0);
+    writeInt(u16, output, 8, opcode);
+    writeInt(u16, output, 10, flags);
+    writeInt(u64, output, 12, request_id);
+    writeInt(u32, output, 20, @intCast(payload.len));
+    writeInt(u32, output, 24, pkvdb.crc32c(payload));
+    writeInt(u32, output, 28, 0);
+    writeInt(u32, output, 28, pkvdb.crc32c(output[0..header_size]));
+    @memcpy(output[header_size..], payload);
+    return output;
+}
+
+fn appendInt(comptime T: type, list: *std.ArrayListUnmanaged(u8), allocator: std.mem.Allocator, value: T) !void {
+    var bytes: [@sizeOf(T)]u8 = undefined;
+    std.mem.writeInt(T, &bytes, value, .little);
+    try list.appendSlice(allocator, &bytes);
+}
+
+fn readInt(comptime T: type, bytes: []const u8, offset: usize) T {
+    return std.mem.readInt(T, bytes[offset..][0..@sizeOf(T)], .little);
+}
+
+fn writeInt(comptime T: type, bytes: []u8, offset: usize, value: T) void {
+    std.mem.writeInt(T, bytes[offset..][0..@sizeOf(T)], value, .little);
+}
+
+test "PKBFI frame round trip and malicious length" {
+    const bytes = try encode(std.testing.allocator, @intFromEnum(Opcode.ping), 0, 42, "a\x00b");
+    defer std.testing.allocator.free(bytes);
+    const frame = try parse(bytes);
+    try std.testing.expectEqual(@as(u64, 42), frame.request_id);
+    try std.testing.expectEqualSlices(u8, "a\x00b", frame.payload);
+    var bad = [_]u8{0} ** header_size;
+    @memcpy(bad[0..4], "PKBF");
+    writeInt(u16, &bad, 4, 1);
+    writeInt(u32, &bad, 20, max_frame_size + 1);
+    try std.testing.expectError(error.FrameTooLarge, parse(&bad));
+}
+
+test "PKBFI binary point batch and streaming scan" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const directory = try tmp.dir.realpath(".", &path_buffer);
+    const path = try std.fmt.allocPrint(std.testing.allocator, "{s}/pkbfi.pkvdb", .{directory});
+    defer std.testing.allocator.free(path);
+    var engine = try engine_mod.Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    var session = Session.init(std.testing.allocator);
+    defer session.deinit();
+    var put_payload = [_]u8{0} ** 14;
+    writeInt(u32, &put_payload, 0, 2);
+    writeInt(u32, &put_payload, 4, 4);
+    @memcpy(put_payload[8..10], "k\x00");
+    @memcpy(put_payload[10..14], "v\x00x\n");
+    const put_frame = Frame{ .opcode = .put, .flags = 0, .request_id = 1, .payload = &put_payload, .consumed = 0 };
+    const response = try session.execute(&engine, put_frame);
+    defer std.testing.allocator.free(response);
+    const parsed = try parse(response);
+    try std.testing.expectEqual(@as(u16, 0), readInt(u16, parsed.payload, 0));
+    var open_payload = [_]u8{0} ** 12;
+    open_payload[0] = 1;
+    writeInt(u32, &open_payload, 4, 1);
+    writeInt(u32, &open_payload, 8, 0);
+    const open_response = try session.execute(&engine, .{ .opcode = .scan_open, .flags = 0, .request_id = 2, .payload = &open_payload, .consumed = 0 });
+    defer std.testing.allocator.free(open_response);
+    const open_frame = try parse(open_response);
+    const scan_id = readInt(u64, open_frame.payload, 2);
+    var next_payload: [12]u8 = undefined;
+    writeInt(u64, &next_payload, 0, scan_id);
+    writeInt(u32, &next_payload, 8, 1);
+    const next_response = try session.execute(&engine, .{ .opcode = .scan_next, .flags = 0, .request_id = 3, .payload = &next_payload, .consumed = 0 });
+    defer std.testing.allocator.free(next_response);
+    const next_frame = try parse(next_response);
+    try std.testing.expectEqual(@as(u32, 1), readInt(u32, next_frame.payload, 6));
+}

pkvdb.zig

@@ -0,0 +1,493 @@
+const std = @import("std");
+
+pub const superblock_size: u64 = 4096;
+pub const data_offset: u64 = 8192;
+pub const extent_header_size: usize = 64;
+pub const transaction_header_size: usize = 56;
+pub const operation_header_size: usize = 16;
+pub const group_header_size: usize = 24;
+pub const checkpoint_header_size: usize = 56;
+pub const checkpoint_entry_size: usize = 48;
+pub const manifest_size: usize = 104;
+pub const max_key_size: u32 = 1024 * 1024;
+pub const max_value_size: u32 = 64 * 1024 * 1024;
+pub const max_transaction_size: u32 = 64 * 1024 * 1024;
+pub const max_operations: u32 = 65535;
+
+pub const ExtentType = enum(u16) {
+    journal = 1,
+    checkpoint_entries = 2,
+    checkpoint_ordered_index = 3,
+    manifest = 4,
+    store_metadata = 5,
+    _,
+};
+
+pub const Opcode = enum(u8) {
+    put = 1,
+    delete = 2,
+};
+
+pub const Superblock = struct {
+    flags: u32 = 0,
+    generation: u64,
+    uuid: [16]u8,
+    manifest_offset: u64,
+    checkpoint_lsn: u64,
+    known_lsn: u64,
+    known_file_length: u64,
+    created_ns: i64,
+    updated_ns: i64,
+};
+
+pub const ExtentHeader = struct {
+    extent_type: ExtentType,
+    version: u16 = 1,
+    flags: u32 = 0,
+    payload_length: u64,
+    first_lsn: u64,
+    last_lsn: u64,
+    previous_offset: u64 = 0,
+    payload_crc: u32,
+};
+
+pub const TransactionHeader = struct {
+    frame_type: u8 = 1,
+    flags: u8 = 0,
+    total_length: u32,
+    lsn: u64,
+    transaction_id: u64,
+    timestamp_ns: i64,
+    operation_count: u32,
+    metadata_length: u32,
+    payload_crc: u32,
+};
+
+pub const OperationHeader = struct {
+    opcode: Opcode,
+    flags: u8 = 0,
+    key_length: u32,
+    value_length: u32,
+    extension_length: u32 = 0,
+};
+
+pub const CheckpointHeader = struct {
+    flags: u32 = 0,
+    lsn: u64,
+    timestamp_ns: i64,
+    entry_count: u64,
+    source_start: u64,
+    source_end: u64,
+};
+
+pub const CheckpointEntry = struct {
+    hash: u64,
+    lsn: u64,
+    key_offset: u64,
+    value_offset: u64,
+    key_len: u32,
+    value_len: u32,
+    flags: u16 = 0,
+};
+
+pub const Manifest = struct {
+    uuid: [16]u8,
+    generation: u64,
+    checkpoint_lsn: u64,
+    entries_offset: u64,
+    ordered_offset: u64,
+    replay_offset: u64,
+    known_tail: u64,
+    known_lsn: u64,
+    history_start_lsn: u64,
+    flags: u32 = 1,
+};
+
+pub fn align8(value: u64) !u64 {
+    const added = try std.math.add(u64, value, 7);
+    return added & ~@as(u64, 7);
+}
+
+const crc32c_tables = blk: {
+    @setEvalBranchQuota(10000);
+    var tables: [8][256]u32 = undefined;
+    for (&tables[0], 0..) |*entry, index| {
+        var value: u32 = index;
+        for (0..8) |_| value = (value >> 1) ^ (@as(u32, 0x82f63b78) & (0 -% (value & 1)));
+        entry.* = value;
+    }
+    for (1..8) |table_index| for (0..256) |index| {
+        const previous = tables[table_index - 1][index];
+        tables[table_index][index] = tables[0][@as(u8, @truncate(previous))] ^ (previous >> 8);
+    };
+    break :blk tables;
+};
+
+pub fn crc32cUpdate(state: u32, bytes: []const u8) u32 {
+    var crc = state;
+    var position: usize = 0;
+    while (bytes.len - position >= 8) : (position += 8) {
+        crc ^= std.mem.readInt(u32, bytes[position..][0..4], .little);
+        crc = crc32c_tables[7][@as(u8, @truncate(crc))] ^
+            crc32c_tables[6][@as(u8, @truncate(crc >> 8))] ^
+            crc32c_tables[5][@as(u8, @truncate(crc >> 16))] ^
+            crc32c_tables[4][@as(u8, @truncate(crc >> 24))] ^
+            crc32c_tables[3][bytes[position + 4]] ^
+            crc32c_tables[2][bytes[position + 5]] ^
+            crc32c_tables[1][bytes[position + 6]] ^
+            crc32c_tables[0][bytes[position + 7]];
+    }
+    for (bytes[position..]) |byte| crc = crc32c_tables[0][@as(u8, @truncate(crc)) ^ byte] ^ (crc >> 8);
+    return crc;
+}
+
+pub fn crc32c(bytes: []const u8) u32 {
+    return ~crc32cUpdate(0xffffffff, bytes);
+}
+
+fn put(comptime T: type, dst: []u8, offset: usize, value: T) void {
+    std.mem.writeInt(T, dst[offset..][0..@sizeOf(T)], value, .little);
+}
+
+fn get(comptime T: type, src: []const u8, offset: usize) T {
+    return std.mem.readInt(T, src[offset..][0..@sizeOf(T)], .little);
+}
+
+fn zeroCrc(bytes: []u8, offset: usize) u32 {
+    const old = get(u32, bytes, offset);
+    put(u32, bytes, offset, 0);
+    const crc = crc32c(bytes);
+    put(u32, bytes, offset, old);
+    return crc;
+}
+
+pub fn encodeSuperblock(sb: Superblock, out: *[4096]u8) void {
+    @memset(out, 0);
+    @memcpy(out[0..8], "PKVDB\x00\x00\x00");
+    put(u16, out, 8, 1);
+    put(u16, out, 10, 0);
+    put(u32, out, 12, 112);
+    put(u32, out, 16, sb.flags);
+    put(u64, out, 24, sb.generation);
+    @memcpy(out[32..48], &sb.uuid);
+    put(u64, out, 48, sb.manifest_offset);
+    put(u64, out, 56, sb.checkpoint_lsn);
+    put(u64, out, 64, sb.known_lsn);
+    put(u64, out, 72, sb.known_file_length);
+    put(i64, out, 80, sb.created_ns);
+    put(i64, out, 88, sb.updated_ns);
+    put(u32, out, 96, 0);
+    put(u32, out, 96, crc32c(out[0..112]));
+}
+
+pub fn decodeSuperblock(bytes: []const u8, file_length: u64) !Superblock {
+    if (bytes.len < superblock_size) return error.Truncated;
+    if (!std.mem.eql(u8, bytes[0..8], "PKVDB\x00\x00\x00")) return error.InvalidMagic;
+    if (get(u16, bytes, 8) != 1) return error.IncompatibleVersion;
+    if (get(u32, bytes, 12) != 112) return error.InvalidLength;
+    var header: [112]u8 = undefined;
+    @memcpy(&header, bytes[0..112]);
+    const stored_crc = get(u32, &header, 96);
+    if (zeroCrc(&header, 96) != stored_crc) return error.ChecksumMismatch;
+    const known_length = get(u64, bytes, 72);
+    const manifest_offset = get(u64, bytes, 48);
+    if (known_length < data_offset or known_length > file_length) return error.UnusableRoot;
+    if (manifest_offset != 0 and (manifest_offset < data_offset or manifest_offset >= known_length or manifest_offset % 8 != 0)) return error.UnusableRoot;
+    var uuid: [16]u8 = undefined;
+    @memcpy(&uuid, bytes[32..48]);
+    return .{
+        .flags = get(u32, bytes, 16),
+        .generation = get(u64, bytes, 24),
+        .uuid = uuid,
+        .manifest_offset = manifest_offset,
+        .checkpoint_lsn = get(u64, bytes, 56),
+        .known_lsn = get(u64, bytes, 64),
+        .known_file_length = known_length,
+        .created_ns = get(i64, bytes, 80),
+        .updated_ns = get(i64, bytes, 88),
+    };
+}
+
+pub fn encodeExtentHeader(header: ExtentHeader, out: *[64]u8) void {
+    @memset(out, 0);
+    @memcpy(out[0..4], "PKEX");
+    put(u16, out, 4, @intFromEnum(header.extent_type));
+    put(u16, out, 6, header.version);
+    put(u32, out, 8, header.flags);
+    put(u32, out, 12, extent_header_size);
+    put(u64, out, 16, header.payload_length);
+    put(u64, out, 24, header.first_lsn);
+    put(u64, out, 32, header.last_lsn);
+    put(u64, out, 40, header.previous_offset);
+    put(u32, out, 48, header.payload_crc);
+    put(u32, out, 52, 0);
+    put(u32, out, 52, crc32c(out));
+}
+
+pub fn decodeExtentHeader(bytes: []const u8) !ExtentHeader {
+    if (bytes.len < extent_header_size) return error.Truncated;
+    if (!std.mem.eql(u8, bytes[0..4], "PKEX")) return error.InvalidMagic;
+    if (get(u32, bytes, 12) != extent_header_size) return error.InvalidLength;
+    var copy: [64]u8 = undefined;
+    @memcpy(&copy, bytes[0..64]);
+    const stored_crc = get(u32, &copy, 52);
+    if (zeroCrc(&copy, 52) != stored_crc) return error.ChecksumMismatch;
+    return .{
+        .extent_type = @enumFromInt(get(u16, bytes, 4)),
+        .version = get(u16, bytes, 6),
+        .flags = get(u32, bytes, 8),
+        .payload_length = get(u64, bytes, 16),
+        .first_lsn = get(u64, bytes, 24),
+        .last_lsn = get(u64, bytes, 32),
+        .previous_offset = get(u64, bytes, 40),
+        .payload_crc = get(u32, bytes, 48),
+    };
+}
+
+pub fn encodeTransactionHeader(header: TransactionHeader, out: *[56]u8) void {
+    @memset(out, 0);
+    @memcpy(out[0..4], "PKTX");
+    put(u16, out, 4, 1);
+    out[6] = header.frame_type;
+    out[7] = header.flags;
+    put(u32, out, 8, header.total_length);
+    put(u16, out, 12, transaction_header_size);
+    put(u64, out, 16, header.lsn);
+    put(u64, out, 24, header.transaction_id);
+    put(i64, out, 32, header.timestamp_ns);
+    put(u32, out, 40, header.operation_count);
+    put(u32, out, 44, header.metadata_length);
+    put(u32, out, 48, header.payload_crc);
+    put(u32, out, 52, 0);
+    put(u32, out, 52, crc32c(out));
+}
+
+pub fn decodeTransactionHeader(bytes: []const u8) !TransactionHeader {
+    if (bytes.len < transaction_header_size) return error.Truncated;
+    if (!std.mem.eql(u8, bytes[0..4], "PKTX")) return error.InvalidMagic;
+    if (get(u16, bytes, 4) != 1) return error.IncompatibleVersion;
+    if (get(u16, bytes, 12) != transaction_header_size) return error.InvalidLength;
+    const total = get(u32, bytes, 8);
+    if (total < transaction_header_size or total > max_transaction_size) return error.InvalidLength;
+    const count = get(u32, bytes, 40);
+    if (count > max_operations) return error.InvalidLength;
+    var copy: [56]u8 = undefined;
+    @memcpy(&copy, bytes[0..56]);
+    const stored_crc = get(u32, &copy, 52);
+    if (zeroCrc(&copy, 52) != stored_crc) return error.ChecksumMismatch;
+    return .{
+        .frame_type = bytes[6],
+        .flags = bytes[7],
+        .total_length = total,
+        .lsn = get(u64, bytes, 16),
+        .transaction_id = get(u64, bytes, 24),
+        .timestamp_ns = get(i64, bytes, 32),
+        .operation_count = count,
+        .metadata_length = get(u32, bytes, 44),
+        .payload_crc = get(u32, bytes, 48),
+    };
+}
+
+pub fn encodeOperationHeader(header: OperationHeader, out: *[16]u8) void {
+    @memset(out, 0);
+    out[0] = @intFromEnum(header.opcode);
+    out[1] = header.flags;
+    put(u32, out, 4, header.key_length);
+    put(u32, out, 8, header.value_length);
+    put(u32, out, 12, header.extension_length);
+}
+
+pub fn decodeOperationHeader(bytes: []const u8) !OperationHeader {
+    if (bytes.len < operation_header_size) return error.Truncated;
+    const opcode: Opcode = std.meta.intToEnum(Opcode, bytes[0]) catch return error.InvalidOpcode;
+    const key_length = get(u32, bytes, 4);
+    const value_length = get(u32, bytes, 8);
+    if (key_length > max_key_size or value_length > max_value_size) return error.InvalidLength;
+    if (opcode == .delete and value_length != 0) return error.InvalidLength;
+    return .{
+        .opcode = opcode,
+        .flags = bytes[1],
+        .key_length = key_length,
+        .value_length = value_length,
+        .extension_length = get(u32, bytes, 12),
+    };
+}
+
+pub fn encodeCheckpointHeader(header: CheckpointHeader, out: *[56]u8) void {
+    @memset(out, 0);
+    put(u16, out, 0, 1);
+    put(u16, out, 2, 1);
+    put(u32, out, 4, header.flags);
+    put(u64, out, 8, header.lsn);
+    put(i64, out, 16, header.timestamp_ns);
+    put(u64, out, 24, header.entry_count);
+    put(u32, out, 32, checkpoint_entry_size);
+    put(u32, out, 36, 1);
+    put(u64, out, 40, header.source_start);
+    put(u64, out, 48, header.source_end);
+}
+
+pub fn decodeCheckpointHeader(bytes: []const u8) !CheckpointHeader {
+    if (bytes.len < checkpoint_header_size) return error.Truncated;
+    return decodeCheckpointHeaderOnly(bytes[0..checkpoint_header_size], bytes.len);
+}
+
+pub fn decodeCheckpointHeaderOnly(bytes: []const u8, payload_length: u64) !CheckpointHeader {
+    if (bytes.len < checkpoint_header_size) return error.Truncated;
+    if (get(u16, bytes, 0) != 1 or get(u16, bytes, 2) != 1) return error.IncompatibleVersion;
+    if (get(u32, bytes, 32) != checkpoint_entry_size or get(u32, bytes, 36) != 1) return error.IncompatibleVersion;
+    const count = get(u64, bytes, 24);
+    const entries_bytes = try std.math.mul(u64, count, checkpoint_entry_size);
+    const needed = try std.math.add(u64, checkpoint_header_size, entries_bytes);
+    if (needed != payload_length) return error.InvalidLength;
+    return .{
+        .flags = get(u32, bytes, 4),
+        .lsn = get(u64, bytes, 8),
+        .timestamp_ns = get(i64, bytes, 16),
+        .entry_count = count,
+        .source_start = get(u64, bytes, 40),
+        .source_end = get(u64, bytes, 48),
+    };
+}
+
+pub fn encodeCheckpointEntry(entry: CheckpointEntry, out: *[48]u8) void {
+    @memset(out, 0);
+    put(u64, out, 0, entry.hash);
+    put(u64, out, 8, entry.lsn);
+    put(u64, out, 16, entry.key_offset);
+    put(u64, out, 24, entry.value_offset);
+    put(u32, out, 32, entry.key_len);
+    put(u32, out, 36, entry.value_len);
+    put(u16, out, 40, entry.flags);
+    put(u32, out, 44, 0);
+    put(u32, out, 44, crc32c(out));
+}
+
+pub fn decodeCheckpointEntry(bytes: []const u8, file_length: u64) !CheckpointEntry {
+    if (bytes.len < checkpoint_entry_size) return error.Truncated;
+    var copy: [48]u8 = undefined;
+    @memcpy(&copy, bytes[0..48]);
+    const stored_crc = get(u32, &copy, 44);
+    if (zeroCrc(&copy, 44) != stored_crc) return error.ChecksumMismatch;
+    const key_offset = get(u64, bytes, 16);
+    const value_offset = get(u64, bytes, 24);
+    const key_len = get(u32, bytes, 32);
+    const value_len = get(u32, bytes, 36);
+    if (key_len > max_key_size or value_len > max_value_size) return error.InvalidLength;
+    if (key_offset < data_offset or value_offset < data_offset) return error.InvalidOffset;
+    if (try std.math.add(u64, key_offset, key_len) > file_length or try std.math.add(u64, value_offset, value_len) > file_length) return error.InvalidOffset;
+    return .{
+        .hash = get(u64, bytes, 0),
+        .lsn = get(u64, bytes, 8),
+        .key_offset = key_offset,
+        .value_offset = value_offset,
+        .key_len = key_len,
+        .value_len = value_len,
+        .flags = get(u16, bytes, 40),
+    };
+}
+
+pub fn encodeManifest(manifest: Manifest, out: *[104]u8) void {
+    @memset(out, 0);
+    put(u16, out, 0, 1);
+    put(u16, out, 2, 0);
+    put(u32, out, 4, manifest_size);
+    @memcpy(out[8..24], &manifest.uuid);
+    put(u64, out, 24, manifest.generation);
+    put(u64, out, 32, manifest.checkpoint_lsn);
+    put(u64, out, 40, manifest.entries_offset);
+    put(u64, out, 48, manifest.ordered_offset);
+    put(u64, out, 56, manifest.replay_offset);
+    put(u64, out, 64, manifest.known_tail);
+    put(u64, out, 72, manifest.known_lsn);
+    put(u64, out, 80, manifest.history_start_lsn);
+    put(u32, out, 88, 1);
+    put(u32, out, 92, manifest.flags);
+    put(u32, out, 96, 0);
+    put(u32, out, 96, crc32c(out));
+}
+
+pub fn decodeManifest(bytes: []const u8) !Manifest {
+    if (bytes.len != manifest_size) return error.InvalidLength;
+    if (get(u16, bytes, 0) != 1 or get(u32, bytes, 4) != manifest_size) return error.IncompatibleVersion;
+    var copy: [104]u8 = undefined;
+    @memcpy(&copy, bytes);
+    const stored_crc = get(u32, &copy, 96);
+    if (zeroCrc(&copy, 96) != stored_crc) return error.ChecksumMismatch;
+    if (get(u32, bytes, 88) != 1) return error.IncompatibleVersion;
+    var uuid: [16]u8 = undefined;
+    @memcpy(&uuid, bytes[8..24]);
+    return .{
+        .uuid = uuid,
+        .generation = get(u64, bytes, 24),
+        .checkpoint_lsn = get(u64, bytes, 32),
+        .entries_offset = get(u64, bytes, 40),
+        .ordered_offset = get(u64, bytes, 48),
+        .replay_offset = get(u64, bytes, 56),
+        .known_tail = get(u64, bytes, 64),
+        .known_lsn = get(u64, bytes, 72),
+        .history_start_lsn = get(u64, bytes, 80),
+        .flags = get(u32, bytes, 92),
+    };
+}
+
+test "crc32c golden" {
+    try std.testing.expectEqual(@as(u32, 0xe3069283), crc32c("123456789"));
+}
+
+test "superblock golden round trip" {
+    const sb = Superblock{ .generation = 7, .uuid = [_]u8{3} ** 16, .manifest_offset = 8192, .checkpoint_lsn = 4, .known_lsn = 6, .known_file_length = 9000, .created_ns = -2, .updated_ns = 8 };
+    var bytes: [4096]u8 = undefined;
+    encodeSuperblock(sb, &bytes);
+    const decoded = try decodeSuperblock(&bytes, 9000);
+    try std.testing.expectEqual(sb.generation, decoded.generation);
+    try std.testing.expectEqual(sb.uuid, decoded.uuid);
+    try std.testing.expectEqual(sb.created_ns, decoded.created_ns);
+    try std.testing.expectEqual(@as(u8, 'P'), bytes[0]);
+    try std.testing.expectEqual(@as(u8, 1), bytes[8]);
+}
+
+test "codec truncation and checksums" {
+    var extent_bytes: [64]u8 = undefined;
+    encodeExtentHeader(.{ .extent_type = .journal, .payload_length = 9, .first_lsn = 1, .last_lsn = 1, .payload_crc = 2 }, &extent_bytes);
+    try std.testing.expectError(error.Truncated, decodeExtentHeader(extent_bytes[0..63]));
+    extent_bytes[20] ^= 1;
+    try std.testing.expectError(error.ChecksumMismatch, decodeExtentHeader(&extent_bytes));
+}
+
+test "checkpoint length overflow" {
+    var bytes: [56]u8 = [_]u8{0} ** 56;
+    put(u16, &bytes, 0, 1);
+    put(u16, &bytes, 2, 1);
+    put(u64, &bytes, 24, std.math.maxInt(u64));
+    put(u32, &bytes, 32, checkpoint_entry_size);
+    put(u32, &bytes, 36, 1);
+    try std.testing.expectError(error.Overflow, decodeCheckpointHeader(&bytes));
+}
+
+test "unknown extent type remains decodable" {
+    var bytes: [64]u8 = undefined;
+    encodeExtentHeader(.{ .extent_type = @enumFromInt(99), .version = 7, .payload_length = 0, .first_lsn = 0, .last_lsn = 0, .payload_crc = crc32c("") }, &bytes);
+    const decoded = try decodeExtentHeader(&bytes);
+    try std.testing.expectEqual(@as(u16, 99), @intFromEnum(decoded.extent_type));
+    try std.testing.expectEqual(@as(u16, 7), decoded.version);
+}
+
+test "every truncated fixed header is rejected" {
+    var extent: [64]u8 = undefined;
+    encodeExtentHeader(.{ .extent_type = .journal, .payload_length = 0, .first_lsn = 1, .last_lsn = 1, .payload_crc = crc32c("") }, &extent);
+    for (0..extent.len) |length| try std.testing.expectError(error.Truncated, decodeExtentHeader(extent[0..length]));
+    var transaction: [56]u8 = undefined;
+    encodeTransactionHeader(.{ .total_length = 56, .lsn = 1, .transaction_id = 2, .timestamp_ns = 3, .operation_count = 0, .metadata_length = 0, .payload_crc = crc32c("") }, &transaction);
+    for (0..transaction.len) |length| try std.testing.expectError(error.Truncated, decodeTransactionHeader(transaction[0..length]));
+}
+
+test "operation validation rejects oversized and invalid delete" {
+    var bytes: [16]u8 = undefined;
+    encodeOperationHeader(.{ .opcode = .delete, .key_length = 1, .value_length = 1 }, &bytes);
+    try std.testing.expectError(error.InvalidLength, decodeOperationHeader(&bytes));
+    encodeOperationHeader(.{ .opcode = .put, .key_length = max_key_size + 1, .value_length = 0 }, &bytes);
+    try std.testing.expectError(error.InvalidLength, decodeOperationHeader(&bytes));
+    bytes[0] = 99;
+    try std.testing.expectError(error.InvalidOpcode, decodeOperationHeader(&bytes));
+}

redis.zig

@@ -1,468 +1,124 @@
 const std = @import("std");
-const storage = @import("storage.zig");
+const engine_mod = @import("engine.zig");
+const max_resp_frame = engine_mod.pkvdb_max_frame + 1024 * 1024 + 1024;
 
-const CommandType = enum {
-    SET,
-    GET,
-    DEL,
-    UNKNOWN,
-};
+pub const CommandType = enum { set, get, del, unknown };
 
-pub const RedisCommand = struct {
-    cmd_type: CommandType,
+pub const Command = struct {
+    command_type: CommandType,
     key: []const u8,
-    value: []const u8,
+    value: []const u8 = "",
 };
 
 pub const ParseResult = struct {
-    cmd: RedisCommand,
-    bytes_consumed: usize,
+    command: Command,
+    consumed: usize,
 };
 
-fn parseInteger(buf: []const u8, start: usize, end: usize) ?usize {
-    if (start >= end) return null;
-    var result: usize = 0;
-    for (buf[start..end]) |c| {
-        if (c < '0' or c > '9') return null;
-        const digit: usize = c - '0';
-        result = std.math.mul(usize, result, 10) catch return null;
-        result = std.math.add(usize, result, digit) catch return null;
-    }
-    return result;
-}
-
-fn parseBulkString(buf: []const u8, pos: *usize) ?[]const u8 {
-    if (pos.* >= buf.len or buf[pos.*] != '$') return null;
-    pos.* += 1;
-
-    const len_end = std.mem.indexOfScalarPos(u8, buf, pos.*, '\r') orelse return null;
-    const len = parseInteger(buf, pos.*, len_end) orelse return null;
-    pos.* = len_end + 2;
-
-    const str_start = pos.*;
-    const str_end = std.math.add(usize, str_start, len) catch return null;
-    if (str_end > buf.len) return null;
-
-    const result = buf[str_start..str_end];
-    pos.* = str_end + 2;
-
-    return result;
-}
-
-pub fn parseCommand(buf: []const u8) ?ParseResult {
-    if (buf.len == 0) return null;
-
-    var pos: usize = 0;
-
-    if (buf[pos] != '*') return null;
-    pos += 1;
-
-    const array_len_end = std.mem.indexOfScalarPos(u8, buf, pos, '\r') orelse return null;
-    const array_len = parseInteger(buf, pos, array_len_end) orelse return null;
-    pos = array_len_end + 2;
-
-    if (array_len < 1 or array_len > 16) return null;
-
-    var elements: [16][]const u8 = undefined;
-    for (0..array_len) |i| {
-        elements[i] = parseBulkString(buf, &pos) orelse return null;
-    }
-
-    const cmd_str = elements[0];
-    var cmd: RedisCommand = undefined;
-
-    if (cmd_str.len == 3) {
-        const upper: u32 = (@as(u32, cmd_str[0]) & 0xDF) << 16 | (@as(u32, cmd_str[1]) & 0xDF) << 8 | (@as(u32, cmd_str[2]) & 0xDF);
-        if (upper == (@as(u32, 'S') << 16 | @as(u32, 'E') << 8 | @as(u32, 'T'))) {
-            if (array_len < 3) return null;
-            cmd = RedisCommand{
-                .cmd_type = .SET,
-                .key = elements[1],
-                .value = elements[2],
-            };
-        } else if (upper == (@as(u32, 'G') << 16 | @as(u32, 'E') << 8 | @as(u32, 'T'))) {
-            if (array_len < 2) return null;
-            cmd = RedisCommand{
-                .cmd_type = .GET,
-                .key = elements[1],
-                .value = "",
-            };
-        } else if (upper == (@as(u32, 'D') << 16 | @as(u32, 'E') << 8 | @as(u32, 'L'))) {
-            if (array_len < 2) return null;
-            cmd = RedisCommand{
-                .cmd_type = .DEL,
-                .key = elements[1],
-                .value = "",
-            };
-        } else {
-            cmd = RedisCommand{
-                .cmd_type = .UNKNOWN,
-                .key = "",
-                .value = "",
-            };
-        }
-    } else {
-        cmd = RedisCommand{
-            .cmd_type = .UNKNOWN,
-            .key = "",
-            .value = "",
-        };
-    }
-
-    return ParseResult{
-        .cmd = cmd,
-        .bytes_consumed = pos,
-    };
-}
-
-fn intDigits(value: usize) usize {
-    if (value == 0) return 1;
-    var v = value;
-    var d: usize = 0;
-    while (v > 0) : (v /= 10) d += 1;
-    return d;
-}
-
-fn formatInt(buf: []u8, value: usize) ?usize {
-    if (value == 0) {
-        if (buf.len < 1) return null;
-        buf[0] = '0';
-        return 1;
-    }
-
-    const len = intDigits(value);
-    if (buf.len < len) return null;
-
-    var v = value;
-    var i: usize = len;
-    while (i > 0) {
-        i -= 1;
-        buf[i] = @intCast('0' + (v % 10));
-        v /= 10;
-    }
-    return len;
-}
-
-fn formatSimpleString(buf: []u8, str: []const u8) ?[]const u8 {
-    const needed = 1 + str.len + 2;
-    if (buf.len < needed) return null;
-    buf[0] = '+';
-    @memcpy(buf[1 .. 1 + str.len], str);
-    buf[1 + str.len] = '\r';
-    buf[2 + str.len] = '\n';
-    return buf[0..needed];
-}
-
-fn formatBulkString(buf: []u8, str: []const u8) ?[]const u8 {
-    const needed = 1 + intDigits(str.len) + 2 + str.len + 2;
-    if (buf.len < needed) return null;
-
-    buf[0] = '$';
-    var pos: usize = 1;
-    pos += formatInt(buf[pos..], str.len) orelse return null;
-    buf[pos] = '\r';
-    buf[pos + 1] = '\n';
-    pos += 2;
-
-    @memcpy(buf[pos .. pos + str.len], str);
-    pos += str.len;
-    buf[pos] = '\r';
-    buf[pos + 1] = '\n';
+pub const Response = union(enum) {
+    simple: []const u8,
+    integer: i64,
+    null_bulk,
+    bulk: engine_mod.Value,
+    failure: []const u8,
 
-    return buf[0 .. pos + 2];
-}
-
-fn formatNullBulkString(buf: []u8) ?[]const u8 {
-    if (buf.len < 5) return null;
-    buf[0] = '$';
-    buf[1] = '-';
-    buf[2] = '1';
-    buf[3] = '\r';
-    buf[4] = '\n';
-    return buf[0..5];
-}
-
-fn formatInteger(buf: []u8, value: i64) ?[]const u8 {
-    var pos: usize = 1;
-    if (value < 0) {
-        const needed = 1 + 1 + intDigits(@intCast(-value)) + 2;
-        if (buf.len < needed) return null;
-        buf[0] = ':';
-        buf[1] = '-';
-        pos = 2;
-        pos += formatInt(buf[pos..], @intCast(-value)) orelse return null;
-    } else {
-        const needed = 1 + intDigits(@intCast(value)) + 2;
-        if (buf.len < needed) return null;
-        buf[0] = ':';
-        pos = 1;
-        pos += formatInt(buf[pos..], @intCast(value)) orelse return null;
+    pub fn deinit(self: *Response, allocator: std.mem.Allocator) void {
+        if (self.* == .bulk) allocator.free(self.bulk.bytes);
+        self.* = undefined;
     }
+};
 
-    buf[pos] = '\r';
-    buf[pos + 1] = '\n';
-    return buf[0 .. pos + 2];
-}
-
-pub fn formatError(buf: []u8, msg: []const u8) ?[]const u8 {
-    const needed = 1 + msg.len + 2;
-    if (buf.len < needed) return null;
-    buf[0] = '-';
-    @memcpy(buf[1 .. 1 + msg.len], msg);
-    buf[1 + msg.len] = '\r';
-    buf[2 + msg.len] = '\n';
-    return buf[0..needed];
-}
-
-pub fn executeCommand(cmd: RedisCommand, response_buf: []u8) ?[]const u8 {
-    switch (cmd.cmd_type) {
-        .SET => {
-            if (storage.write(cmd.key, cmd.value)) {
-                return formatSimpleString(response_buf, "OK");
-            } else {
-                return formatError(response_buf, "ERR write failed");
-            }
-        },
-        .GET => {
-            if (storage.read(cmd.key)) |value| {
-                return formatBulkString(response_buf, value);
-            } else {
-                return formatNullBulkString(response_buf);
-            }
-        },
-        .DEL => {
-            const deleted = storage.delete(cmd.key);
-            return formatInteger(response_buf, if (deleted) 1 else 0);
-        },
-        .UNKNOWN => {
-            return formatError(response_buf, "ERR unknown command");
-        },
+fn parseUnsigned(bytes: []const u8) !usize {
+    if (bytes.len == 0) return error.InvalidFrame;
+    var value: usize = 0;
+    for (bytes) |byte| {
+        if (byte < '0' or byte > '9') return error.InvalidFrame;
+        value = try std.math.add(usize, try std.math.mul(usize, value, 10), byte - '0');
     }
-}
-
-// -- Tests --
-
-fn buildRedisArray(parts: []const []const u8) []u8 {
-    var buf: [4096]u8 = undefined;
-    var pos: usize = 0;
-
-    buf[pos] = '*';
-    pos += 1;
-    pos += formatInt(buf[pos..], parts.len).?;
-    buf[pos] = '\r';
-    buf[pos + 1] = '\n';
-    pos += 2;
-
-    for (parts) |part| {
-        buf[pos] = '$';
-        pos += 1;
-        pos += formatInt(buf[pos..], part.len).?;
-        buf[pos] = '\r';
-        buf[pos + 1] = '\n';
-        pos += 2;
-        @memcpy(buf[pos .. pos + part.len], part);
-        pos += part.len;
-        buf[pos] = '\r';
-        buf[pos + 1] = '\n';
-        pos += 2;
+    return value;
+}
+
+fn lineEnd(bytes: []const u8, start: usize) !usize {
+    const end = std.mem.indexOfPos(u8, bytes, start, "\r\n") orelse return error.Incomplete;
+    return end;
+}
+
+fn bulk(bytes: []const u8, position: *usize) ![]const u8 {
+    if (position.* >= bytes.len) return error.Incomplete;
+    if (bytes[position.*] != '$') return error.InvalidFrame;
+    const end = try lineEnd(bytes, position.* + 1);
+    const length = try parseUnsigned(bytes[position.* + 1 .. end]);
+    if (length > engine_mod.pkvdb_max_frame) return error.FrameTooLarge;
+    const start = try std.math.add(usize, end, 2);
+    const data_end = try std.math.add(usize, start, length);
+    const final = try std.math.add(usize, data_end, 2);
+    if (final > max_resp_frame) return error.FrameTooLarge;
+    if (final > bytes.len) return error.Incomplete;
+    if (!std.mem.eql(u8, bytes[data_end..final], "\r\n")) return error.InvalidFrame;
+    position.* = final;
+    return bytes[start..data_end];
+}
+
+pub fn parse(bytes: []const u8) !ParseResult {
+    if (bytes.len == 0) return error.Incomplete;
+    if (bytes[0] != '*') return error.InvalidFrame;
+    const end = try lineEnd(bytes, 1);
+    const count = try parseUnsigned(bytes[1..end]);
+    if (count == 0 or count > 16) return error.InvalidFrame;
+    var position = end + 2;
+    var fields: [16][]const u8 = undefined;
+    for (0..count) |index| fields[index] = try bulk(bytes, &position);
+    const command_type: CommandType = if (std.ascii.eqlIgnoreCase(fields[0], "SET")) .set else if (std.ascii.eqlIgnoreCase(fields[0], "GET")) .get else if (std.ascii.eqlIgnoreCase(fields[0], "DEL")) .del else .unknown;
+    switch (command_type) {
+        .set => if (count != 3) return error.InvalidFrame,
+        .get, .del => if (count != 2) return error.InvalidFrame,
+        .unknown => {},
     }
-
-    return buf[0..pos];
-}
-
-test "parseCommand SET" {
-    const input = buildRedisArray(&.{ "SET", "mykey", "myvalue" });
-    const result = parseCommand(input) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqual(CommandType.SET, result.cmd.cmd_type);
-    try std.testing.expectEqualStrings("mykey", result.cmd.key);
-    try std.testing.expectEqualStrings("myvalue", result.cmd.value);
-}
-
-test "parseCommand GET" {
-    const input = buildRedisArray(&.{ "GET", "mykey" });
-    const result = parseCommand(input) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqual(CommandType.GET, result.cmd.cmd_type);
-    try std.testing.expectEqualStrings("mykey", result.cmd.key);
-}
-
-test "parseCommand DEL" {
-    const input = buildRedisArray(&.{ "DEL", "mykey" });
-    const result = parseCommand(input) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqual(CommandType.DEL, result.cmd.cmd_type);
-    try std.testing.expectEqualStrings("mykey", result.cmd.key);
+    return .{ .command = .{ .command_type = command_type, .key = if (count > 1) fields[1] else "", .value = if (count > 2) fields[2] else "" }, .consumed = position };
 }
 
-test "parseCommand case insensitive" {
-    const input = buildRedisArray(&.{ "set", "k", "v" });
-    const result = parseCommand(input) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqual(CommandType.SET, result.cmd.cmd_type);
-}
-
-test "parseCommand unknown command" {
-    const input = buildRedisArray(&.{ "FOO", "bar" });
-    const result = parseCommand(input) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqual(CommandType.UNKNOWN, result.cmd.cmd_type);
-}
-
-test "parseCommand empty input" {
-    try std.testing.expectEqual(@as(?ParseResult, null), parseCommand(""));
-}
-
-test "parseCommand malformed input" {
-    try std.testing.expectEqual(@as(?ParseResult, null), parseCommand("garbage"));
-    try std.testing.expectEqual(@as(?ParseResult, null), parseCommand("*"));
-    try std.testing.expectEqual(@as(?ParseResult, null), parseCommand("*1\r\n"));
-}
-
-test "parseCommand bytes_consumed" {
-    const input = buildRedisArray(&.{ "GET", "key1" });
-    const result = parseCommand(input) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqual(input.len, result.bytes_consumed);
-}
-
-test "parseInteger max usize fits" {
-    const s = "18446744073709551615";
-    try std.testing.expectEqual(@as(?usize, std.math.maxInt(usize)), parseInteger(s, 0, s.len));
-}
-
-test "parseInteger overflow returns null" {
-    try std.testing.expectEqual(@as(?usize, null), parseInteger("18446744073709551616", 0, 20));
-    try std.testing.expectEqual(@as(?usize, null), parseInteger("999999999999999999999999999999", 0, 30));
-}
-
-test "parseCommand huge bulk length returns null" {
-    const input = "*2\r\n$3\r\nGET\r\n$18446744073709551615\r\n";
-    try std.testing.expectEqual(@as(?ParseResult, null), parseCommand(input));
-}
-
-test "formatInt zero" {
-    var buf: [20]u8 = undefined;
-    const len = formatInt(&buf, 0) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("0", buf[0..len]);
-}
-
-test "formatInt positive" {
-    var buf: [20]u8 = undefined;
-    const len = formatInt(&buf, 12345) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("12345", buf[0..len]);
-}
-
-test "formatInt respects buffer bounds" {
-    var buf: [4]u8 = undefined;
-    try std.testing.expectEqual(@as(?usize, null), formatInt(buf[0..3], 1234));
-    const len = formatInt(buf[0..4], 1234) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("1234", buf[0..len]);
-}
-
-test "formatSimpleString" {
-    var buf: [64]u8 = undefined;
-    const result = formatSimpleString(&buf, "OK") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("+OK\r\n", result);
-}
-
-test "formatSimpleString respects buffer bounds" {
-    var exact: [5]u8 = undefined;
-    const ok = formatSimpleString(exact[0..5], "OK") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("+OK\r\n", ok);
-
-    var short: [4]u8 = undefined;
-    try std.testing.expectEqual(@as(?[]const u8, null), formatSimpleString(short[0..4], "OK"));
-}
-
-test "formatBulkString" {
-    var buf: [64]u8 = undefined;
-    const result = formatBulkString(&buf, "hello") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("$5\r\nhello\r\n", result);
-}
-
-test "formatBulkString respects buffer bounds" {
-    const value = "hello world";
-    const needed = 1 + intDigits(value.len) + 2 + value.len + 2;
-
-    var exact: [64]u8 = undefined;
-    const ok = formatBulkString(exact[0..needed], value) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("$11\r\nhello world\r\n", ok);
-
-    var short: [64]u8 = undefined;
-    try std.testing.expectEqual(@as(?[]const u8, null), formatBulkString(short[0 .. needed - 1], value));
-}
-
-test "formatNullBulkString" {
-    var buf: [64]u8 = undefined;
-    const result = formatNullBulkString(&buf) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("$-1\r\n", result);
-}
-
-test "formatNullBulkString respects buffer bounds" {
-    var exact: [5]u8 = undefined;
-    const ok = formatNullBulkString(exact[0..5]) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("$-1\r\n", ok);
-
-    var short: [4]u8 = undefined;
-    try std.testing.expectEqual(@as(?[]const u8, null), formatNullBulkString(short[0..4]));
-}
-
-test "formatError" {
-    var buf: [64]u8 = undefined;
-    const result = formatError(&buf, "ERR bad") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("-ERR bad\r\n", result);
-}
-
-test "formatError respects buffer bounds" {
-    const msg = "ERR bad";
-    const needed = 1 + msg.len + 2;
-
-    var exact: [16]u8 = undefined;
-    const ok = formatError(exact[0..needed], msg) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("-ERR bad\r\n", ok);
-
-    var short: [16]u8 = undefined;
-    try std.testing.expectEqual(@as(?[]const u8, null), formatError(short[0 .. needed - 1], msg));
-}
-
-test "formatInteger positive" {
-    var buf: [64]u8 = undefined;
-    const result = formatInteger(&buf, 42) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings(":42\r\n", result);
-}
-
-test "formatInteger zero" {
-    var buf: [64]u8 = undefined;
-    const result = formatInteger(&buf, 0) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings(":0\r\n", result);
-}
-
-test "formatInteger negative" {
-    var buf: [64]u8 = undefined;
-    const result = formatInteger(&buf, -7) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings(":-7\r\n", result);
-}
-
-test "formatInteger respects buffer bounds" {
-    var exact: [16]u8 = undefined;
-    const ok = formatInteger(exact[0..5], 42) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings(":42\r\n", ok);
-
-    var short: [16]u8 = undefined;
-    try std.testing.expectEqual(@as(?[]const u8, null), formatInteger(short[0..4], 42));
+pub fn execute(engine: *engine_mod.Engine, allocator: std.mem.Allocator, command: Command) !Response {
+    return switch (command.command_type) {
+        .set => if (engine.put(command.key, command.value)) |_| Response{ .simple = "OK" } else |_| Response{ .failure = "ERR write failed" },
+        .get => if (try engine.get(allocator, command.key)) |value| Response{ .bulk = value } else Response.null_bulk,
+        .del => Response{ .integer = if (try engine.delete(command.key)) 1 else 0 },
+        .unknown => Response{ .failure = "ERR unknown command" },
+    };
 }
 
-test "executeCommand UNKNOWN" {
-    var buf: [256]u8 = undefined;
-    const cmd = RedisCommand{ .cmd_type = .UNKNOWN, .key = "", .value = "" };
-    const result = executeCommand(cmd, &buf) orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("-ERR unknown command\r\n", result);
+pub fn encodePrefix(response: Response, output: []u8) ![]const u8 {
+    return switch (response) {
+        .simple => |value| std.fmt.bufPrint(output, "+{s}\r\n", .{value}),
+        .failure => |value| std.fmt.bufPrint(output, "-{s}\r\n", .{value}),
+        .integer => |value| std.fmt.bufPrint(output, ":{d}\r\n", .{value}),
+        .null_bulk => std.fmt.bufPrint(output, "$-1\r\n", .{}),
+        .bulk => |value| std.fmt.bufPrint(output, "${d}\r\n", .{value.bytes.len}),
+    };
 }
 
-test "executeCommand GET with insufficient buffer returns null" {
-    storage.init();
-    _ = storage.restore("overflow_key", "this value is far too long to fit in a tiny buffer");
-    const cmd = RedisCommand{ .cmd_type = .GET, .key = "overflow_key", .value = "" };
-
-    var tiny: [16]u8 = undefined;
-    try std.testing.expectEqual(@as(?[]const u8, null), executeCommand(cmd, tiny[0..]));
-
-    var enough: [512]u8 = undefined;
-    const resp = executeCommand(cmd, enough[0..]) orelse return error.TestUnexpectedResult;
-    try std.testing.expect(std.mem.startsWith(u8, resp, "$"));
+test "RESP binary parsing and pipelining" {
+    const first = "*3\r\n$3\r\nSET\r\n$3\r\na\x00b\r\n$4\r\nx\r\ny\r\n";
+    const second = "*2\r\n$3\r\nGET\r\n$3\r\na\x00b\r\n";
+    const bytes = first ++ second;
+    const parsed = try parse(bytes);
+    try std.testing.expectEqual(first.len, parsed.consumed);
+    try std.testing.expectEqualSlices(u8, "a\x00b", parsed.command.key);
+    const next = try parse(bytes[parsed.consumed..]);
+    try std.testing.expectEqual(CommandType.get, next.command.command_type);
+}
+
+test "RESP engine compatibility" {
+    var tmp = std.testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buffer: [std.fs.max_path_bytes]u8 = undefined;
+    const directory = try tmp.dir.realpath(".", &path_buffer);
+    const path = try std.fmt.allocPrint(std.testing.allocator, "{s}/resp.pkvdb", .{directory});
+    defer std.testing.allocator.free(path);
+    var engine = try engine_mod.Engine.open(std.testing.allocator, path);
+    defer engine.close();
+    var response = try execute(&engine, std.testing.allocator, .{ .command_type = .set, .key = "k", .value = "v" });
+    response.deinit(std.testing.allocator);
+    response = try execute(&engine, std.testing.allocator, .{ .command_type = .get, .key = "k" });
+    defer response.deinit(std.testing.allocator);
+    try std.testing.expectEqualStrings("v", response.bulk.bytes);
 }

socket.zig

@@ -27,10 +27,10 @@ pub fn init(host: []const u8, port: u16) !posix.socket_t {
     const listener = try posix.socket(address.any.family, tpe, protocol);
 
     try posix.setsockopt(listener, posix.SOL.SOCKET, posix.SO.REUSEADDR, &std.mem.toBytes(@as(c_int, 1)));
-    try posix.setsockopt(listener, posix.SOL.SOCKET, posix.SO.RCVBUF, &std.mem.toBytes(@as(c_int, 1048576))); // 1MB receive buffer
-    try posix.setsockopt(listener, posix.SOL.SOCKET, posix.SO.SNDBUF, &std.mem.toBytes(@as(c_int, 1048576))); // 1MB send buffer
+    try posix.setsockopt(listener, posix.SOL.SOCKET, posix.SO.RCVBUF, &std.mem.toBytes(@as(c_int, 1048576)));
+    try posix.setsockopt(listener, posix.SOL.SOCKET, posix.SO.SNDBUF, &std.mem.toBytes(@as(c_int, 1048576)));
     try posix.bind(listener, &address.any, address.getOsSockLen());
-    try posix.listen(listener, 1024); // Increased backlog
+    try posix.listen(listener, 1024);
 
     return listener;
 }
@@ -43,50 +43,3 @@ pub fn initUnix(path: []const u8) !posix.socket_t {
     try posix.listen(listener, 1024);
     return listener;
 }
-
-pub fn readUntilCR(conn: posix.socket_t, buf: []u8) !usize {
-    var total: usize = 0;
-
-    while (total < buf.len) {
-        const n = try posix.read(conn, buf[total..]);
-        if (n == 0) {
-            return if (total > 0) total else error.ConnectionClosed;
-        }
-
-        if (std.mem.indexOfScalar(u8, buf[total .. total + n], '\r')) |offset| {
-            return total + offset;
-        }
-
-        total += n;
-    }
-
-    return total;
-}
-
-pub fn read(conn: posix.socket_t, buf: []u8) !usize {
-    var pos: usize = 0;
-    while (pos < buf.len) {
-        const n = try posix.read(conn, buf[pos..]);
-        if (n == 0) {
-            return pos;
-        }
-        pos += n;
-    }
-    return pos;
-}
-
-pub fn write(conn: posix.socket_t, msg: []const u8) !void {
-    var offset: usize = 0;
-    while (offset < msg.len) {
-        const written = try posix.write(conn, msg[offset..]);
-        if (written == 0) return error.ConnectionClosed;
-        offset += written;
-    }
-}
-
-pub fn writev(conn: posix.socket_t, iovecs: []const posix.iovec_const) !void {
-    for (iovecs) |iov| {
-        const bytes: [*]const u8 = @ptrCast(iov.base);
-        try write(conn, bytes[0..iov.len]);
-    }
-}

storage.zig

@@ -1,334 +0,0 @@
-const std = @import("std");
-
-const index = @import("index.zig");
-const hashing = @import("hashing.zig");
-const persistence = @import("persistence.zig");
-
-const NUM_SHARDS = 64;
-const TOTAL_BUCKETS = 1_048_576;
-const BUCKETS_PER_SHARD = TOTAL_BUCKETS / NUM_SHARDS;
-
-const Entry = struct {
-    key: []const u8,
-    value: []const u8,
-    hash: u32,
-    next: ?*Entry,
-};
-
-const Shard = struct {
-    buckets: []?*Entry,
-    rwlock: std.Thread.RwLock,
-    arena: std.heap.ArenaAllocator,
-    allocator: std.mem.Allocator,
-};
-
-var shards: [NUM_SHARDS]Shard = undefined;
-var shards_initialized: bool = false;
-
-var init_mutex: std.Thread.Mutex = .{};
-
-fn getShardIndex(hash: u32) usize {
-    return hash % NUM_SHARDS;
-}
-
-pub fn init() void {
-    if (shards_initialized) return;
-
-    init_mutex.lock();
-    defer init_mutex.unlock();
-
-    if (!shards_initialized) {
-        for (&shards) |*shard| {
-            shard.arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-            shard.allocator = shard.arena.allocator();
-            shard.buckets = shard.allocator.alloc(?*Entry, BUCKETS_PER_SHARD) catch unreachable;
-            @memset(shard.buckets, null);
-            shard.rwlock = .{};
-        }
-        shards_initialized = true;
-    }
-}
-
-pub fn restore(key: []const u8, value: []const u8) bool {
-    const hash = hashing.hashKey(key);
-    const shard_idx = getShardIndex(hash);
-
-    var entry_key: []const u8 = undefined;
-    {
-        shards[shard_idx].rwlock.lock();
-        defer shards[shard_idx].rwlock.unlock();
-
-        const entry = writeVolatile(hash, key, value) orelse return false;
-        entry_key = entry.key;
-    }
-
-    index.insert(entry_key);
-    return true;
-}
-
-pub fn restoreDelete(key: []const u8) bool {
-    const hash = hashing.hashKey(key);
-    const shard_idx = getShardIndex(hash);
-
-    const deleted = blk: {
-        shards[shard_idx].rwlock.lock();
-        defer shards[shard_idx].rwlock.unlock();
-        break :blk deleteVolatile(hash, key);
-    };
-
-    if (deleted) {
-        index.delete(key);
-        return true;
-    }
-    return false;
-}
-
-pub fn writeVolatile(hash: u32, key: []const u8, value: []const u8) ?*Entry {
-    const shard_idx = getShardIndex(hash);
-    const bucketIdx = hash % shards[shard_idx].buckets.len;
-    const alloc = shards[shard_idx].allocator;
-
-    var current = shards[shard_idx].buckets[bucketIdx];
-    while (current) |entry| {
-        if (entry.hash == hash and std.mem.eql(u8, entry.key, key)) {
-            alloc.free(entry.value);
-            entry.value = alloc.dupe(u8, value) catch return null;
-            return entry;
-        }
-
-        current = entry.next;
-    }
-
-    const newEntry = alloc.create(Entry) catch return null;
-    errdefer alloc.destroy(newEntry);
-    newEntry.* = Entry{
-        .key = alloc.dupe(u8, key) catch return null,
-        .value = alloc.dupe(u8, value) catch return null,
-        .hash = hash, // Cache hash value
-        .next = shards[shard_idx].buckets[bucketIdx],
-    };
-
-    shards[shard_idx].buckets[bucketIdx] = newEntry;
-    return newEntry;
-}
-
-pub fn write(key: []const u8, value: []const u8) bool {
-    const hash = hashing.hashKey(key);
-    const shard_idx = getShardIndex(hash);
-
-    {
-        shards[shard_idx].rwlock.lock();
-        defer shards[shard_idx].rwlock.unlock();
-
-        // Persist before publishing the mutation so a durability failure is
-        // never returned after the new value has become visible in memory.
-        persistence.persist('W', key, value) catch return false;
-        const entry = writeVolatile(hash, key, value) orelse return false;
-        index.insert(entry.key);
-    }
-    return true;
-}
-
-pub fn read(key: []const u8) ?[]const u8 {
-    if (!shards_initialized) return null;
-
-    const hash = hashing.hashKey(key);
-    const shard_idx = getShardIndex(hash);
-
-    shards[shard_idx].rwlock.lockShared();
-    defer shards[shard_idx].rwlock.unlockShared();
-
-    var current = shards[shard_idx].buckets[hash % shards[shard_idx].buckets.len];
-    while (current) |entry| {
-        if (entry.hash == hash and std.mem.eql(u8, entry.key, key)) {
-            return entry.value;
-        }
-        current = entry.next;
-    }
-
-    return null;
-}
-
-pub fn readAlloc(key: []const u8, allocator: std.mem.Allocator) ?[]const u8 {
-    if (!shards_initialized) return null;
-
-    const hash = hashing.hashKey(key);
-    const shard_idx = getShardIndex(hash);
-    shards[shard_idx].rwlock.lockShared();
-    defer shards[shard_idx].rwlock.unlockShared();
-
-    var current = shards[shard_idx].buckets[hash % shards[shard_idx].buckets.len];
-    while (current) |entry| {
-        if (entry.hash == hash and std.mem.eql(u8, entry.key, key)) {
-            return allocator.dupe(u8, entry.value) catch null;
-        }
-        current = entry.next;
-    }
-    return null;
-}
-
-pub fn deleteVolatile(hash: u32, key: []const u8) bool {
-    if (!shards_initialized) return false;
-    const shard_idx = getShardIndex(hash);
-    const bucketIdx = hash % shards[shard_idx].buckets.len;
-    const alloc = shards[shard_idx].allocator;
-
-    var current = shards[shard_idx].buckets[bucketIdx];
-    var prev: ?*Entry = null;
-
-    while (current) |entry| {
-        if (entry.hash == hash and std.mem.eql(u8, entry.key, key)) {
-            if (prev) |p| {
-                p.next = entry.next;
-            } else {
-                shards[shard_idx].buckets[bucketIdx] = entry.next;
-            }
-
-            alloc.free(entry.key);
-            alloc.free(entry.value);
-            alloc.destroy(entry);
-            return true;
-        }
-
-        prev = entry;
-        current = entry.next;
-    }
-
-    return false;
-}
-
-pub fn delete(key: []const u8) bool {
-    const hash = hashing.hashKey(key);
-    const shard_idx = getShardIndex(hash);
-
-    shards[shard_idx].rwlock.lock();
-    defer shards[shard_idx].rwlock.unlock();
-
-    const bucket_idx = hash % shards[shard_idx].buckets.len;
-    var current = shards[shard_idx].buckets[bucket_idx];
-    var exists = false;
-    while (current) |entry| {
-        if (entry.hash == hash and std.mem.eql(u8, entry.key, key)) {
-            exists = true;
-            break;
-        }
-        current = entry.next;
-    }
-    if (!exists) return false;
-
-    // Keep the hash table and radix index unchanged if persistence fails.
-    persistence.persist('D', key, "") catch return false;
-    const deleted = deleteVolatile(hash, key);
-    if (deleted) {
-        index.delete(key);
-    }
-
-    return deleted;
-}
-
-// -- Tests --
-
-test "writeVolatile and read basic" {
-    init();
-    const hash = hashing.hashKey("test_key");
-    const shard_idx = getShardIndex(hash);
-
-    shards[shard_idx].rwlock.lock();
-    _ = writeVolatile(hash, "test_key", "test_value");
-    shards[shard_idx].rwlock.unlock();
-
-    const val = read("test_key") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("test_value", val);
-}
-
-test "writeVolatile overwrites existing key" {
-    init();
-    const hash = hashing.hashKey("overwrite_key");
-    const shard_idx = getShardIndex(hash);
-
-    shards[shard_idx].rwlock.lock();
-    _ = writeVolatile(hash, "overwrite_key", "first");
-    shards[shard_idx].rwlock.unlock();
-
-    shards[shard_idx].rwlock.lock();
-    _ = writeVolatile(hash, "overwrite_key", "second");
-    shards[shard_idx].rwlock.unlock();
-
-    const val = read("overwrite_key") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("second", val);
-}
-
-test "read nonexistent key returns null" {
-    init();
-    try std.testing.expectEqual(@as(?[]const u8, null), read("no_such_key_xyz"));
-}
-
-test "deleteVolatile removes entry" {
-    init();
-    const hash = hashing.hashKey("del_key");
-    const shard_idx = getShardIndex(hash);
-
-    shards[shard_idx].rwlock.lock();
-    _ = writeVolatile(hash, "del_key", "val");
-    shards[shard_idx].rwlock.unlock();
-
-    try std.testing.expect(read("del_key") != null);
-
-    shards[shard_idx].rwlock.lock();
-    const deleted = deleteVolatile(hash, "del_key");
-    shards[shard_idx].rwlock.unlock();
-
-    try std.testing.expect(deleted);
-    try std.testing.expectEqual(@as(?[]const u8, null), read("del_key"));
-}
-
-test "deleteVolatile nonexistent key returns false" {
-    init();
-    const hash = hashing.hashKey("ghost_key");
-    const shard_idx = getShardIndex(hash);
-
-    shards[shard_idx].rwlock.lock();
-    const deleted = deleteVolatile(hash, "ghost_key");
-    shards[shard_idx].rwlock.unlock();
-
-    try std.testing.expect(!deleted);
-}
-
-test "multiple keys in same shard" {
-    init();
-    // Write several keys and verify they don't interfere
-    const keys = [_][]const u8{ "shard_a", "shard_b", "shard_c" };
-    const vals = [_][]const u8{ "val_a", "val_b", "val_c" };
-
-    for (keys, vals) |k, v| {
-        const hash = hashing.hashKey(k);
-        const shard_idx = getShardIndex(hash);
-        shards[shard_idx].rwlock.lock();
-        _ = writeVolatile(hash, k, v);
-        shards[shard_idx].rwlock.unlock();
-    }
-
-    for (keys, vals) |k, v| {
-        const val = read(k) orelse return error.TestUnexpectedResult;
-        try std.testing.expectEqualStrings(v, val);
-    }
-}
-
-test "empty key and value" {
-    init();
-    const hash = hashing.hashKey("");
-    const shard_idx = getShardIndex(hash);
-
-    shards[shard_idx].rwlock.lock();
-    _ = writeVolatile(hash, "", "");
-    shards[shard_idx].rwlock.unlock();
-
-    const val = read("") orelse return error.TestUnexpectedResult;
-    try std.testing.expectEqualStrings("", val);
-}
-
-test "getShardIndex stays in bounds" {
-    try std.testing.expect(getShardIndex(0) < NUM_SHARDS);
-    try std.testing.expect(getShardIndex(std.math.maxInt(u32)) < NUM_SHARDS);
-    try std.testing.expect(getShardIndex(12345) < NUM_SHARDS);
-}

wal.zig

@@ -1,115 +0,0 @@
-const std = @import("std");
-
-var wal_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
-const wal_allocator = wal_arena.allocator();
-
-pub fn compactUPKVFile(filename: []const u8) !void {
-    const extension = std.fs.path.extension(filename);
-    if (!std.mem.eql(u8, extension, "upkv")) {
-        std.debug.print("Invalid file extension: {s}\n", .{extension});
-        return;
-    }
-
-    const records = try getRecordsFromFile(filename);
-    const compactedRecords = compact(records);
-    for (compactedRecords) |record| {
-        std.debug.print("{s}\n", .{record});
-    }
-}
-
-test "compactUPKVFile test" {}
-
-pub fn getRecordsFromFile(filename: []const u8) ![]const []const u8 {
-    const cwd = std.fs.cwd();
-    var upkv: ?std.fs.File = null;
-
-    upkv = cwd.openFile(filename, .{ .mode = .read_only }) catch |err| {
-        std.debug.print("Failed to open file: {any}\n", .{err});
-        return err;
-    };
-
-    const upkvData = upkv.?.readToEndAlloc(wal_allocator, 10_000_000 * 100) catch |err| {
-        std.debug.print("Failed to read file: {any}\n", .{err});
-        return err;
-    };
-
-    var records = std.ArrayListUnmanaged([]const u8){};
-    var it = std.mem.splitScalar(u8, upkvData, '\r');
-    while (it.next()) |record| {
-        if (record.len > 0) {
-            try records.append(wal_allocator, record);
-        }
-    }
-
-    return records.items;
-}
-
-test "getRecordsFromFile test" {
-    var tmp_dir = std.testing.tmpDir(.{});
-    defer tmp_dir.cleanup();
-
-    const file = try tmp_dir.dir.createFile("test.upkv", .{});
-    defer file.close();
-    try file.writeAll("W|key1|value1\rD|key2|\rW|key3|value3");
-
-    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
-    const tmp_path = try tmp_dir.dir.realpath(".", &path_buf);
-    const full_path = try std.fmt.allocPrint(std.testing.allocator, "{s}/test.upkv", .{tmp_path});
-    defer std.testing.allocator.free(full_path);
-
-    const records = try getRecordsFromFile(full_path);
-    try std.testing.expect(records.len == 3);
-    try std.testing.expectEqualStrings("W|key1|value1", records[0]);
-    try std.testing.expectEqualStrings("D|key2|", records[1]);
-    try std.testing.expectEqualStrings("W|key3|value3", records[2]);
-}
-
-pub fn compact(records: []const []const u8) []const []const u8 {
-    var compactMap = std.StringHashMap([]const u8).init(wal_allocator);
-
-    for (records) |record| {
-        var parts = std.mem.splitScalar(u8, record, '|');
-
-        const opcode = parts.next() orelse continue;
-        const key = parts.next() orelse continue;
-        const value = parts.next() orelse "";
-
-        if (std.mem.eql(u8, opcode, "D")) {
-            _ = compactMap.remove(key);
-        } else if (std.mem.eql(u8, opcode, "W")) {
-            compactMap.put(key, value) catch continue;
-        }
-    }
-
-    var compactedRecords = std.ArrayListUnmanaged([]const u8){};
-
-    var it = compactMap.iterator();
-    while (it.next()) |entry| {
-        const record = std.fmt.allocPrint(wal_allocator, "W|{s}|{s}", .{ entry.key_ptr.*, entry.value_ptr.* }) catch continue;
-        compactedRecords.append(wal_allocator, record) catch continue;
-    }
-
-    return compactedRecords.items;
-}
-
-test "compaction test" {
-    const records = &[_][]const u8{
-        "W|key1|value1v1",
-        "W|key1|value1v2",
-        "W|key2|value2v1",
-        "D|key2|",
-        "W|key3|value3v1",
-        "W|key4|value4v1",
-        "D|key4|",
-    };
-
-    const compactedExpected = &[_][]const u8{
-        "W|key1|value1v2",
-        "W|key3|value3v1",
-    };
-
-    const compacted = compact(records[0..]);
-    for (compacted, 0..) |record, i| {
-        try std.testing.expectEqualStrings(compactedExpected[i], record);
-    }
-}