From 2962864f3dab6f3f11bdb686512ddbfb62e1c35c Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 18 Mar 2026 07:16:30 -0700 Subject: [PATCH 01/48] create Cache interface file --- src/network/cache/Cache.zig | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 src/network/cache/Cache.zig diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig new file mode 100644 index 00000000..58a5c413 --- /dev/null +++ b/src/network/cache/Cache.zig @@ -0,0 +1,97 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); + +/// A browser-wide cache for resources across the network. +/// This mostly conforms to RFC9111 with regards to caching behavior. +pub const Cache = @This(); + +ptr: *anyopaque, +vtable: *const VTable, + +const VTable = struct { + get: *const fn (ptr: *anyopaque, allocator: std.mem.Allocator, key: []const u8) ?CachedResponse, + put: *const fn (ptr: *anyopaque, key: []const u8, metadata: CachedMetadata, body: []const u8) anyerror!void, +}; + +pub fn init(ptr: anytype) Cache { + const T = @TypeOf(ptr.*); + + return .{ + .ptr = ptr, + .vtable = &.{ + .get = T.get, + .put = T.put, + }, + }; +} + +pub fn get(self: Cache, allocator: std.mem.Allocator, key: []const u8) ?CachedResponse { + return self.vtable.get(self.ptr, allocator, key); +} + +pub fn put(self: Cache, key: []const u8, metadata: CachedMetadata, body: []const u8) !void { + return self.vtable.put(self.ptr, key, metadata, body); +} + +pub const CachedMetadata = struct { + url: [:0]const u8, + content_type: []const u8, + + status: u16, + stored_at: i64, + age_at_store: u64, + max_age: u64, + + // for If-None-Match + etag: ?[]const u8, + // for If-Modified-Since + last_modified: ?[]const u8, + + must_revalidate: bool, + no_cache: bool, + immutable: bool, + + // If non-null, must be incorporated into cache key. + vary: ?[]const u8, + + pub fn deinit(self: CachedMetadata, allocator: std.mem.Allocator) void { + allocator.free(self.url); + allocator.free(self.content_type); + if (self.etag) |e| allocator.free(e); + if (self.last_modified) |lm| allocator.free(lm); + if (self.vary) |v| allocator.free(v); + } + + pub fn isAgeStale(self: *const CachedMetadata) bool { + const now = std.time.timestamp(); + const age = now - self.stored_at + @as(i64, @intCast(self.age_at_store)); + return age < @as(i64, @intCast(self.max_age)); + } +}; + +pub const CachedData = union(enum) { + buffer: []const u8, + file: std.fs.File, +}; + +pub const CachedResponse = struct { + metadata: CachedMetadata, + data: CachedData, +}; From 6633b6effcfc48f63ba51a30f1c62542346e7d0d Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 18 Mar 2026 07:42:11 -0700 Subject: [PATCH 02/48] add cache dir to configuration opts --- src/Config.zig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/Config.zig b/src/Config.zig index 1e0e6f69..9ad6e8bc 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -156,6 +156,13 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 { }; } +pub fn cacheDir(self: *const Config) ?[]const u8 { + return switch (self.mode) { + inline .serve, .fetch, .mcp => |opts| opts.common.cache_dir, + else => null, + }; +} + pub fn cdpTimeout(self: *const Config) usize { return switch (self.mode) { .serve => |opts| if (opts.timeout > 604_800) 604_800_000 else @as(usize, opts.timeout) * 1000, @@ -273,6 +280,7 @@ pub const Common = struct { log_format: ?log.Format = null, log_filter_scopes: ?[]log.Scope = null, user_agent_suffix: ?[]const u8 = null, + cache_dir: ?[]const u8 = null, web_bot_auth_key_file: ?[]const u8 = null, web_bot_auth_keyid: ?[]const u8 = null, @@ -1066,5 +1074,14 @@ fn parseCommonArg( return true; } + if (std.mem.eql(u8, "--cache_dir", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--cache_dir" }); + return error.InvalidArgument; + }; + common.cache_dir = try allocator.dupe(u8, str); + return true; + } + return false; } From 9568c86326cb54c697fc055405fce4752e25645f Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 18 Mar 2026 11:26:10 -0700 Subject: [PATCH 03/48] allow Mime parse to use []const u8 --- src/browser/Mime.zig | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/browser/Mime.zig b/src/browser/Mime.zig index eca97a2d..27fed71e 100644 --- a/src/browser/Mime.zig +++ b/src/browser/Mime.zig @@ -112,14 +112,13 @@ fn parseCharset(value: []const u8) error{ CharsetTooBig, Invalid }![]const u8 { return value; } -pub fn parse(input: []u8) !Mime { +pub fn parse(input: []const u8) !Mime { if (input.len > 255) { return error.TooBig; } - // Zig's trim API is broken. The return type is always `[]const u8`, - // even if the input type is `[]u8`. @constCast is safe here. - var normalized = @constCast(std.mem.trim(u8, input, &std.ascii.whitespace)); + var buf: [255]u8 = undefined; + const normalized = std.ascii.lowerString(&buf, std.mem.trim(u8, input, &std.ascii.whitespace)); _ = std.ascii.lowerString(normalized, normalized); const content_type, const type_len = try parseContentType(normalized); From 4f78f299a3bf2736b46d0d614fde363d2820018f Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 18 Mar 2026 11:26:30 -0700 Subject: [PATCH 04/48] use Response instead of Transfer in callbacks --- src/browser/HttpClient.zig | 131 ++++++++++++++++++---- src/browser/Page.zig | 18 ++- src/browser/ScriptManager.zig | 105 ++++++++--------- src/browser/webapi/net/Fetch.zig | 42 +++---- src/browser/webapi/net/Response.zig | 10 +- src/browser/webapi/net/XMLHttpRequest.zig | 67 ++++++----- 6 files changed, 231 insertions(+), 142 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index e183bc6a..c1f623bf 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -399,8 +399,10 @@ fn fetchRobotsThenProcessRequest(self: *Client, robots_url: [:0]const u8, req: R try entry.value_ptr.append(self.allocator, req); } -fn robotsHeaderCallback(transfer: *Transfer) !bool { - const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx)); +fn robotsHeaderCallback(response: Response) !bool { + const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx)); + // Robots callbacks only happen on real live requests. + const transfer = response.inner.transfer; if (transfer.response_header) |hdr| { log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = ctx.robots_url }); @@ -414,8 +416,8 @@ fn robotsHeaderCallback(transfer: *Transfer) !bool { return true; } -fn robotsDataCallback(transfer: *Transfer, data: []const u8) !void { - const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx)); +fn robotsDataCallback(response: Response, data: []const u8) !void { + const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx)); try ctx.buffer.appendSlice(ctx.client.allocator, data); } @@ -634,13 +636,43 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer { .id = id, .url = req.url, .req = req, - .ctx = req.ctx, .client = self, .max_response_size = self.network.config.httpMaxResponseSize(), }; return transfer; } +fn requestFailed(transfer: *Transfer, err: anyerror, comptime execute_callback: bool) void { + if (transfer._notified_fail) { + // we can force a failed request within a callback, which will eventually + // result in this being called again in the more general loop. We do this + // because we can raise a more specific error inside a callback in some cases + return; + } + + transfer._notified_fail = true; + + transfer.req.notification.dispatch(.http_request_fail, &.{ + .transfer = transfer, + .err = err, + }); + + if (execute_callback) { + transfer.req.error_callback(transfer.req.ctx, err); + } else if (transfer.req.shutdown_callback) |cb| { + cb(transfer.req.ctx); + } +} + +// Same restriction as changeProxy. Should be ok since this is only called on +// BrowserContext deinit. +pub fn restoreOriginalProxy(self: *Client) !void { + try self.ensureNoActiveConnection(); + + self.http_proxy = self.network.config.httpProxy(); + self.use_proxy = self.http_proxy != null; +} + fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyerror!void { { // Reset per-response state for retries (auth challenge, queue). @@ -674,7 +706,7 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyer self.active += 1; if (transfer.req.start_callback) |cb| { - cb(transfer) catch |err| { + cb(Response.fromTransfer(transfer)) catch |err| { transfer.deinit(); return err; }; @@ -742,7 +774,10 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T // TODO give a way to configure the number of auth retries. if (transfer._auth_challenge != null and transfer._tries < 10) { var wait_for_interception = false; - transfer.req.notification.dispatch(.http_request_auth_required, &.{ .transfer = transfer, .wait_for_interception = &wait_for_interception }); + transfer.req.notification.dispatch( + .http_request_auth_required, + &.{ .transfer = transfer, .wait_for_interception = &wait_for_interception }, + ); if (wait_for_interception) { self.intercepted += 1; if (comptime IS_DEBUG) { @@ -844,7 +879,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T // Replay buffered body through user's data_callback. if (transfer._stream_buffer.items.len > 0) { const body = transfer._stream_buffer.items; - try transfer.req.data_callback(transfer, body); + try transfer.req.data_callback(Response.fromTransfer(transfer), body); transfer.req.notification.dispatch(.http_response_data, &.{ .data = body, @@ -861,7 +896,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T // will load more resources. transfer.releaseConn(); - try transfer.req.done_callback(transfer.ctx); + try transfer.req.done_callback(transfer.req.ctx); transfer.req.notification.dispatch(.http_request_done, &.{ .transfer = transfer, }); @@ -939,9 +974,9 @@ pub const Request = struct { // arbitrary data that can be associated with this request ctx: *anyopaque = undefined, - start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null, - header_callback: *const fn (transfer: *Transfer) anyerror!bool, - data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void, + start_callback: ?*const fn (response: Response) anyerror!void = null, + header_callback: *const fn (response: Response) anyerror!bool, + data_callback: *const fn (response: Response, data: []const u8) anyerror!void, done_callback: *const fn (ctx: *anyopaque) anyerror!void, error_callback: *const fn (ctx: *anyopaque, err: anyerror) void, shutdown_callback: ?*const fn (ctx: *anyopaque) void = null, @@ -967,12 +1002,66 @@ pub const Request = struct { }; }; +pub const Response = struct { + ctx: *anyopaque, + inner: union(enum) { + transfer: *Transfer, + }, + + pub fn fromTransfer(transfer: *Transfer) Response { + return .{ .ctx = transfer.req.ctx, .inner = .{ .transfer = transfer } }; + } + + pub fn status(self: Response) ?u16 { + return switch (self.inner) { + .transfer => |t| if (t.response_header) |rh| rh.status else null, + }; + } + + pub fn contentType(self: Response) ?[]const u8 { + return switch (self.inner) { + .transfer => |t| if (t.response_header) |*rh| rh.contentType() else null, + }; + } + + pub fn contentLength(self: Response) ?u32 { + return switch (self.inner) { + .transfer => |t| t.getContentLength(), + }; + } + + pub fn redirectCount(self: Response) ?u32 { + return switch (self.inner) { + .transfer => |t| if (t.response_header) |rh| rh.redirect_count else null, + }; + } + + pub fn url(self: Response) [:0]const u8 { + return switch (self.inner) { + .transfer => |t| t.url, + }; + } + + // TODO: Headers Iterator. + + pub fn abort(self: Response, err: anyerror) void { + switch (self.inner) { + .transfer => |t| t.abort(err), + } + } + + pub fn terminate(self: Response) void { + switch (self.inner) { + .transfer => |t| t.terminate(), + } + } +}; + pub const Transfer = struct { arena: ArenaAllocator, id: u32 = 0, req: Request, url: [:0]const u8, - ctx: *anyopaque, // copied from req.ctx to make it easier for callback handlers client: *Client, // total bytes received in the response, including the response status line, // the headers, and the [encoded] body. @@ -1065,7 +1154,7 @@ pub const Transfer = struct { // as abort (doesn't send a notification, doesn't invoke an error callback) fn kill(self: *Transfer) void { if (self.req.shutdown_callback) |cb| { - cb(self.ctx); + cb(self.req.ctx); } if (self._performing or self.client.performing) { @@ -1101,7 +1190,7 @@ pub const Transfer = struct { }); if (execute_callback) { - self.req.error_callback(self.ctx, err); + self.req.error_callback(self.req.ctx, err); } else if (self.req.shutdown_callback) |cb| { cb(self.ctx); } @@ -1352,7 +1441,7 @@ pub const Transfer = struct { .transfer = transfer, }); - const proceed = transfer.req.header_callback(transfer) catch |err| { + const proceed = transfer.req.header_callback(Response.fromTransfer(transfer)) catch |err| { log.err(.http, "header_callback", .{ .err = err, .req = transfer }); return err; }; @@ -1455,7 +1544,7 @@ pub const Transfer = struct { fn _fulfill(transfer: *Transfer, status: u16, headers: []const http.Header, body: ?[]const u8) !void { const req = &transfer.req; if (req.start_callback) |cb| { - try cb(transfer); + try cb(Response.fromTransfer(transfer)); } transfer.response_header = .{ @@ -1474,13 +1563,13 @@ pub const Transfer = struct { } lp.assert(transfer._header_done_called == false, "Transfer.fulfill header_done_called", .{}); - if (try req.header_callback(transfer) == false) { + if (try req.header_callback(Response.fromTransfer(transfer)) == false) { transfer.abort(error.Abort); return; } if (body) |b| { - try req.data_callback(transfer, b); + try req.data_callback(Response.fromTransfer(transfer), b); } try req.done_callback(req.ctx); @@ -1517,10 +1606,10 @@ pub const Transfer = struct { }; const Noop = struct { - fn headerCallback(_: *Transfer) !bool { + fn headerCallback(_: Response) !bool { return true; } - fn dataCallback(_: *Transfer, _: []const u8) !void {} + fn dataCallback(_: Response, _: []const u8) !void {} fn doneCallback(_: *anyopaque) !void {} fn errorCallback(_: *anyopaque, _: anyerror) void {} }; diff --git a/src/browser/Page.zig b/src/browser/Page.zig index df47ab60..b265a45d 100644 --- a/src/browser/Page.zig +++ b/src/browser/Page.zig @@ -886,12 +886,10 @@ fn notifyParentLoadComplete(self: *Page) void { parent.iframeCompletedLoading(self.iframe.?); } -fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { - var self: *Page = @ptrCast(@alignCast(transfer.ctx)); +fn pageHeaderDoneCallback(response: HttpClient.Response) !bool { + var self: *Page = @ptrCast(@alignCast(response.ctx)); - const header = &transfer.response_header.?; - - const response_url = std.mem.span(header.url); + const response_url = response.url(); if (std.mem.eql(u8, response_url, self.url) == false) { // would be different than self.url in the case of a redirect self.url = try self.arena.dupeZ(u8, response_url); @@ -905,8 +903,8 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { if (comptime IS_DEBUG) { log.debug(.page, "navigate header", .{ .url = self.url, - .status = header.status, - .content_type = header.contentType(), + .status = response.status(), + .content_type = response.contentType(), .type = self._type, }); } @@ -927,14 +925,14 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { return true; } -fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { - var self: *Page = @ptrCast(@alignCast(transfer.ctx)); +fn pageDataCallback(response: HttpClient.Response, data: []const u8) !void { + var self: *Page = @ptrCast(@alignCast(response.ctx)); if (self._parse_state == .pre) { // we lazily do this, because we might need the first chunk of data // to sniff the content type var mime: Mime = blk: { - if (transfer.response_header.?.contentType()) |ct| { + if (response.contentType()) |ct| { break :blk try Mime.parse(ct); } break :blk Mime.sniff(data); diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 39dd71bb..7733f80d 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -694,82 +694,85 @@ pub const Script = struct { self.manager.page.releaseArena(self.arena); } - fn startCallback(transfer: *HttpClient.Transfer) !void { - log.debug(.http, "script fetch start", .{ .req = transfer }); + fn startCallback(response: HttpClient.Response) !void { + log.debug(.http, "script fetch start", .{ .req = response }); } - fn headerCallback(transfer: *HttpClient.Transfer) !bool { - const self: *Script = @ptrCast(@alignCast(transfer.ctx)); - const header = &transfer.response_header.?; - self.status = header.status; - if (header.status != 200) { + fn headerCallback(response: HttpClient.Response) !bool { + const self: *Script = @ptrCast(@alignCast(response.ctx)); + + self.status = response.status().?; + if (response.status() != 200) { log.info(.http, "script header", .{ - .req = transfer, - .status = header.status, - .content_type = header.contentType(), + .req = response, + .status = response.status(), + .content_type = response.contentType(), }); return false; } if (comptime IS_DEBUG) { log.debug(.http, "script header", .{ - .req = transfer, - .status = header.status, - .content_type = header.contentType(), + .req = response, + .status = response.status(), + .content_type = response.contentType(), }); } - { - // temp debug, trying to figure out why the next assert sometimes - // fails. Is the buffer just corrupt or is headerCallback really - // being called twice? - lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{ - .m = @tagName(std.meta.activeTag(self.mode)), - .a1 = self.debug_transfer_id, - .a2 = self.debug_transfer_tries, - .a3 = self.debug_transfer_aborted, - .a4 = self.debug_transfer_bytes_received, - .a5 = self.debug_transfer_notified_fail, - .a7 = self.debug_transfer_intercept_state, - .a8 = self.debug_transfer_auth_challenge, - .a9 = self.debug_transfer_easy_id, - .b1 = transfer.id, - .b2 = transfer._tries, - .b3 = transfer.aborted, - .b4 = transfer.bytes_received, - .b5 = transfer._notified_fail, - .b7 = @intFromEnum(transfer._intercept_state), - .b8 = transfer._auth_challenge != null, - .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0, - }); - self.header_callback_called = true; - self.debug_transfer_id = transfer.id; - self.debug_transfer_tries = transfer._tries; - self.debug_transfer_aborted = transfer.aborted; - self.debug_transfer_bytes_received = transfer.bytes_received; - self.debug_transfer_notified_fail = transfer._notified_fail; - self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state); - self.debug_transfer_auth_challenge = transfer._auth_challenge != null; - self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0; + switch (response.inner) { + .transfer => |transfer| { + // temp debug, trying to figure out why the next assert sometimes + // fails. Is the buffer just corrupt or is headerCallback really + // being called twice? + lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{ + .m = @tagName(std.meta.activeTag(self.mode)), + .a1 = self.debug_transfer_id, + .a2 = self.debug_transfer_tries, + .a3 = self.debug_transfer_aborted, + .a4 = self.debug_transfer_bytes_received, + .a5 = self.debug_transfer_notified_fail, + .a7 = self.debug_transfer_intercept_state, + .a8 = self.debug_transfer_auth_challenge, + .a9 = self.debug_transfer_easy_id, + .b1 = transfer.id, + .b2 = transfer._tries, + .b3 = transfer.aborted, + .b4 = transfer.bytes_received, + .b5 = transfer._notified_fail, + .b7 = @intFromEnum(transfer._intercept_state), + .b8 = transfer._auth_challenge != null, + .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0, + }); + self.header_callback_called = true; + self.debug_transfer_id = transfer.id; + self.debug_transfer_tries = transfer._tries; + self.debug_transfer_aborted = transfer.aborted; + self.debug_transfer_bytes_received = transfer.bytes_received; + self.debug_transfer_notified_fail = transfer._notified_fail; + self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state); + self.debug_transfer_auth_challenge = transfer._auth_challenge != null; + self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0; + }, } lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity }); var buffer: std.ArrayList(u8) = .empty; - if (transfer.getContentLength()) |cl| { + if (response.contentLength()) |cl| { try buffer.ensureTotalCapacity(self.arena, cl); } self.source = .{ .remote = buffer }; return true; } - fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { - const self: *Script = @ptrCast(@alignCast(transfer.ctx)); - self._dataCallback(transfer, data) catch |err| { - log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len }); + fn dataCallback(response: HttpClient.Response, data: []const u8) !void { + const self: *Script = @ptrCast(@alignCast(response.ctx)); + self._dataCallback(response, data) catch |err| { + log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = response, .len = data.len }); return err; }; } - fn _dataCallback(self: *Script, _: *HttpClient.Transfer, data: []const u8) !void { + + fn _dataCallback(self: *Script, _: HttpClient.Response, data: []const u8) !void { try self.source.remote.appendSlice(self.arena, data); } diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig index acda1f86..dc42f9b2 100644 --- a/src/browser/webapi/net/Fetch.zig +++ b/src/browser/webapi/net/Fetch.zig @@ -127,16 +127,16 @@ fn handleBlobUrl(url: []const u8, resolver: js.PromiseResolver, page: *Page) !js return resolver.promise(); } -fn httpStartCallback(transfer: *HttpClient.Transfer) !void { - const self: *Fetch = @ptrCast(@alignCast(transfer.ctx)); +fn httpStartCallback(response: HttpClient.Response) !void { + const self: *Fetch = @ptrCast(@alignCast(response.ctx)); if (comptime IS_DEBUG) { log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" }); } - self._response._transfer = transfer; + self._response._http_response = response; } -fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { - const self: *Fetch = @ptrCast(@alignCast(transfer.ctx)); +fn httpHeaderDoneCallback(response: HttpClient.Response) !bool { + const self: *Fetch = @ptrCast(@alignCast(response.ctx)); if (self._signal) |signal| { if (signal._aborted) { @@ -145,25 +145,24 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { } const arena = self._response._arena; - if (transfer.getContentLength()) |cl| { + if (response.contentLength()) |cl| { try self._buf.ensureTotalCapacity(arena, cl); } const res = self._response; - const header = transfer.response_header.?; if (comptime IS_DEBUG) { log.debug(.http, "request header", .{ .source = "fetch", .url = self._url, - .status = header.status, + .status = response.status(), }); } - res._status = header.status; - res._status_text = std.http.Status.phrase(@enumFromInt(header.status)) orelse ""; - res._url = try arena.dupeZ(u8, std.mem.span(header.url)); - res._is_redirected = header.redirect_count > 0; + res._status = response.status().?; + res._status_text = std.http.Status.phrase(@enumFromInt(response.status().?)) orelse ""; + res._url = try arena.dupeZ(u8, response.url()); + res._is_redirected = response.redirectCount().? > 0; // Determine response type based on origin comparison const page_origin = URL.getOrigin(arena, self._page.url) catch null; @@ -183,16 +182,17 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { res._type = .basic; } - var it = transfer.responseHeaderIterator(); - while (it.next()) |hdr| { - try res._headers.append(hdr.name, hdr.value, self._page); - } + // TODO: Header Iterator + // var it = transfer.responseHeaderIterator(); + // while (it.next()) |hdr| { + // try res._headers.append(hdr.name, hdr.value, self._page); + // } return true; } -fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { - const self: *Fetch = @ptrCast(@alignCast(transfer.ctx)); +fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void { + const self: *Fetch = @ptrCast(@alignCast(response.ctx)); // Check if aborted if (self._signal) |signal| { @@ -207,7 +207,7 @@ fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { fn httpDoneCallback(ctx: *anyopaque) !void { const self: *Fetch = @ptrCast(@alignCast(ctx)); var response = self._response; - response._transfer = null; + response._http_response = null; response._body = self._buf.items; log.info(.http, "request complete", .{ @@ -230,7 +230,7 @@ fn httpErrorCallback(ctx: *anyopaque, _: anyerror) void { const self: *Fetch = @ptrCast(@alignCast(ctx)); var response = self._response; - response._transfer = null; + response._http_response = null; // the response is only passed on v8 on success, if we're here, it's safe to // clear this. (defer since `self is in the response's arena). @@ -256,7 +256,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void { if (self._owns_response) { var response = self._response; - response._transfer = null; + response._http_response = null; response.deinit(self._page._session); // Do not access `self` after this point: the Fetch struct was // allocated from response._arena which has been released. diff --git a/src/browser/webapi/net/Response.zig b/src/browser/webapi/net/Response.zig index b9df006e..e4fbd46d 100644 --- a/src/browser/webapi/net/Response.zig +++ b/src/browser/webapi/net/Response.zig @@ -48,7 +48,7 @@ _type: Type, _status_text: []const u8, _url: [:0]const u8, _is_redirected: bool, -_transfer: ?*HttpClient.Transfer = null, +_http_response: ?HttpClient.Response = null, const InitOpts = struct { status: u16 = 200, @@ -81,9 +81,9 @@ pub fn init(body_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*Response { } pub fn deinit(self: *Response, session: *Session) void { - if (self._transfer) |transfer| { - transfer.abort(error.Abort); - self._transfer = null; + if (self._http_response) |resp| { + resp.abort(error.Abort); + self._http_response = null; } session.releaseArena(self._arena); } @@ -191,7 +191,7 @@ pub fn clone(self: *const Response, page: *Page) !*Response { ._type = self._type, ._is_redirected = self._is_redirected, ._headers = try Headers.init(.{ .obj = self._headers }, page), - ._transfer = null, + ._http_response = null, }; return cloned; } diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig index d34e2f2b..1a02e627 100644 --- a/src/browser/webapi/net/XMLHttpRequest.zig +++ b/src/browser/webapi/net/XMLHttpRequest.zig @@ -43,7 +43,7 @@ _rc: lp.RC(u8) = .{}, _page: *Page, _proto: *XMLHttpRequestEventTarget, _arena: Allocator, -_transfer: ?*HttpClient.Transfer = null, +_http_response: ?HttpClient.Response = null, _active_request: bool = false, _url: [:0]const u8 = "", @@ -100,9 +100,9 @@ pub fn init(page: *Page) !*XMLHttpRequest { } pub fn deinit(self: *XMLHttpRequest, session: *Session) void { - if (self._transfer) |transfer| { - transfer.abort(error.Abort); - self._transfer = null; + if (self._http_response) |resp| { + resp.abort(error.Abort); + self._http_response = null; } if (self._on_ready_state_change) |func| { @@ -184,9 +184,9 @@ pub fn setWithCredentials(self: *XMLHttpRequest, value: bool) !void { // TODO: url should be a union, as it can be multiple things pub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void { // Abort any in-progress request - if (self._transfer) |transfer| { + if (self._http_response) |transfer| { transfer.abort(error.Abort); - self._transfer = null; + self._http_response = null; } // Reset internal state @@ -402,34 +402,32 @@ pub fn getResponseXML(self: *XMLHttpRequest, page: *Page) !?*Node.Document { }; } -fn httpStartCallback(transfer: *HttpClient.Transfer) !void { - const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); +fn httpStartCallback(response: HttpClient.Response) !void { + const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx)); if (comptime IS_DEBUG) { log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" }); } - self._transfer = transfer; + self._http_response = response; } -fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: http.Header) !void { - const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); +fn httpHeaderCallback(response: HttpClient.Response, header: http.Header) !void { + const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx)); const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value }); try self._response_headers.append(self._arena, joined); } -fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { - const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); - - const header = &transfer.response_header.?; +fn httpHeaderDoneCallback(response: HttpClient.Response) !bool { + const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx)); if (comptime IS_DEBUG) { log.debug(.http, "request header", .{ .source = "xhr", .url = self._url, - .status = header.status, + .status = response.status(), }); } - if (header.contentType()) |ct| { + if (response.contentType()) |ct| { self._response_mime = Mime.parse(ct) catch |e| { log.info(.http, "invalid content type", .{ .content_Type = ct, @@ -440,18 +438,19 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { }; } - var it = transfer.responseHeaderIterator(); - while (it.next()) |hdr| { - const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value }); - try self._response_headers.append(self._arena, joined); - } + // TODO: Header Iterator + // var it = transfer.responseHeaderIterator(); + // while (it.next()) |hdr| { + // const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value }); + // try self._response_headers.append(self._arena, joined); + // } - self._response_status = header.status; - if (transfer.getContentLength()) |cl| { + self._response_status = response.status().?; + if (response.contentLength()) |cl| { self._response_len = cl; try self._response_data.ensureTotalCapacity(self._arena, cl); } - self._response_url = try self._arena.dupeZ(u8, std.mem.span(header.url)); + self._response_url = try self._arena.dupeZ(u8, response.url()); const page = self._page; @@ -466,8 +465,8 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { return true; } -fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { - const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); +fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void { + const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx)); try self._response_data.appendSlice(self._arena, data); const page = self._page; @@ -490,7 +489,7 @@ fn httpDoneCallback(ctx: *anyopaque) !void { // Not that the request is done, the http/client will free the transfer // object. It isn't safe to keep it around. - self._transfer = null; + self._http_response = null; const page = self._page; @@ -513,23 +512,23 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void { const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx)); // http client will close it after an error, it isn't safe to keep around self.handleError(err); - if (self._transfer != null) { - self._transfer = null; + if (self._http_response != null) { + self._http_response = null; } self.releaseSelfRef(); } fn httpShutdownCallback(ctx: *anyopaque) void { const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx)); - self._transfer = null; + self._http_response = null; self.releaseSelfRef(); } pub fn abort(self: *XMLHttpRequest) void { self.handleError(error.Abort); - if (self._transfer) |transfer| { - self._transfer = null; - transfer.abort(error.Abort); + if (self._http_response) |resp| { + self._http_response = null; + resp.abort(error.Abort); } self.releaseSelfRef(); } From 070baa8f46fbe928f3379be1bc0d427e9ed574b8 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 18 Mar 2026 11:34:44 -0700 Subject: [PATCH 05/48] add headerIterator to Http Response --- src/browser/HttpClient.zig | 6 +++++- src/browser/webapi/net/Fetch.zig | 9 ++++----- src/browser/webapi/net/XMLHttpRequest.zig | 11 +++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index c1f623bf..41b5dd93 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -1042,7 +1042,11 @@ pub const Response = struct { }; } - // TODO: Headers Iterator. + pub fn headerIterator(self: Response) HeaderIterator { + return switch (self.inner) { + .live => |live| live.responseHeaderIterator(), + }; + } pub fn abort(self: Response, err: anyerror) void { switch (self.inner) { diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig index dc42f9b2..d26771e2 100644 --- a/src/browser/webapi/net/Fetch.zig +++ b/src/browser/webapi/net/Fetch.zig @@ -182,11 +182,10 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !bool { res._type = .basic; } - // TODO: Header Iterator - // var it = transfer.responseHeaderIterator(); - // while (it.next()) |hdr| { - // try res._headers.append(hdr.name, hdr.value, self._page); - // } + var it = response.headerIterator(); + while (it.next()) |hdr| { + try res._headers.append(hdr.name, hdr.value, self._page); + } return true; } diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig index 1a02e627..9024a1f7 100644 --- a/src/browser/webapi/net/XMLHttpRequest.zig +++ b/src/browser/webapi/net/XMLHttpRequest.zig @@ -438,12 +438,11 @@ fn httpHeaderDoneCallback(response: HttpClient.Response) !bool { }; } - // TODO: Header Iterator - // var it = transfer.responseHeaderIterator(); - // while (it.next()) |hdr| { - // const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value }); - // try self._response_headers.append(self._arena, joined); - // } + var it = response.headerIterator(); + while (it.next()) |hdr| { + const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value }); + try self._response_headers.append(self._arena, joined); + } self._response_status = response.status().?; if (response.contentLength()) |cl| { From d9a3d912c0d6602ff275b0c2c6b9363d0e15d502 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 18 Mar 2026 11:38:39 -0700 Subject: [PATCH 06/48] add CachedResponse variant to Response --- src/browser/HttpClient.zig | 26 +++++++++++++++++++------- src/browser/ScriptManager.zig | 1 + 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 41b5dd93..7988c839 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -32,6 +32,8 @@ const CookieJar = @import("webapi/storage/Cookie.zig").Jar; const http = @import("../network/http.zig"); const Network = @import("../network/Network.zig"); const Robots = @import("../network/Robots.zig"); +const Cache = @import("../network/cache/Cache.zig"); +const CachedResponse = Cache.CachedResponse; const IS_DEBUG = builtin.mode == .Debug; @@ -1006,57 +1008,67 @@ pub const Response = struct { ctx: *anyopaque, inner: union(enum) { transfer: *Transfer, + cached: *const CachedResponse, }, pub fn fromTransfer(transfer: *Transfer) Response { return .{ .ctx = transfer.req.ctx, .inner = .{ .transfer = transfer } }; } + pub fn fromCached(ctx: *anyopaque, resp: *const CachedResponse) Response { + return .{ .ctx = ctx, .inner = .{ .cached = resp } }; + } + pub fn status(self: Response) ?u16 { return switch (self.inner) { .transfer => |t| if (t.response_header) |rh| rh.status else null, + .cached => |c| c.metadata.status, }; } pub fn contentType(self: Response) ?[]const u8 { return switch (self.inner) { .transfer => |t| if (t.response_header) |*rh| rh.contentType() else null, + .cached => |c| c.metadata.content_type, }; } pub fn contentLength(self: Response) ?u32 { return switch (self.inner) { .transfer => |t| t.getContentLength(), + .cached => |c| switch (c.data) { + .buffer => |buf| @intCast(buf.len), + .file => |f| @intCast(f.getEndPos() catch 0), + }, }; } pub fn redirectCount(self: Response) ?u32 { return switch (self.inner) { .transfer => |t| if (t.response_header) |rh| rh.redirect_count else null, + .cached => 0, }; } pub fn url(self: Response) [:0]const u8 { return switch (self.inner) { .transfer => |t| t.url, + .cached => |c| c.metadata.url, }; } pub fn headerIterator(self: Response) HeaderIterator { return switch (self.inner) { - .live => |live| live.responseHeaderIterator(), + .transfer => |t| t.responseHeaderIterator(), + // TODO: Cache HTTP Headers + .cached => unreachable, }; } pub fn abort(self: Response, err: anyerror) void { switch (self.inner) { .transfer => |t| t.abort(err), - } - } - - pub fn terminate(self: Response) void { - switch (self.inner) { - .transfer => |t| t.terminate(), + .cached => {}, } } }; diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 7733f80d..4bd4148c 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -753,6 +753,7 @@ pub const Script = struct { self.debug_transfer_auth_challenge = transfer._auth_challenge != null; self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0; }, + else => {}, } lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity }); From 41a24623fa359a02603dabf01f153673f0984f64 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 19 Mar 2026 18:49:40 -0700 Subject: [PATCH 07/48] add basic FsCache impl --- src/network/cache/FsCache.zig | 239 ++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 src/network/cache/FsCache.zig diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig new file mode 100644 index 00000000..db916cfa --- /dev/null +++ b/src/network/cache/FsCache.zig @@ -0,0 +1,239 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const Cache = @import("Cache.zig"); +const CachedMetadata = Cache.CachedMetadata; +const CachedResponse = Cache.CachedResponse; + +pub const FsCache = @This(); + +dir: std.fs.Dir, + +pub fn init(path: []const u8) !FsCache { + const cwd = std.fs.cwd(); + + cwd.makeDir(path) catch |err| switch (err) { + error.PathAlreadyExists => {}, + else => return err, + }; + + const dir = try cwd.openDir(path, .{ .iterate = true }); + return .{ .dir = dir }; +} + +pub fn deinit(self: *FsCache) void { + self.dir.close(); +} + +pub fn cache(self: *FsCache) Cache { + return Cache.init(self); +} + +const HASHED_KEY_LEN = 16; +const HASHED_PATH_LEN = HASHED_KEY_LEN + 5; +const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4; + +fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 { + const h = std.hash.Wyhash.hash(0, key); + var hex: [HASHED_KEY_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&hex, "{x:0>16}", .{h}) catch unreachable; + return hex; +} + +fn serializeMeta(writer: *std.Io.Writer, meta: *const CachedMetadata) !void { + try writer.print("{s}\n{s}\n", .{ meta.url, meta.content_type }); + try writer.print("{d}\n{d}\n{d}\n{d}\n", .{ + meta.status, + meta.stored_at, + meta.age_at_store, + meta.max_age, + }); + try writer.print("{s}\n", .{meta.etag orelse "null"}); + try writer.print("{s}\n", .{meta.last_modified orelse "null"}); + try writer.print("{s}\n", .{meta.vary orelse "null"}); + try writer.print("{}\n{}\n{}\n", .{ + meta.must_revalidate, + meta.no_cache, + meta.immutable, + }); + try writer.flush(); +} + +fn deserializeMetaOptionalString(bytes: []const u8) ?[]const u8 { + if (std.mem.eql(u8, bytes, "null")) return null else return bytes; +} + +fn deserializeMetaBoolean(bytes: []const u8) !bool { + if (std.mem.eql(u8, bytes, "true")) return true; + if (std.mem.eql(u8, bytes, "false")) return false; + return error.Malformed; +} + +fn deserializeMeta(allocator: std.mem.Allocator, file: std.fs.File) !CachedMetadata { + var file_buf: [1024]u8 = undefined; + var file_reader = file.reader(&file_buf); + const reader = &file_reader.interface; + + const url = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try allocator.dupeZ(u8, line); + }; + errdefer allocator.free(url); + + const content_type = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try allocator.dupe(u8, line); + }; + errdefer allocator.free(content_type); + + const status = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk std.fmt.parseInt(u16, line, 10) catch return error.Malformed; + }; + const stored_at = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk std.fmt.parseInt(i64, line, 10) catch return error.Malformed; + }; + const age_at_store = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk std.fmt.parseInt(u64, line, 10) catch return error.Malformed; + }; + const max_age = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk std.fmt.parseInt(u64, line, 10) catch return error.Malformed; + }; + + const etag = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); + }; + const last_modified = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); + }; + const vary = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); + }; + + const must_revalidate = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try deserializeMetaBoolean(line); + }; + const no_cache = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try deserializeMetaBoolean(line); + }; + const immutable = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try deserializeMetaBoolean(line); + }; + + return .{ + .url = url, + .content_type = content_type, + .status = status, + .stored_at = stored_at, + .age_at_store = age_at_store, + .max_age = max_age, + .etag = etag, + .last_modified = last_modified, + .must_revalidate = must_revalidate, + .no_cache = no_cache, + .immutable = immutable, + .vary = vary, + }; +} + +pub fn get(self: *FsCache, allocator: std.mem.Allocator, key: []const u8) ?Cache.CachedResponse { + const hashed_key = hashKey(key); + + var meta_path: [HASHED_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&meta_path, "{s}.meta", .{hashed_key}) catch @panic("FsCache.get meta path overflowed"); + + var body_path: [HASHED_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&body_path, "{s}.body", .{hashed_key}) catch @panic("FsCache.get body path overflowed"); + + const meta_file = self.dir.openFile(&meta_path, .{ .mode = .read_only }) catch return null; + defer meta_file.close(); + + const meta = deserializeMeta(allocator, meta_file) catch { + self.dir.deleteFile(&meta_path) catch {}; + self.dir.deleteFile(&body_path) catch {}; + return null; + }; + + const body_file = self.dir.openFile(&body_path, .{ .mode = .read_only }) catch return null; + + return .{ + .metadata = meta, + .data = .{ .file = body_file }, + }; +} + +pub fn put(self: *FsCache, key: []const u8, meta: CachedMetadata, body: []const u8) !void { + const hashed_key = hashKey(key); + + // Write meta to a temp file, then atomically rename into place + var meta_path: [HASHED_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&meta_path, "{s}.meta", .{hashed_key}) catch + @panic("FsCache.put meta path overflowed"); + + var meta_tmp_path: [HASHED_TMP_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&meta_tmp_path, "{s}.meta.tmp", .{hashed_key}) catch + @panic("FsCache.put meta tmp path overflowed"); + + { + const meta_file = try self.dir.createFile(&meta_tmp_path, .{}); + errdefer { + meta_file.close(); + self.dir.deleteFile(&meta_tmp_path) catch {}; + } + + var buf: [512]u8 = undefined; + var meta_file_writer = meta_file.writer(&buf); + try serializeMeta(&meta_file_writer.interface, &meta); + meta_file.close(); + } + errdefer self.dir.deleteFile(&meta_tmp_path) catch {}; + try self.dir.rename(&meta_tmp_path, &meta_path); + + // Write body to a temp file, then atomically rename into place + var body_path: [HASHED_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&body_path, "{s}.body", .{hashed_key}) catch + @panic("FsCache.put body path overflowed"); + + var body_tmp_path: [HASHED_TMP_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&body_tmp_path, "{s}.body.tmp", .{hashed_key}) catch + @panic("FsCache.put body tmp path overflowed"); + + { + const body_file = try self.dir.createFile(&body_tmp_path, .{}); + errdefer { + body_file.close(); + self.dir.deleteFile(&body_tmp_path) catch {}; + } + try body_file.writeAll(body); + body_file.close(); + } + errdefer self.dir.deleteFile(&body_tmp_path) catch {}; + + errdefer self.dir.deleteFile(&meta_path) catch {}; + try self.dir.rename(&body_tmp_path, &body_path); +} From 647d989191eed762a69af2efa53370879d7d9275 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 19 Mar 2026 18:49:54 -0700 Subject: [PATCH 08/48] use enum approach instead of vtable --- src/network/cache/Cache.zig | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 58a5c413..56bc6c10 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -17,37 +17,26 @@ // along with this program. If not, see . const std = @import("std"); +const FsCache = @import("FsCache.zig"); /// A browser-wide cache for resources across the network. /// This mostly conforms to RFC9111 with regards to caching behavior. pub const Cache = @This(); -ptr: *anyopaque, -vtable: *const VTable, +kind: union(enum) { + fs: FsCache, +}, -const VTable = struct { - get: *const fn (ptr: *anyopaque, allocator: std.mem.Allocator, key: []const u8) ?CachedResponse, - put: *const fn (ptr: *anyopaque, key: []const u8, metadata: CachedMetadata, body: []const u8) anyerror!void, -}; - -pub fn init(ptr: anytype) Cache { - const T = @TypeOf(ptr.*); - - return .{ - .ptr = ptr, - .vtable = &.{ - .get = T.get, - .put = T.put, - }, +pub fn get(self: *Cache, allocator: std.mem.Allocator, key: []const u8) ?CachedResponse { + return switch (self.kind) { + inline else => |*c| c.get(allocator, key), }; } -pub fn get(self: Cache, allocator: std.mem.Allocator, key: []const u8) ?CachedResponse { - return self.vtable.get(self.ptr, allocator, key); -} - -pub fn put(self: Cache, key: []const u8, metadata: CachedMetadata, body: []const u8) !void { - return self.vtable.put(self.ptr, key, metadata, body); +pub fn put(self: *Cache, key: []const u8, metadata: CachedMetadata, body: []const u8) !void { + return switch (self.kind) { + inline else => |*c| c.put(key, metadata, body), + }; } pub const CachedMetadata = struct { From 349d5a0a0b7ef34b9e5ca007cc4182475ef9e8cf Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 19 Mar 2026 18:50:05 -0700 Subject: [PATCH 09/48] create cache owned by the network struct --- src/network/Network.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/network/Network.zig b/src/network/Network.zig index fad62652..1c347c52 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -29,6 +29,7 @@ const libcurl = @import("../sys/libcurl.zig"); const http = @import("http.zig"); const RobotStore = @import("Robots.zig").RobotStore; const WebBotAuth = @import("WebBotAuth.zig"); +const Cache = @import("cache/Cache.zig"); const Network = @This(); @@ -49,6 +50,7 @@ config: *const Config, ca_blob: ?http.Blob, robot_store: RobotStore, web_bot_auth: ?WebBotAuth, +cache: ?Cache, connections: []http.Connection, available: std.DoublyLinkedList = .{}, @@ -233,6 +235,11 @@ pub fn init(allocator: Allocator, config: *const Config) !Network { else null; + const cache = if (config.cacheDir()) |cache_dir_path| + Cache{ .kind = .{ .fs = try .init(cache_dir_path) } } + else + null; + return .{ .allocator = allocator, .config = config, @@ -246,6 +253,7 @@ pub fn init(allocator: Allocator, config: *const Config) !Network { .robot_store = RobotStore.init(allocator), .web_bot_auth = web_bot_auth, + .cache = cache, }; } From 02f611bbc89fef33734faa4dc3b09a3ddcfbf3f8 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 19 Mar 2026 18:59:46 -0700 Subject: [PATCH 10/48] add basic caching support --- src/browser/HttpClient.zig | 85 +++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 7988c839..91b696b9 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -313,7 +313,62 @@ pub fn request(self: *Client, req: Request) !void { return self.fetchRobotsThenProcessRequest(robots_url, req); } +fn serveFromCache(allocator: std.mem.Allocator, req: Request, cached: *const CachedResponse) !void { + const response = Response.fromCached(req.ctx, cached); + defer cached.metadata.deinit(allocator); + + if (req.start_callback) |cb| { + try cb(response); + } + + const proceed = try req.header_callback(response); + if (!proceed) { + req.error_callback(req.ctx, error.Abort); + return; + } + + switch (cached.data) { + .buffer => |data| { + if (data.len > 0) { + try req.data_callback(response, data); + } + }, + .file => |file| { + var buf: [1024]u8 = undefined; + var file_reader = file.reader(&buf); + + const reader = &file_reader.interface; + var read_buf: [1024]u8 = undefined; + + while (true) { + const curr = try reader.readSliceShort(&read_buf); + if (curr == 0) break; + try req.data_callback(response, read_buf[0..curr]); + } + }, + } + + try req.done_callback(req.ctx); +} + fn processRequest(self: *Client, req: Request) !void { + if (self.network.cache) |*cache| { + if (req.method == .GET) { + if (cache.get(self.allocator, req.url)) |cached| { + log.debug(.browser, "http.cache.get", .{ + .url = req.url, + .found = true, + .metadata = cached.metadata, + }); + + defer req.headers.deinit(); + return serveFromCache(self.allocator, req, &cached); + } else { + log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = false }); + } + } + } + const transfer = try self.makeTransfer(req); transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer }); @@ -878,9 +933,10 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T } } + const body = transfer._stream_buffer.items; + // Replay buffered body through user's data_callback. if (transfer._stream_buffer.items.len > 0) { - const body = transfer._stream_buffer.items; try transfer.req.data_callback(Response.fromTransfer(transfer), body); transfer.req.notification.dispatch(.http_response_data, &.{ @@ -899,6 +955,33 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T transfer.releaseConn(); try transfer.req.done_callback(transfer.req.ctx); + + if (self.network.cache) |*cache| { + var headers = &transfer.response_header.?; + + if (transfer.req.method == .GET and headers.status == 200) { + cache.put( + transfer.req.url, + .{ + .url = transfer.req.url, + .content_type = headers.contentType() orelse "application/octet-stream", + .status = headers.status, + .stored_at = std.time.timestamp(), + .age_at_store = 0, + .max_age = 3600, + .etag = null, + .last_modified = null, + .must_revalidate = false, + .no_cache = false, + .immutable = false, + .vary = null, + }, + body, + ) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); + log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); + } + } + transfer.req.notification.dispatch(.http_request_done, &.{ .transfer = transfer, }); From 29dfbbfdea82f8ec2cd68107986f1fde47bb3762 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Fri, 20 Mar 2026 01:34:51 -0700 Subject: [PATCH 11/48] cache headers along with response --- src/browser/HttpClient.zig | 56 ++++++++++------- src/network/cache/Cache.zig | 111 +++++++++++++++++++++++++++++++++- src/network/cache/FsCache.zig | 34 +++++++++++ 3 files changed, 177 insertions(+), 24 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 91b696b9..012d8270 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -33,6 +33,7 @@ const http = @import("../network/http.zig"); const Network = @import("../network/Network.zig"); const Robots = @import("../network/Robots.zig"); const Cache = @import("../network/cache/Cache.zig"); +const CacheMetadata = Cache.CachedMetadata; const CachedResponse = Cache.CachedResponse; const IS_DEBUG = builtin.mode == .Debug; @@ -950,34 +951,46 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T } } + const allocator = transfer.arena.allocator(); + var header_list: std.ArrayList(http.Header) = .empty; + + var it = transfer.responseHeaderIterator(); + while (it.next()) |hdr| { + header_list.append( + allocator, + .{ + .name = try allocator.dupe(u8, hdr.name), + .value = try allocator.dupe(u8, hdr.value), + }, + ) catch |err| { + log.warn(.http, "cache header collect failed", .{ .err = err }); + break; + }; + } + // release conn ASAP so that it's available; some done_callbacks // will load more resources. transfer.releaseConn(); try transfer.req.done_callback(transfer.req.ctx); - if (self.network.cache) |*cache| { - var headers = &transfer.response_header.?; + cache: { + if (self.network.cache) |*cache| { + const headers = &transfer.response_header.?; - if (transfer.req.method == .GET and headers.status == 200) { - cache.put( + const metadata = try CacheMetadata.fromHeaders( transfer.req.url, - .{ - .url = transfer.req.url, - .content_type = headers.contentType() orelse "application/octet-stream", - .status = headers.status, - .stored_at = std.time.timestamp(), - .age_at_store = 0, - .max_age = 3600, - .etag = null, - .last_modified = null, - .must_revalidate = false, - .no_cache = false, - .immutable = false, - .vary = null, - }, - body, - ) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); + headers.status, + std.time.timestamp(), + header_list.items, + ) orelse break :cache; + + // TODO: Support Vary Keying + const cache_key = transfer.req.url; + + log.err(.browser, "http cache", .{ .key = cache_key, .metadata = metadata }); + + cache.put(cache_key, metadata, body) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); } } @@ -1143,8 +1156,7 @@ pub const Response = struct { pub fn headerIterator(self: Response) HeaderIterator { return switch (self.inner) { .transfer => |t| t.responseHeaderIterator(), - // TODO: Cache HTTP Headers - .cached => unreachable, + .cached => |c| HeaderIterator{ .list = .{ .list = c.metadata.headers } }, }; } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 56bc6c10..79374e66 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -17,6 +17,7 @@ // along with this program. If not, see . const std = @import("std"); +const Http = @import("../http.zig"); const FsCache = @import("FsCache.zig"); /// A browser-wide cache for resources across the network. @@ -39,6 +40,55 @@ pub fn put(self: *Cache, key: []const u8, metadata: CachedMetadata, body: []cons }; } +pub const CacheControl = struct { + max_age: ?u64 = null, + must_revalidate: bool = false, + no_cache: bool = false, + no_store: bool = false, + immutable: bool = false, + + pub fn parse(value: []const u8) CacheControl { + var cc: CacheControl = .{}; + + var iter = std.mem.splitScalar(u8, value, ','); + while (iter.next()) |part| { + const directive = std.mem.trim(u8, part, &std.ascii.whitespace); + if (std.ascii.eqlIgnoreCase(directive, "no-store")) { + cc.no_store = true; + } else if (std.ascii.eqlIgnoreCase(directive, "no-cache")) { + cc.no_cache = true; + } else if (std.ascii.eqlIgnoreCase(directive, "must-revalidate")) { + cc.must_revalidate = true; + } else if (std.ascii.eqlIgnoreCase(directive, "immutable")) { + cc.immutable = true; + } else if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) { + cc.max_age = std.fmt.parseInt(u64, directive[8..], 10) catch null; + } else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) { + // s-maxage takes precedence over max-age + cc.max_age = std.fmt.parseInt(u64, directive[9..], 10) catch cc.max_age; + } + } + return cc; + } +}; + +pub const Vary = union(enum) { + wildcard: void, + value: []const u8, + + pub fn parse(value: []const u8) Vary { + if (std.mem.eql(u8, value, "*")) return .wildcard; + return .{ .value = value }; + } + + pub fn toString(self: Vary) []const u8 { + return switch (self) { + .wildcard => "*", + .value => |v| v, + }; + } +}; + pub const CachedMetadata = struct { url: [:0]const u8, content_type: []const u8, @@ -52,17 +102,74 @@ pub const CachedMetadata = struct { etag: ?[]const u8, // for If-Modified-Since last_modified: ?[]const u8, + // If non-null, must be incorporated into cache key. + vary: ?[]const u8, must_revalidate: bool, no_cache: bool, immutable: bool, - // If non-null, must be incorporated into cache key. - vary: ?[]const u8, + headers: []const Http.Header, + + pub fn fromHeaders( + url: [:0]const u8, + status: u16, + timestamp: i64, + headers: []const Http.Header, + ) !?CachedMetadata { + var cc: CacheControl = .{}; + var vary: ?Vary = null; + var etag: ?[]const u8 = null; + var last_modified: ?[]const u8 = null; + var age_at_store: u64 = 0; + var content_type: []const u8 = "application/octet-stream"; + + for (headers) |hdr| { + if (std.ascii.eqlIgnoreCase(hdr.name, "cache-control")) { + cc = CacheControl.parse(hdr.value); + } else if (std.ascii.eqlIgnoreCase(hdr.name, "etag")) { + etag = hdr.value; + } else if (std.ascii.eqlIgnoreCase(hdr.name, "last-modified")) { + last_modified = hdr.value; + } else if (std.ascii.eqlIgnoreCase(hdr.name, "vary")) { + vary = Vary.parse(hdr.value); + } else if (std.ascii.eqlIgnoreCase(hdr.name, "age")) { + age_at_store = std.fmt.parseInt(u64, hdr.value, 10) catch 0; + } else if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { + content_type = hdr.value; + } + } + + // return null for uncacheable responses + if (cc.no_store) return null; + if (vary) |v| if (v == .wildcard) return null; + const resolved_max_age = cc.max_age orelse return null; + + return .{ + .url = url, + .content_type = content_type, + .status = status, + .stored_at = timestamp, + .age_at_store = age_at_store, + .max_age = resolved_max_age, + .etag = etag, + .last_modified = last_modified, + .must_revalidate = cc.must_revalidate, + .no_cache = cc.no_cache, + .immutable = cc.immutable, + .vary = if (vary) |v| v.toString() else null, + .headers = headers, + }; + } pub fn deinit(self: CachedMetadata, allocator: std.mem.Allocator) void { allocator.free(self.url); allocator.free(self.content_type); + for (self.headers) |header| { + allocator.free(header.name); + allocator.free(header.value); + } + allocator.free(self.headers); if (self.etag) |e| allocator.free(e); if (self.last_modified) |lm| allocator.free(lm); if (self.vary) |v| allocator.free(v); diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index db916cfa..c931e4a2 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -18,6 +18,7 @@ const std = @import("std"); const Cache = @import("Cache.zig"); +const Http = @import("../http.zig"); const CachedMetadata = Cache.CachedMetadata; const CachedResponse = Cache.CachedResponse; @@ -73,6 +74,12 @@ fn serializeMeta(writer: *std.Io.Writer, meta: *const CachedMetadata) !void { meta.immutable, }); try writer.flush(); + + try writer.print("{d}\n", .{meta.headers.len}); + for (meta.headers) |hdr| { + try writer.print("{s}\n{s}\n", .{ hdr.name, hdr.value }); + } + try writer.flush(); } fn deserializeMetaOptionalString(bytes: []const u8) ?[]const u8 { @@ -145,6 +152,32 @@ fn deserializeMeta(allocator: std.mem.Allocator, file: std.fs.File) !CachedMetad break :blk try deserializeMetaBoolean(line); }; + const headers = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + const count = std.fmt.parseInt(usize, line, 10) catch return error.Malformed; + + const hdrs = try allocator.alloc(Http.Header, count); + errdefer allocator.free(hdrs); + + for (hdrs) |*hdr| { + const name = try reader.takeDelimiter('\n') orelse return error.Malformed; + const value = try reader.takeDelimiter('\n') orelse return error.Malformed; + hdr.* = .{ + .name = try allocator.dupe(u8, name), + .value = try allocator.dupe(u8, value), + }; + } + + break :blk hdrs; + }; + errdefer { + for (headers) |hdr| { + allocator.free(hdr.name); + allocator.free(hdr.value); + } + allocator.free(headers); + } + return .{ .url = url, .content_type = content_type, @@ -158,6 +191,7 @@ fn deserializeMeta(allocator: std.mem.Allocator, file: std.fs.File) !CachedMetad .no_cache = no_cache, .immutable = immutable, .vary = vary, + .headers = headers, }; } From 18d347e247e7167f64945a64e890ed1a5f84289e Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Fri, 20 Mar 2026 01:54:41 -0700 Subject: [PATCH 12/48] use CacheControl and Vary --- src/network/cache/Cache.zig | 33 ++++++-------- src/network/cache/FsCache.zig | 84 +++++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 79374e66..a6a711bb 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -81,6 +81,13 @@ pub const Vary = union(enum) { return .{ .value = value }; } + pub fn deinit(self: Vary, allocator: std.mem.Allocator) void { + switch (self) { + .wildcard => {}, + .value => |v| allocator.free(v), + } + } + pub fn toString(self: Vary) []const u8 { return switch (self) { .wildcard => "*", @@ -96,19 +103,14 @@ pub const CachedMetadata = struct { status: u16, stored_at: i64, age_at_store: u64, - max_age: u64, // for If-None-Match etag: ?[]const u8, // for If-Modified-Since last_modified: ?[]const u8, - // If non-null, must be incorporated into cache key. - vary: ?[]const u8, - - must_revalidate: bool, - no_cache: bool, - immutable: bool, + cache_control: CacheControl, + vary: ?Vary, headers: []const Http.Header, pub fn fromHeaders( @@ -143,7 +145,7 @@ pub const CachedMetadata = struct { // return null for uncacheable responses if (cc.no_store) return null; if (vary) |v| if (v == .wildcard) return null; - const resolved_max_age = cc.max_age orelse return null; + if (cc.max_age == null) return null; return .{ .url = url, @@ -151,13 +153,10 @@ pub const CachedMetadata = struct { .status = status, .stored_at = timestamp, .age_at_store = age_at_store, - .max_age = resolved_max_age, .etag = etag, .last_modified = last_modified, - .must_revalidate = cc.must_revalidate, - .no_cache = cc.no_cache, - .immutable = cc.immutable, - .vary = if (vary) |v| v.toString() else null, + .cache_control = cc, + .vary = vary, .headers = headers, }; } @@ -170,15 +169,9 @@ pub const CachedMetadata = struct { allocator.free(header.value); } allocator.free(self.headers); + if (self.vary) |v| v.deinit(allocator); if (self.etag) |e| allocator.free(e); if (self.last_modified) |lm| allocator.free(lm); - if (self.vary) |v| allocator.free(v); - } - - pub fn isAgeStale(self: *const CachedMetadata) bool { - const now = std.time.timestamp(); - const age = now - self.stored_at + @as(i64, @intCast(self.age_at_store)); - return age < @as(i64, @intCast(self.max_age)); } }; diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index c931e4a2..de62cd30 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -59,25 +59,35 @@ fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 { fn serializeMeta(writer: *std.Io.Writer, meta: *const CachedMetadata) !void { try writer.print("{s}\n{s}\n", .{ meta.url, meta.content_type }); - try writer.print("{d}\n{d}\n{d}\n{d}\n", .{ + try writer.print("{d}\n{d}\n{d}\n", .{ meta.status, meta.stored_at, meta.age_at_store, - meta.max_age, }); try writer.print("{s}\n", .{meta.etag orelse "null"}); try writer.print("{s}\n", .{meta.last_modified orelse "null"}); - try writer.print("{s}\n", .{meta.vary orelse "null"}); - try writer.print("{}\n{}\n{}\n", .{ - meta.must_revalidate, - meta.no_cache, - meta.immutable, + + // cache-control + try writer.print("{d}\n", .{meta.cache_control.max_age orelse 0}); + try writer.print("{}\n{}\n{}\n{}\n", .{ + meta.cache_control.max_age != null, + meta.cache_control.must_revalidate, + meta.cache_control.no_cache, + meta.cache_control.immutable, }); + + // vary + if (meta.vary) |v| { + try writer.print("{s}\n", .{v.toString()}); + } else { + try writer.print("null\n", .{}); + } try writer.flush(); try writer.print("{d}\n", .{meta.headers.len}); for (meta.headers) |hdr| { try writer.print("{s}\n{s}\n", .{ hdr.name, hdr.value }); + try writer.flush(); } try writer.flush(); } @@ -121,36 +131,57 @@ fn deserializeMeta(allocator: std.mem.Allocator, file: std.fs.File) !CachedMetad const line = try reader.takeDelimiter('\n') orelse return error.Malformed; break :blk std.fmt.parseInt(u64, line, 10) catch return error.Malformed; }; - const max_age = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk std.fmt.parseInt(u64, line, 10) catch return error.Malformed; - }; const etag = blk: { const line = try reader.takeDelimiter('\n') orelse return error.Malformed; break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); }; + errdefer if (etag) |e| allocator.free(e); + const last_modified = blk: { const line = try reader.takeDelimiter('\n') orelse return error.Malformed; break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); }; - const vary = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); + errdefer if (last_modified) |lm| allocator.free(lm); + + // cache-control + const cc = cache_control: { + const max_age_val = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk std.fmt.parseInt(u64, line, 10) catch return error.Malformed; + }; + const max_age_present = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try deserializeMetaBoolean(line); + }; + const must_revalidate = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try deserializeMetaBoolean(line); + }; + const no_cache = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try deserializeMetaBoolean(line); + }; + const immutable = blk: { + const line = try reader.takeDelimiter('\n') orelse return error.Malformed; + break :blk try deserializeMetaBoolean(line); + }; + break :cache_control Cache.CacheControl{ + .max_age = if (max_age_present) max_age_val else null, + .must_revalidate = must_revalidate, + .no_cache = no_cache, + .immutable = immutable, + }; }; - const must_revalidate = blk: { + // vary + const vary = blk: { const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try deserializeMetaBoolean(line); - }; - const no_cache = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try deserializeMetaBoolean(line); - }; - const immutable = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try deserializeMetaBoolean(line); + if (std.mem.eql(u8, line, "null")) break :blk null; + const duped = try allocator.dupe(u8, line); + break :blk Cache.Vary.parse(duped); }; + errdefer if (vary) |v| if (v == .value) allocator.free(v.value); const headers = blk: { const line = try reader.takeDelimiter('\n') orelse return error.Malformed; @@ -184,12 +215,9 @@ fn deserializeMeta(allocator: std.mem.Allocator, file: std.fs.File) !CachedMetad .status = status, .stored_at = stored_at, .age_at_store = age_at_store, - .max_age = max_age, + .cache_control = cc, .etag = etag, .last_modified = last_modified, - .must_revalidate = must_revalidate, - .no_cache = no_cache, - .immutable = immutable, .vary = vary, .headers = headers, }; From 5c2207ecc39636be715fa33fe048f607f4654416 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Fri, 20 Mar 2026 02:02:21 -0700 Subject: [PATCH 13/48] add more http caching rules --- src/network/cache/Cache.zig | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index a6a711bb..cd176b91 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -42,6 +42,8 @@ pub fn put(self: *Cache, key: []const u8, metadata: CachedMetadata, body: []cons pub const CacheControl = struct { max_age: ?u64 = null, + s_maxage: ?u64 = null, + is_public: bool = false, must_revalidate: bool = false, no_cache: bool = false, no_store: bool = false, @@ -61,11 +63,14 @@ pub const CacheControl = struct { cc.must_revalidate = true; } else if (std.ascii.eqlIgnoreCase(directive, "immutable")) { cc.immutable = true; + } else if (std.ascii.eqlIgnoreCase(directive, "public")) { + cc.is_public = true; } else if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) { cc.max_age = std.fmt.parseInt(u64, directive[8..], 10) catch null; } else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) { + cc.s_maxage = std.fmt.parseInt(u64, directive[9..], 10) catch null; // s-maxage takes precedence over max-age - cc.max_age = std.fmt.parseInt(u64, directive[9..], 10) catch cc.max_age; + cc.max_age = cc.s_maxage orelse cc.max_age; } } return cc; @@ -125,6 +130,8 @@ pub const CachedMetadata = struct { var last_modified: ?[]const u8 = null; var age_at_store: u64 = 0; var content_type: []const u8 = "application/octet-stream"; + var has_set_cookie = false; + var has_authorization = false; for (headers) |hdr| { if (std.ascii.eqlIgnoreCase(hdr.name, "cache-control")) { @@ -139,14 +146,34 @@ pub const CachedMetadata = struct { age_at_store = std.fmt.parseInt(u64, hdr.value, 10) catch 0; } else if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { content_type = hdr.value; + } else if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) { + has_set_cookie = true; + } else if (std.ascii.eqlIgnoreCase(hdr.name, "authorization")) { + has_authorization = true; } } - // return null for uncacheable responses + // no-store: must not be stored if (cc.no_store) return null; + + // Vary: * means the response cannot be cached if (vary) |v| if (v == .wildcard) return null; + + // must have an explicit max-age to be cacheable if (cc.max_age == null) return null; + // Set-Cookie without explicit public + if (has_set_cookie and !cc.is_public) return null; + + // Authorization header without explicit public or s-maxage + if (has_authorization and !cc.is_public and cc.s_maxage == null) return null; + + // Only cache 200 for now. Technically, we can cache others. + switch (status) { + 200 => {}, + else => return null, + } + return .{ .url = url, .content_type = content_type, From 66d190c047073fea4429e7ac1e3f64ea7ddbdd13 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 23 Mar 2026 08:19:29 -0700 Subject: [PATCH 14/48] store type_buf and sub_type_buf in Mime --- src/browser/Mime.zig | 50 +++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/src/browser/Mime.zig b/src/browser/Mime.zig index 27fed71e..5ebe2622 100644 --- a/src/browser/Mime.zig +++ b/src/browser/Mime.zig @@ -27,6 +27,9 @@ charset: [41]u8 = default_charset, charset_len: usize = default_charset_len, is_default_charset: bool = true, +type_buf: [127]u8 = @splat(0), +sub_type_buf: [127]u8 = @splat(0), + /// String "UTF-8" continued by null characters. const default_charset = .{ 'U', 'T', 'F', '-', '8' } ++ .{0} ** 36; const default_charset_len = 5; @@ -61,7 +64,10 @@ pub const ContentType = union(ContentTypeEnum) { image_webp: void, application_json: void, unknown: void, - other: struct { type: []const u8, sub_type: []const u8 }, + other: struct { + type: []const u8, + sub_type: []const u8, + }, }; pub fn contentTypeString(mime: *const Mime) []const u8 { @@ -121,7 +127,9 @@ pub fn parse(input: []const u8) !Mime { const normalized = std.ascii.lowerString(&buf, std.mem.trim(u8, input, &std.ascii.whitespace)); _ = std.ascii.lowerString(normalized, normalized); - const content_type, const type_len = try parseContentType(normalized); + var mime = Mime{ .content_type = undefined }; + + const content_type, const type_len = try parseContentType(normalized, &mime.type_buf, &mime.sub_type_buf); if (type_len >= normalized.len) { return .{ .content_type = content_type }; } @@ -162,13 +170,12 @@ pub fn parse(input: []const u8) !Mime { } } - return .{ - .params = params, - .charset = charset, - .charset_len = charset_len, - .content_type = content_type, - .is_default_charset = !has_explicit_charset, - }; + mime.params = params; + mime.charset = charset; + mime.charset_len = charset_len; + mime.content_type = content_type; + mime.is_default_charset = !has_explicit_charset; + return mime; } /// Prescan the first 1024 bytes of an HTML document for a charset declaration. @@ -394,7 +401,7 @@ pub fn isText(mime: *const Mime) bool { } // we expect value to be lowercase -fn parseContentType(value: []const u8) !struct { ContentType, usize } { +fn parseContentType(value: []const u8, type_buf: []u8, sub_type_buf: []u8) !struct { ContentType, usize } { const end = std.mem.indexOfScalarPos(u8, value, 0, ';') orelse value.len; const type_name = trimRight(value[0..end]); const attribute_start = end + 1; @@ -443,10 +450,18 @@ fn parseContentType(value: []const u8) !struct { ContentType, usize } { return error.Invalid; } - return .{ .{ .other = .{ - .type = main_type, - .sub_type = sub_type, - } }, attribute_start }; + @memcpy(type_buf[0..main_type.len], main_type); + @memcpy(sub_type_buf[0..sub_type.len], sub_type); + + return .{ + .{ + .other = .{ + .type = type_buf[0..main_type.len], + .sub_type = sub_type_buf[0..sub_type.len], + }, + }, + attribute_start, + }; } const VALID_CODEPOINTS = blk: { @@ -460,6 +475,13 @@ const VALID_CODEPOINTS = blk: { break :blk v; }; +pub fn typeString(self: *const Mime) []const u8 { + return switch (self.content_type) { + .other => |o| o.type[0..o.type_len], + else => "", + }; +} + fn validType(value: []const u8) bool { for (value) |b| { if (VALID_CODEPOINTS[b] == false) { From 3c8bb5bc0028f0d23c748255de61c13b127d3257 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 15:04:27 -0700 Subject: [PATCH 15/48] use sha256 instead of wyhash --- src/network/cache/FsCache.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index de62cd30..df8a558f 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -46,14 +46,15 @@ pub fn cache(self: *FsCache) Cache { return Cache.init(self); } -const HASHED_KEY_LEN = 16; +const HASHED_KEY_LEN = 64; const HASHED_PATH_LEN = HASHED_KEY_LEN + 5; const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4; fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 { - const h = std.hash.Wyhash.hash(0, key); + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(key, &digest, .{}); var hex: [HASHED_KEY_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&hex, "{x:0>16}", .{h}) catch unreachable; + _ = std.fmt.bufPrint(&hex, "{s}", .{std.fmt.bytesToHex(&digest, .lower)}) catch unreachable; return hex; } From 186fdee59b48a17634e25cdaba7e16498fbf6ea8 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 15:36:53 -0700 Subject: [PATCH 16/48] use json for fs cache metadata file --- src/network/cache/FsCache.zig | 314 ++++++++++------------------------ 1 file changed, 93 insertions(+), 221 deletions(-) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index df8a558f..6f708201 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -22,10 +22,53 @@ const Http = @import("../http.zig"); const CachedMetadata = Cache.CachedMetadata; const CachedResponse = Cache.CachedResponse; +const CACHE_VERSION: usize = 1; + pub const FsCache = @This(); dir: std.fs.Dir, +const CacheMetadataFile = struct { + version: usize, + metadata: CachedMetadata, +}; + +const HASHED_KEY_LEN = 64; +const HASHED_PATH_LEN = HASHED_KEY_LEN + 5; +const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4; + +fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 { + var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(key, &digest, .{}); + var hex: [HASHED_KEY_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&hex, "{s}", .{std.fmt.bytesToHex(&digest, .lower)}) catch unreachable; + return hex; +} + +fn metaPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 { + var path: [HASHED_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&path, "{s}.meta", .{hashed_key}) catch unreachable; + return path; +} + +fn bodyPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 { + var path: [HASHED_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&path, "{s}.body", .{hashed_key}) catch unreachable; + return path; +} + +fn metaTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 { + var path: [HASHED_TMP_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&path, "{s}.meta.tmp", .{hashed_key}) catch unreachable; + return path; +} + +fn bodyTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 { + var path: [HASHED_TMP_PATH_LEN]u8 = undefined; + _ = std.fmt.bufPrint(&path, "{s}.body.tmp", .{hashed_key}) catch unreachable; + return path; +} + pub fn init(path: []const u8) !FsCache { const cwd = std.fs.cwd(); @@ -46,257 +89,86 @@ pub fn cache(self: *FsCache) Cache { return Cache.init(self); } -const HASHED_KEY_LEN = 64; -const HASHED_PATH_LEN = HASHED_KEY_LEN + 5; -const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4; - -fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 { - var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; - std.crypto.hash.sha2.Sha256.hash(key, &digest, .{}); - var hex: [HASHED_KEY_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&hex, "{s}", .{std.fmt.bytesToHex(&digest, .lower)}) catch unreachable; - return hex; -} - -fn serializeMeta(writer: *std.Io.Writer, meta: *const CachedMetadata) !void { - try writer.print("{s}\n{s}\n", .{ meta.url, meta.content_type }); - try writer.print("{d}\n{d}\n{d}\n", .{ - meta.status, - meta.stored_at, - meta.age_at_store, - }); - try writer.print("{s}\n", .{meta.etag orelse "null"}); - try writer.print("{s}\n", .{meta.last_modified orelse "null"}); - - // cache-control - try writer.print("{d}\n", .{meta.cache_control.max_age orelse 0}); - try writer.print("{}\n{}\n{}\n{}\n", .{ - meta.cache_control.max_age != null, - meta.cache_control.must_revalidate, - meta.cache_control.no_cache, - meta.cache_control.immutable, - }); - - // vary - if (meta.vary) |v| { - try writer.print("{s}\n", .{v.toString()}); - } else { - try writer.print("null\n", .{}); - } - try writer.flush(); - - try writer.print("{d}\n", .{meta.headers.len}); - for (meta.headers) |hdr| { - try writer.print("{s}\n{s}\n", .{ hdr.name, hdr.value }); - try writer.flush(); - } - try writer.flush(); -} - -fn deserializeMetaOptionalString(bytes: []const u8) ?[]const u8 { - if (std.mem.eql(u8, bytes, "null")) return null else return bytes; -} - -fn deserializeMetaBoolean(bytes: []const u8) !bool { - if (std.mem.eql(u8, bytes, "true")) return true; - if (std.mem.eql(u8, bytes, "false")) return false; - return error.Malformed; -} - -fn deserializeMeta(allocator: std.mem.Allocator, file: std.fs.File) !CachedMetadata { - var file_buf: [1024]u8 = undefined; - var file_reader = file.reader(&file_buf); - const reader = &file_reader.interface; - - const url = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try allocator.dupeZ(u8, line); - }; - errdefer allocator.free(url); - - const content_type = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try allocator.dupe(u8, line); - }; - errdefer allocator.free(content_type); - - const status = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk std.fmt.parseInt(u16, line, 10) catch return error.Malformed; - }; - const stored_at = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk std.fmt.parseInt(i64, line, 10) catch return error.Malformed; - }; - const age_at_store = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk std.fmt.parseInt(u64, line, 10) catch return error.Malformed; - }; - - const etag = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); - }; - errdefer if (etag) |e| allocator.free(e); - - const last_modified = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk if (std.mem.eql(u8, line, "null")) null else try allocator.dupe(u8, line); - }; - errdefer if (last_modified) |lm| allocator.free(lm); - - // cache-control - const cc = cache_control: { - const max_age_val = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk std.fmt.parseInt(u64, line, 10) catch return error.Malformed; - }; - const max_age_present = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try deserializeMetaBoolean(line); - }; - const must_revalidate = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try deserializeMetaBoolean(line); - }; - const no_cache = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try deserializeMetaBoolean(line); - }; - const immutable = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - break :blk try deserializeMetaBoolean(line); - }; - break :cache_control Cache.CacheControl{ - .max_age = if (max_age_present) max_age_val else null, - .must_revalidate = must_revalidate, - .no_cache = no_cache, - .immutable = immutable, - }; - }; - - // vary - const vary = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - if (std.mem.eql(u8, line, "null")) break :blk null; - const duped = try allocator.dupe(u8, line); - break :blk Cache.Vary.parse(duped); - }; - errdefer if (vary) |v| if (v == .value) allocator.free(v.value); - - const headers = blk: { - const line = try reader.takeDelimiter('\n') orelse return error.Malformed; - const count = std.fmt.parseInt(usize, line, 10) catch return error.Malformed; - - const hdrs = try allocator.alloc(Http.Header, count); - errdefer allocator.free(hdrs); - - for (hdrs) |*hdr| { - const name = try reader.takeDelimiter('\n') orelse return error.Malformed; - const value = try reader.takeDelimiter('\n') orelse return error.Malformed; - hdr.* = .{ - .name = try allocator.dupe(u8, name), - .value = try allocator.dupe(u8, value), - }; - } - - break :blk hdrs; - }; - errdefer { - for (headers) |hdr| { - allocator.free(hdr.name); - allocator.free(hdr.value); - } - allocator.free(headers); - } - - return .{ - .url = url, - .content_type = content_type, - .status = status, - .stored_at = stored_at, - .age_at_store = age_at_store, - .cache_control = cc, - .etag = etag, - .last_modified = last_modified, - .vary = vary, - .headers = headers, - }; -} - -pub fn get(self: *FsCache, allocator: std.mem.Allocator, key: []const u8) ?Cache.CachedResponse { +pub fn get(self: *FsCache, arena: std.mem.Allocator, key: []const u8) ?Cache.CachedResponse { const hashed_key = hashKey(key); + const meta_p = metaPath(&hashed_key); + const body_p = bodyPath(&hashed_key); - var meta_path: [HASHED_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&meta_path, "{s}.meta", .{hashed_key}) catch @panic("FsCache.get meta path overflowed"); - - var body_path: [HASHED_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&body_path, "{s}.body", .{hashed_key}) catch @panic("FsCache.get body path overflowed"); - - const meta_file = self.dir.openFile(&meta_path, .{ .mode = .read_only }) catch return null; + const meta_file = self.dir.openFile(&meta_p, .{ .mode = .read_only }) catch return null; defer meta_file.close(); - const meta = deserializeMeta(allocator, meta_file) catch { - self.dir.deleteFile(&meta_path) catch {}; - self.dir.deleteFile(&body_path) catch {}; + const contents = meta_file.readToEndAlloc(arena, 1 * 1024 * 1024) catch return null; + defer arena.free(contents); + + const cache_file: CacheMetadataFile = std.json.parseFromSliceLeaky( + CacheMetadataFile, + arena, + contents, + .{ .allocate = .alloc_always }, + ) catch { + self.dir.deleteFile(&meta_p) catch {}; + self.dir.deleteFile(&body_p) catch {}; return null; }; - const body_file = self.dir.openFile(&body_path, .{ .mode = .read_only }) catch return null; + const metadata = cache_file.metadata; + + if (cache_file.version != CACHE_VERSION) { + self.dir.deleteFile(&meta_p) catch {}; + self.dir.deleteFile(&body_p) catch {}; + return null; + } + + const body_file = self.dir.openFile( + &body_p, + .{ .mode = .read_only }, + ) catch return null; return .{ - .metadata = meta, + .metadata = metadata, .data = .{ .file = body_file }, }; } pub fn put(self: *FsCache, key: []const u8, meta: CachedMetadata, body: []const u8) !void { const hashed_key = hashKey(key); - - // Write meta to a temp file, then atomically rename into place - var meta_path: [HASHED_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&meta_path, "{s}.meta", .{hashed_key}) catch - @panic("FsCache.put meta path overflowed"); - - var meta_tmp_path: [HASHED_TMP_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&meta_tmp_path, "{s}.meta.tmp", .{hashed_key}) catch - @panic("FsCache.put meta tmp path overflowed"); + const meta_p = metaPath(&hashed_key); + const meta_tmp_p = metaTmpPath(&hashed_key); + const body_p = bodyPath(&hashed_key); + const body_tmp_p = bodyTmpPath(&hashed_key); { - const meta_file = try self.dir.createFile(&meta_tmp_path, .{}); + const meta_file = try self.dir.createFile(&meta_tmp_p, .{}); errdefer { meta_file.close(); - self.dir.deleteFile(&meta_tmp_path) catch {}; + self.dir.deleteFile(&meta_tmp_p) catch {}; } - var buf: [512]u8 = undefined; - var meta_file_writer = meta_file.writer(&buf); - try serializeMeta(&meta_file_writer.interface, &meta); + var meta_file_writer_buf: [512]u8 = undefined; + var meta_file_writer = meta_file.writer(&meta_file_writer_buf); + const meta_file_writer_iface = &meta_file_writer.interface; + try std.json.Stringify.value( + CacheMetadataFile{ .version = CACHE_VERSION, .metadata = meta }, + .{ .whitespace = .minified }, + meta_file_writer_iface, + ); + try meta_file_writer_iface.flush(); meta_file.close(); } - errdefer self.dir.deleteFile(&meta_tmp_path) catch {}; - try self.dir.rename(&meta_tmp_path, &meta_path); - - // Write body to a temp file, then atomically rename into place - var body_path: [HASHED_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&body_path, "{s}.body", .{hashed_key}) catch - @panic("FsCache.put body path overflowed"); - - var body_tmp_path: [HASHED_TMP_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&body_tmp_path, "{s}.body.tmp", .{hashed_key}) catch - @panic("FsCache.put body tmp path overflowed"); + errdefer self.dir.deleteFile(&meta_tmp_p) catch {}; + try self.dir.rename(&meta_tmp_p, &meta_p); { - const body_file = try self.dir.createFile(&body_tmp_path, .{}); + const body_file = try self.dir.createFile(&body_tmp_p, .{}); errdefer { body_file.close(); - self.dir.deleteFile(&body_tmp_path) catch {}; + self.dir.deleteFile(&body_tmp_p) catch {}; } try body_file.writeAll(body); body_file.close(); } - errdefer self.dir.deleteFile(&body_tmp_path) catch {}; + errdefer self.dir.deleteFile(&body_tmp_p) catch {}; - errdefer self.dir.deleteFile(&meta_path) catch {}; - try self.dir.rename(&body_tmp_path, &body_path); + errdefer self.dir.deleteFile(&meta_p) catch {}; + try self.dir.rename(&body_tmp_p, &body_p); } From 3eb05fdd1afc0ec4313bad353d9a4a4bcb13007a Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 15:39:55 -0700 Subject: [PATCH 17/48] use writer for fs cache body file --- src/network/cache/FsCache.zig | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 6f708201..5e0e9e46 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -137,6 +137,8 @@ pub fn put(self: *FsCache, key: []const u8, meta: CachedMetadata, body: []const const body_p = bodyPath(&hashed_key); const body_tmp_p = bodyTmpPath(&hashed_key); + var writer_buf: [512]u8 = undefined; + { const meta_file = try self.dir.createFile(&meta_tmp_p, .{}); errdefer { @@ -144,8 +146,7 @@ pub fn put(self: *FsCache, key: []const u8, meta: CachedMetadata, body: []const self.dir.deleteFile(&meta_tmp_p) catch {}; } - var meta_file_writer_buf: [512]u8 = undefined; - var meta_file_writer = meta_file.writer(&meta_file_writer_buf); + var meta_file_writer = meta_file.writer(&writer_buf); const meta_file_writer_iface = &meta_file_writer.interface; try std.json.Stringify.value( CacheMetadataFile{ .version = CACHE_VERSION, .metadata = meta }, @@ -164,7 +165,11 @@ pub fn put(self: *FsCache, key: []const u8, meta: CachedMetadata, body: []const body_file.close(); self.dir.deleteFile(&body_tmp_p) catch {}; } - try body_file.writeAll(body); + + var body_file_writer = body_file.writer(&writer_buf); + const body_file_writer_iface = &body_file_writer.interface; + try body_file_writer_iface.writeAll(body); + try body_file_writer_iface.flush(); body_file.close(); } errdefer self.dir.deleteFile(&body_tmp_p) catch {}; From 2de35a9db2f8c6e0dae821595c658bc66cf85aca Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 15:54:49 -0700 Subject: [PATCH 18/48] use arena_pool for cache get --- src/App.zig | 2 +- src/browser/HttpClient.zig | 10 ++++++---- src/network/Network.zig | 5 ++++- src/network/cache/Cache.zig | 17 ++--------------- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/App.zig b/src/App.zig index 8c4471ad..41ca9822 100644 --- a/src/App.zig +++ b/src/App.zig @@ -55,7 +55,7 @@ pub fn init(allocator: Allocator, config: *const Config) !*App { .arena_pool = undefined, }; - app.network = try Network.init(allocator, config); + app.network = try Network.init(allocator, app, config); errdefer app.network.deinit(); app.platform = try Platform.init(); diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 012d8270..ae1984b3 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -314,9 +314,8 @@ pub fn request(self: *Client, req: Request) !void { return self.fetchRobotsThenProcessRequest(robots_url, req); } -fn serveFromCache(allocator: std.mem.Allocator, req: Request, cached: *const CachedResponse) !void { +fn serveFromCache(req: Request, cached: *const CachedResponse) !void { const response = Response.fromCached(req.ctx, cached); - defer cached.metadata.deinit(allocator); if (req.start_callback) |cb| { try cb(response); @@ -355,7 +354,10 @@ fn serveFromCache(allocator: std.mem.Allocator, req: Request, cached: *const Cac fn processRequest(self: *Client, req: Request) !void { if (self.network.cache) |*cache| { if (req.method == .GET) { - if (cache.get(self.allocator, req.url)) |cached| { + const arena = try self.network.app.arena_pool.acquire(); + defer self.network.app.arena_pool.release(arena); + + if (cache.get(arena, req.url)) |cached| { log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = true, @@ -363,7 +365,7 @@ fn processRequest(self: *Client, req: Request) !void { }); defer req.headers.deinit(); - return serveFromCache(self.allocator, req, &cached); + return serveFromCache(req, &cached); } else { log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = false }); } diff --git a/src/network/Network.zig b/src/network/Network.zig index 1c347c52..8e218a9f 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -31,6 +31,7 @@ const RobotStore = @import("Robots.zig").RobotStore; const WebBotAuth = @import("WebBotAuth.zig"); const Cache = @import("cache/Cache.zig"); +const App = @import("../App.zig"); const Network = @This(); const Listener = struct { @@ -46,6 +47,7 @@ const MAX_TICK_CALLBACKS = 16; allocator: Allocator, +app: *App, config: *const Config, ca_blob: ?http.Blob, robot_store: RobotStore, @@ -202,7 +204,7 @@ fn globalDeinit() void { libcurl.curl_global_cleanup(); } -pub fn init(allocator: Allocator, config: *const Config) !Network { +pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { globalInit(allocator); errdefer globalDeinit(); @@ -251,6 +253,7 @@ pub fn init(allocator: Allocator, config: *const Config) !Network { .available = available, .connections = connections, + .app = app, .robot_store = RobotStore.init(allocator), .web_bot_auth = web_bot_auth, .cache = cache, diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index cd176b91..17cf199c 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -28,9 +28,9 @@ kind: union(enum) { fs: FsCache, }, -pub fn get(self: *Cache, allocator: std.mem.Allocator, key: []const u8) ?CachedResponse { +pub fn get(self: *Cache, arena: std.mem.Allocator, key: []const u8) ?CachedResponse { return switch (self.kind) { - inline else => |*c| c.get(allocator, key), + inline else => |*c| c.get(arena, key), }; } @@ -187,19 +187,6 @@ pub const CachedMetadata = struct { .headers = headers, }; } - - pub fn deinit(self: CachedMetadata, allocator: std.mem.Allocator) void { - allocator.free(self.url); - allocator.free(self.content_type); - for (self.headers) |header| { - allocator.free(header.name); - allocator.free(header.value); - } - allocator.free(self.headers); - if (self.vary) |v| v.deinit(allocator); - if (self.etag) |e| allocator.free(e); - if (self.last_modified) |lm| allocator.free(lm); - } }; pub const CachedData = union(enum) { From ce620e208d9a74275852cb366f213aae4a1ed645 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 16:14:54 -0700 Subject: [PATCH 19/48] add striped lock to FsCache --- src/network/cache/FsCache.zig | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 5e0e9e46..87df4f83 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -23,16 +23,23 @@ const CachedMetadata = Cache.CachedMetadata; const CachedResponse = Cache.CachedResponse; const CACHE_VERSION: usize = 1; +const LOCK_STRIPES = 16; pub const FsCache = @This(); dir: std.fs.Dir, +locks: [LOCK_STRIPES]std.Thread.Mutex = .{std.Thread.Mutex{}} ** LOCK_STRIPES, const CacheMetadataFile = struct { version: usize, metadata: CachedMetadata, }; +fn getLockPtr(self: *FsCache, key: *const [HASHED_KEY_LEN]u8) *std.Thread.Mutex { + const lock_idx: usize = @truncate(std.hash.Wyhash.hash(0, key) % LOCK_STRIPES); + return &self.locks[lock_idx]; +} + const HASHED_KEY_LEN = 64; const HASHED_PATH_LEN = HASHED_KEY_LEN + 5; const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4; @@ -94,6 +101,10 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, key: []const u8) ?Cache.Cac const meta_p = metaPath(&hashed_key); const body_p = bodyPath(&hashed_key); + const lock = self.getLockPtr(&hashed_key); + lock.lock(); + defer lock.unlock(); + const meta_file = self.dir.openFile(&meta_p, .{ .mode = .read_only }) catch return null; defer meta_file.close(); @@ -136,9 +147,12 @@ pub fn put(self: *FsCache, key: []const u8, meta: CachedMetadata, body: []const const meta_tmp_p = metaTmpPath(&hashed_key); const body_p = bodyPath(&hashed_key); const body_tmp_p = bodyTmpPath(&hashed_key); - var writer_buf: [512]u8 = undefined; + const lock = self.getLockPtr(&hashed_key); + lock.lock(); + defer lock.unlock(); + { const meta_file = try self.dir.createFile(&meta_tmp_p, .{}); errdefer { From 557a4458a4e30a16a25bbcc0023816200d212cdf Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 16:20:38 -0700 Subject: [PATCH 20/48] use CacheRequest instead of key --- src/browser/HttpClient.zig | 10 +++++++--- src/network/cache/Cache.zig | 12 ++++++++---- src/network/cache/FsCache.zig | 9 +++++---- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index ae1984b3..16078718 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -354,10 +354,10 @@ fn serveFromCache(req: Request, cached: *const CachedResponse) !void { fn processRequest(self: *Client, req: Request) !void { if (self.network.cache) |*cache| { if (req.method == .GET) { - const arena = try self.network.app.arena_pool.acquire(); + const arena = try self.network.app.arena_pool.acquire(.{ .debug = "HttpClient.processRequest.cache" }); defer self.network.app.arena_pool.release(arena); - if (cache.get(arena, req.url)) |cached| { + if (cache.get(arena, .{ .url = req.url })) |cached| { log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = true, @@ -992,7 +992,11 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T log.err(.browser, "http cache", .{ .key = cache_key, .metadata = metadata }); - cache.put(cache_key, metadata, body) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); + cache.put( + .{ .url = cache_key }, + metadata, + body, + ) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); } } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 17cf199c..7a1e8521 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -28,15 +28,15 @@ kind: union(enum) { fs: FsCache, }, -pub fn get(self: *Cache, arena: std.mem.Allocator, key: []const u8) ?CachedResponse { +pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse { return switch (self.kind) { - inline else => |*c| c.get(arena, key), + inline else => |*c| c.get(arena, req), }; } -pub fn put(self: *Cache, key: []const u8, metadata: CachedMetadata, body: []const u8) !void { +pub fn put(self: *Cache, req: CacheRequest, metadata: CachedMetadata, body: []const u8) !void { return switch (self.kind) { - inline else => |*c| c.put(key, metadata, body), + inline else => |*c| c.put(req, metadata, body), }; } @@ -189,6 +189,10 @@ pub const CachedMetadata = struct { } }; +pub const CacheRequest = struct { + url: []const u8, +}; + pub const CachedData = union(enum) { buffer: []const u8, file: std.fs.File, diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 87df4f83..a34c7a83 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -19,6 +19,7 @@ const std = @import("std"); const Cache = @import("Cache.zig"); const Http = @import("../http.zig"); +const CacheRequest = Cache.CacheRequest; const CachedMetadata = Cache.CachedMetadata; const CachedResponse = Cache.CachedResponse; @@ -96,8 +97,8 @@ pub fn cache(self: *FsCache) Cache { return Cache.init(self); } -pub fn get(self: *FsCache, arena: std.mem.Allocator, key: []const u8) ?Cache.CachedResponse { - const hashed_key = hashKey(key); +pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.CachedResponse { + const hashed_key = hashKey(req.url); const meta_p = metaPath(&hashed_key); const body_p = bodyPath(&hashed_key); @@ -141,8 +142,8 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, key: []const u8) ?Cache.Cac }; } -pub fn put(self: *FsCache, key: []const u8, meta: CachedMetadata, body: []const u8) !void { - const hashed_key = hashKey(key); +pub fn put(self: *FsCache, req: CacheRequest, meta: CachedMetadata, body: []const u8) !void { + const hashed_key = hashKey(req.url); const meta_p = metaPath(&hashed_key); const meta_tmp_p = metaTmpPath(&hashed_key); const body_p = bodyPath(&hashed_key); From cd3e6b2364e69a9f0c1686785387a53d52d89d08 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 16:25:12 -0700 Subject: [PATCH 21/48] ensure fs cache file is closed after use --- src/browser/HttpClient.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 16078718..7507b71b 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -323,6 +323,10 @@ fn serveFromCache(req: Request, cached: *const CachedResponse) !void { const proceed = try req.header_callback(response); if (!proceed) { + switch (cached.data) { + .buffer => |_| {}, + .file => |file| file.close(), + } req.error_callback(req.ctx, error.Abort); return; } @@ -334,6 +338,7 @@ fn serveFromCache(req: Request, cached: *const CachedResponse) !void { } }, .file => |file| { + defer file.close(); var buf: [1024]u8 = undefined; var file_reader = file.reader(&buf); From 65f77af84d79cfd44ba87d71be1b914f5a9872ff Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 17:36:32 -0700 Subject: [PATCH 22/48] shortcircuit a lot of caching checks --- src/browser/HttpClient.zig | 6 +-- src/network/cache/Cache.zig | 94 ++++++++++++++++------------------- src/network/cache/FsCache.zig | 4 +- 3 files changed, 46 insertions(+), 58 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 7507b71b..2259dc7b 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -997,11 +997,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T log.err(.browser, "http cache", .{ .key = cache_key, .metadata = metadata }); - cache.put( - .{ .url = cache_key }, - metadata, - body, - ) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); + cache.put(metadata, body) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); } } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 7a1e8521..ce375852 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -34,45 +34,57 @@ pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedRes }; } -pub fn put(self: *Cache, req: CacheRequest, metadata: CachedMetadata, body: []const u8) !void { +pub fn put(self: *Cache, metadata: CachedMetadata, body: []const u8) !void { return switch (self.kind) { - inline else => |*c| c.put(req, metadata, body), + inline else => |*c| c.put(metadata, body), }; } pub const CacheControl = struct { - max_age: ?u64 = null, - s_maxage: ?u64 = null, - is_public: bool = false, + max_age: u64, must_revalidate: bool = false, - no_cache: bool = false, - no_store: bool = false, immutable: bool = false, - pub fn parse(value: []const u8) CacheControl { - var cc: CacheControl = .{}; + pub fn parse(value: []const u8) ?CacheControl { + var cc: CacheControl = .{ .max_age = undefined }; + + var max_age_set = false; + var max_s_age_set = false; + var is_public = false; var iter = std.mem.splitScalar(u8, value, ','); while (iter.next()) |part| { const directive = std.mem.trim(u8, part, &std.ascii.whitespace); if (std.ascii.eqlIgnoreCase(directive, "no-store")) { - cc.no_store = true; + return null; } else if (std.ascii.eqlIgnoreCase(directive, "no-cache")) { - cc.no_cache = true; + return null; } else if (std.ascii.eqlIgnoreCase(directive, "must-revalidate")) { cc.must_revalidate = true; } else if (std.ascii.eqlIgnoreCase(directive, "immutable")) { cc.immutable = true; } else if (std.ascii.eqlIgnoreCase(directive, "public")) { - cc.is_public = true; + is_public = true; } else if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) { - cc.max_age = std.fmt.parseInt(u64, directive[8..], 10) catch null; + if (!max_s_age_set) { + if (std.fmt.parseInt(u64, directive[8..], 10) catch null) |max_age| { + cc.max_age = max_age; + max_age_set = true; + } + } } else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) { - cc.s_maxage = std.fmt.parseInt(u64, directive[9..], 10) catch null; - // s-maxage takes precedence over max-age - cc.max_age = cc.s_maxage orelse cc.max_age; + if (std.fmt.parseInt(u64, directive[9..], 10) catch null) |max_age| { + cc.max_age = max_age; + max_age_set = true; + max_s_age_set = true; + } } } + + if (!max_age_set) return null; + if (!is_public) return null; + if (cc.max_age == 0) return null; + return cc; } }; @@ -86,13 +98,6 @@ pub const Vary = union(enum) { return .{ .value = value }; } - pub fn deinit(self: Vary, allocator: std.mem.Allocator) void { - switch (self) { - .wildcard => {}, - .value => |v| allocator.free(v), - } - } - pub fn toString(self: Vary) []const u8 { return switch (self) { .wildcard => "*", @@ -124,56 +129,43 @@ pub const CachedMetadata = struct { timestamp: i64, headers: []const Http.Header, ) !?CachedMetadata { - var cc: CacheControl = .{}; + var cc: ?CacheControl = null; var vary: ?Vary = null; var etag: ?[]const u8 = null; var last_modified: ?[]const u8 = null; var age_at_store: u64 = 0; var content_type: []const u8 = "application/octet-stream"; - var has_set_cookie = false; - var has_authorization = false; + + // Only cache 200 for now. Technically, we can cache others. + switch (status) { + 200 => {}, + else => return null, + } for (headers) |hdr| { if (std.ascii.eqlIgnoreCase(hdr.name, "cache-control")) { - cc = CacheControl.parse(hdr.value); + cc = CacheControl.parse(hdr.value) orelse return null; } else if (std.ascii.eqlIgnoreCase(hdr.name, "etag")) { etag = hdr.value; } else if (std.ascii.eqlIgnoreCase(hdr.name, "last-modified")) { last_modified = hdr.value; } else if (std.ascii.eqlIgnoreCase(hdr.name, "vary")) { vary = Vary.parse(hdr.value); + // Vary: * means the response cannot be cached + if (vary) |v| if (v == .wildcard) return null; } else if (std.ascii.eqlIgnoreCase(hdr.name, "age")) { age_at_store = std.fmt.parseInt(u64, hdr.value, 10) catch 0; } else if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { content_type = hdr.value; } else if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) { - has_set_cookie = true; + // Don't cache if has Set-Cookie. + return null; } else if (std.ascii.eqlIgnoreCase(hdr.name, "authorization")) { - has_authorization = true; + // Don't cache if has Authorization. + return null; } } - // no-store: must not be stored - if (cc.no_store) return null; - - // Vary: * means the response cannot be cached - if (vary) |v| if (v == .wildcard) return null; - - // must have an explicit max-age to be cacheable - if (cc.max_age == null) return null; - - // Set-Cookie without explicit public - if (has_set_cookie and !cc.is_public) return null; - - // Authorization header without explicit public or s-maxage - if (has_authorization and !cc.is_public and cc.s_maxage == null) return null; - - // Only cache 200 for now. Technically, we can cache others. - switch (status) { - 200 => {}, - else => return null, - } - return .{ .url = url, .content_type = content_type, @@ -182,7 +174,7 @@ pub const CachedMetadata = struct { .age_at_store = age_at_store, .etag = etag, .last_modified = last_modified, - .cache_control = cc, + .cache_control = cc orelse return null, .vary = vary, .headers = headers, }; diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index a34c7a83..fc5b4b71 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -142,8 +142,8 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C }; } -pub fn put(self: *FsCache, req: CacheRequest, meta: CachedMetadata, body: []const u8) !void { - const hashed_key = hashKey(req.url); +pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void { + const hashed_key = hashKey(meta.url); const meta_p = metaPath(&hashed_key); const meta_tmp_p = metaTmpPath(&hashed_key); const body_p = bodyPath(&hashed_key); From 609983da876b26b4e6d5cd7da089a7975f414bc0 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 18:18:26 -0700 Subject: [PATCH 23/48] only store stuff when we know we will cache --- src/browser/HttpClient.zig | 61 ++++++++++++++++++------- src/network/cache/Cache.zig | 90 ++++++++++++++----------------------- 2 files changed, 77 insertions(+), 74 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 2259dc7b..3f222a69 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -981,25 +981,17 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T try transfer.req.done_callback(transfer.req.ctx); - cache: { - if (self.network.cache) |*cache| { - const headers = &transfer.response_header.?; + if (transfer._pending_cache_metadata) |metadata| { + const cache = &self.network.cache.?; - const metadata = try CacheMetadata.fromHeaders( - transfer.req.url, - headers.status, - std.time.timestamp(), - header_list.items, - ) orelse break :cache; + // TODO: Support Vary Keying + const cache_key = transfer.req.url; - // TODO: Support Vary Keying - const cache_key = transfer.req.url; - - log.err(.browser, "http cache", .{ .key = cache_key, .metadata = metadata }); - - cache.put(metadata, body) catch |err| log.warn(.http, "cache put failed", .{ .err = err }); - log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); - } + log.debug(.browser, "http cache", .{ .key = cache_key, .metadata = metadata }); + cache.put(metadata, body) catch |err| { + log.warn(.http, "cache put failed", .{ .err = err }); + }; + log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); } transfer.req.notification.dispatch(.http_request_done, &.{ @@ -1184,6 +1176,7 @@ pub const Transfer = struct { // total bytes received in the response, including the response status line, // the headers, and the [encoded] body. bytes_received: usize = 0, + _pending_cache_metadata: ?CacheMetadata = null, aborted: bool = false, @@ -1564,6 +1557,40 @@ pub const Transfer = struct { return err; }; + if (transfer.client.network.cache != null and transfer.req.method == .GET) { + const rh = &transfer.response_header.?; + const allocator = transfer.arena.allocator(); + + const maybe_cm = try Cache.tryCache( + allocator, + std.time.timestamp(), + transfer.url, + rh.status, + rh.contentType(), + if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null, + if (conn.getResponseHeader("vary", 0)) |h| h.value else null, + if (conn.getResponseHeader("etag", 0)) |h| h.value else null, + if (conn.getResponseHeader("last-modified", 0)) |h| h.value else null, + if (conn.getResponseHeader("age", 0)) |h| h.value else null, + conn.getResponseHeader("set-cookie", 0) != null, + conn.getResponseHeader("authorization", 0) != null, + ); + + if (maybe_cm) |cm| { + var header_list: std.ArrayList(http.Header) = .empty; + var it = transfer.responseHeaderIterator(); + while (it.next()) |hdr| { + try header_list.append(allocator, .{ + .name = try allocator.dupe(u8, hdr.name), + .value = try allocator.dupe(u8, hdr.value), + }); + } + + transfer._pending_cache_metadata = cm; + transfer._pending_cache_metadata.?.headers = header_list.items; + } + } + return proceed and transfer.aborted == false; } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index ce375852..98bea107 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -122,63 +122,6 @@ pub const CachedMetadata = struct { cache_control: CacheControl, vary: ?Vary, headers: []const Http.Header, - - pub fn fromHeaders( - url: [:0]const u8, - status: u16, - timestamp: i64, - headers: []const Http.Header, - ) !?CachedMetadata { - var cc: ?CacheControl = null; - var vary: ?Vary = null; - var etag: ?[]const u8 = null; - var last_modified: ?[]const u8 = null; - var age_at_store: u64 = 0; - var content_type: []const u8 = "application/octet-stream"; - - // Only cache 200 for now. Technically, we can cache others. - switch (status) { - 200 => {}, - else => return null, - } - - for (headers) |hdr| { - if (std.ascii.eqlIgnoreCase(hdr.name, "cache-control")) { - cc = CacheControl.parse(hdr.value) orelse return null; - } else if (std.ascii.eqlIgnoreCase(hdr.name, "etag")) { - etag = hdr.value; - } else if (std.ascii.eqlIgnoreCase(hdr.name, "last-modified")) { - last_modified = hdr.value; - } else if (std.ascii.eqlIgnoreCase(hdr.name, "vary")) { - vary = Vary.parse(hdr.value); - // Vary: * means the response cannot be cached - if (vary) |v| if (v == .wildcard) return null; - } else if (std.ascii.eqlIgnoreCase(hdr.name, "age")) { - age_at_store = std.fmt.parseInt(u64, hdr.value, 10) catch 0; - } else if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { - content_type = hdr.value; - } else if (std.ascii.eqlIgnoreCase(hdr.name, "set-cookie")) { - // Don't cache if has Set-Cookie. - return null; - } else if (std.ascii.eqlIgnoreCase(hdr.name, "authorization")) { - // Don't cache if has Authorization. - return null; - } - } - - return .{ - .url = url, - .content_type = content_type, - .status = status, - .stored_at = timestamp, - .age_at_store = age_at_store, - .etag = etag, - .last_modified = last_modified, - .cache_control = cc orelse return null, - .vary = vary, - .headers = headers, - }; - } }; pub const CacheRequest = struct { @@ -194,3 +137,36 @@ pub const CachedResponse = struct { metadata: CachedMetadata, data: CachedData, }; + +pub fn tryCache( + arena: std.mem.Allocator, + timestamp: i64, + url: [:0]const u8, + status: u16, + content_type: ?[]const u8, + cache_control: ?[]const u8, + vary: ?[]const u8, + etag: ?[]const u8, + last_modified: ?[]const u8, + age: ?[]const u8, + has_set_cookie: bool, + has_authorization: bool, +) !?CachedMetadata { + if (status != 200) return null; + if (has_set_cookie) return null; + if (has_authorization) return null; + const cc = CacheControl.parse(cache_control orelse return null) orelse return null; + + return .{ + .url = url, + .content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream", + .status = status, + .stored_at = timestamp, + .age_at_store = if (age) |a| std.fmt.parseInt(u64, a, 10) catch 0 else 0, + .cache_control = cc, + .vary = if (vary) |v| Vary.parse(v) else null, + .etag = if (etag) |e| try arena.dupe(u8, e) else null, + .last_modified = if (last_modified) |lm| try arena.dupe(u8, lm) else null, + .headers = &.{}, + }; +} From 9d62e58c9a5145f5ebef961316f3ca1c8f10fcfb Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 18:32:04 -0700 Subject: [PATCH 24/48] check age on fs cache get --- src/network/cache/FsCache.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index fc5b4b71..720e28f0 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -131,6 +131,14 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C return null; } + const now = std.time.timestamp(); + const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store)); + if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) { + self.dir.deleteFile(&meta_p) catch {}; + self.dir.deleteFile(&body_p) catch {}; + return null; + } + const body_file = self.dir.openFile( &body_p, .{ .mode = .read_only }, From cedc89444542356ba9e79c8887006282079a3e03 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 18:35:17 -0700 Subject: [PATCH 25/48] add basic fs cache get/put test --- src/network/cache/FsCache.zig | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 720e28f0..e7efcdd2 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -200,3 +200,48 @@ pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void { errdefer self.dir.deleteFile(&meta_p) catch {}; try self.dir.rename(&body_tmp_p, &body_p); } + +const testing = std.testing; + +test "FsCache: basic put and get" { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + + const path = try tmp.dir.realpathAlloc(testing.allocator, "."); + defer testing.allocator.free(path); + + var fs_cache = try FsCache.init(path); + defer fs_cache.deinit(); + var c = Cache{ .kind = .{ .fs = fs_cache } }; + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .etag = null, + .last_modified = null, + .cache_control = .{ .max_age = 600 }, + .vary = null, + .headers = &.{}, + }; + + const body = "hello world"; + try c.put(meta, body); + + const result = c.get(arena.allocator(), .{ .url = "https://example.com" }) orelse return error.CacheMiss; + defer result.data.file.close(); + + var buf: [64]u8 = undefined; + var file_reader = result.data.file.reader(&buf); + + const read_buf = try file_reader.interface.allocRemaining(testing.allocator, .unlimited); + defer testing.allocator.free(read_buf); + + try testing.expectEqualStrings(body, read_buf); +} From 77e9f5caf72891568864508b521fbf3c27ad6a17 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 18:41:40 -0700 Subject: [PATCH 26/48] remove unused cache method on fs cache --- src/network/cache/FsCache.zig | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index e7efcdd2..29afd9b0 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -93,10 +93,6 @@ pub fn deinit(self: *FsCache) void { self.dir.close(); } -pub fn cache(self: *FsCache) Cache { - return Cache.init(self); -} - pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.CachedResponse { const hashed_key = hashKey(req.url); const meta_p = metaPath(&hashed_key); From a60932bbe0662e2ea54e07c650330c00eadcf6b3 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 26 Mar 2026 18:50:46 -0700 Subject: [PATCH 27/48] require timestamp passed in with cache request --- src/browser/HttpClient.zig | 2 +- src/network/cache/Cache.zig | 1 + src/network/cache/FsCache.zig | 58 ++++++++++++++++++++++++++++++++--- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 3f222a69..399c1011 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -362,7 +362,7 @@ fn processRequest(self: *Client, req: Request) !void { const arena = try self.network.app.arena_pool.acquire(.{ .debug = "HttpClient.processRequest.cache" }); defer self.network.app.arena_pool.release(arena); - if (cache.get(arena, .{ .url = req.url })) |cached| { + if (cache.get(arena, .{ .url = req.url, .timestamp = std.time.timestamp() })) |cached| { log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = true, diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 98bea107..54603172 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -126,6 +126,7 @@ pub const CachedMetadata = struct { pub const CacheRequest = struct { url: []const u8, + timestamp: i64, }; pub const CachedData = union(enum) { diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 29afd9b0..4fd53e2f 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -127,7 +127,7 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C return null; } - const now = std.time.timestamp(); + const now = req.timestamp; const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store)); if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) { self.dir.deleteFile(&meta_p) catch {}; @@ -208,7 +208,7 @@ test "FsCache: basic put and get" { var fs_cache = try FsCache.init(path); defer fs_cache.deinit(); - var c = Cache{ .kind = .{ .fs = fs_cache } }; + var cache = Cache{ .kind = .{ .fs = fs_cache } }; var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -228,9 +228,9 @@ test "FsCache: basic put and get" { }; const body = "hello world"; - try c.put(meta, body); + try cache.put(meta, body); - const result = c.get(arena.allocator(), .{ .url = "https://example.com" }) orelse return error.CacheMiss; + const result = cache.get(arena.allocator(), .{ .url = "https://example.com", .timestamp = now }) orelse return error.CacheMiss; defer result.data.file.close(); var buf: [64]u8 = undefined; @@ -241,3 +241,53 @@ test "FsCache: basic put and get" { try testing.expectEqualStrings(body, read_buf); } + +test "FsCache: get expiration" { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + + const path = try tmp.dir.realpathAlloc(testing.allocator, "."); + defer testing.allocator.free(path); + + var fs_cache = try FsCache.init(path); + defer fs_cache.deinit(); + var cache = Cache{ .kind = .{ .fs = fs_cache } }; + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = 5000; + const max_age = 1000; + + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 900, + .etag = null, + .last_modified = null, + .cache_control = .{ .max_age = max_age }, + .vary = null, + .headers = &.{}, + }; + + const body = "hello world"; + try cache.put(meta, body); + + const result = cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 50 }, + ) orelse return error.CacheMiss; + result.data.file.close(); + + try testing.expectEqual(null, cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 200 }, + )); + + try testing.expectEqual(null, cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now }, + )); +} From 7edb24e54ddf2fc494913f9c3a76b3ad325e7556 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 08:39:56 -0700 Subject: [PATCH 28/48] use wyhash for power of two lock stripes --- src/network/cache/FsCache.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 4fd53e2f..2f800be8 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -25,6 +25,9 @@ const CachedResponse = Cache.CachedResponse; const CACHE_VERSION: usize = 1; const LOCK_STRIPES = 16; +comptime { + std.debug.assert(std.math.isPowerOfTwo(LOCK_STRIPES)); +} pub const FsCache = @This(); @@ -37,7 +40,7 @@ const CacheMetadataFile = struct { }; fn getLockPtr(self: *FsCache, key: *const [HASHED_KEY_LEN]u8) *std.Thread.Mutex { - const lock_idx: usize = @truncate(std.hash.Wyhash.hash(0, key) % LOCK_STRIPES); + const lock_idx = std.hash.Wyhash.hash(0, key[0..]) & (LOCK_STRIPES - 1); return &self.locks[lock_idx]; } From 9c5e67fbf5c2493619d8806bddf72491a332b0ae Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 08:44:10 -0700 Subject: [PATCH 29/48] properly deinit cache --- src/network/Network.zig | 2 ++ src/network/cache/Cache.zig | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/network/Network.zig b/src/network/Network.zig index 8e218a9f..0cfc7a20 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -289,6 +289,8 @@ pub fn deinit(self: *Network) void { wba.deinit(self.allocator); } + if (self.cache) |*cache| cache.deinit(); + globalDeinit(); } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 54603172..ae79f82c 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -28,6 +28,12 @@ kind: union(enum) { fs: FsCache, }, +pub fn deinit(self: *Cache) void { + return switch (self.kind) { + inline else => |*c| c.deinit(), + }; +} + pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse { return switch (self.kind) { inline else => |*c| c.get(arena, req), From 6a57d69359ea0b02cbc72e7bab44548df6eeb17f Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 09:56:23 -0700 Subject: [PATCH 30/48] switch to single file cache --- src/browser/HttpClient.zig | 24 +++-- src/network/cache/Cache.zig | 6 +- src/network/cache/FsCache.zig | 164 +++++++++++++++------------------- 3 files changed, 94 insertions(+), 100 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 399c1011..46068a7f 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -325,7 +325,7 @@ fn serveFromCache(req: Request, cached: *const CachedResponse) !void { if (!proceed) { switch (cached.data) { .buffer => |_| {}, - .file => |file| file.close(), + .file => |f| f.file.close(), } req.error_callback(req.ctx, error.Abort); return; @@ -337,18 +337,24 @@ fn serveFromCache(req: Request, cached: *const CachedResponse) !void { try req.data_callback(response, data); } }, - .file => |file| { + .file => |f| { + const file = f.file; defer file.close(); + var buf: [1024]u8 = undefined; var file_reader = file.reader(&buf); - + try file_reader.seekTo(f.offset); const reader = &file_reader.interface; - var read_buf: [1024]u8 = undefined; - while (true) { - const curr = try reader.readSliceShort(&read_buf); - if (curr == 0) break; - try req.data_callback(response, read_buf[0..curr]); + var read_buf: [1024]u8 = undefined; + var remaining = f.len; + + while (remaining > 0) { + const read_len = @min(read_buf.len, remaining); + const n = try reader.readSliceShort(read_buf[0..read_len]); + if (n == 0) break; + remaining -= n; + try req.data_callback(response, read_buf[0..n]); } }, } @@ -1133,7 +1139,7 @@ pub const Response = struct { .transfer => |t| t.getContentLength(), .cached => |c| switch (c.data) { .buffer => |buf| @intCast(buf.len), - .file => |f| @intCast(f.getEndPos() catch 0), + .file => |f| @intCast(f.len), }, }; } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index ae79f82c..c5d9af56 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -137,7 +137,11 @@ pub const CacheRequest = struct { pub const CachedData = union(enum) { buffer: []const u8, - file: std.fs.File, + file: struct { + file: std.fs.File, + offset: usize, + len: usize, + }, }; pub const CachedResponse = struct { diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 2f800be8..84bd59c4 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -34,7 +34,7 @@ pub const FsCache = @This(); dir: std.fs.Dir, locks: [LOCK_STRIPES]std.Thread.Mutex = .{std.Thread.Mutex{}} ** LOCK_STRIPES, -const CacheMetadataFile = struct { +const CacheMetadataJson = struct { version: usize, metadata: CachedMetadata, }; @@ -44,8 +44,9 @@ fn getLockPtr(self: *FsCache, key: *const [HASHED_KEY_LEN]u8) *std.Thread.Mutex return &self.locks[lock_idx]; } +const BODY_LEN_HEADER_LEN = 8; const HASHED_KEY_LEN = 64; -const HASHED_PATH_LEN = HASHED_KEY_LEN + 5; +const HASHED_PATH_LEN = HASHED_KEY_LEN + 6; const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4; fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 { @@ -56,27 +57,15 @@ fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 { return hex; } -fn metaPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 { +fn cachePath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 { var path: [HASHED_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&path, "{s}.meta", .{hashed_key}) catch unreachable; + _ = std.fmt.bufPrint(&path, "{s}.cache", .{hashed_key}) catch unreachable; return path; } -fn bodyPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 { - var path: [HASHED_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&path, "{s}.body", .{hashed_key}) catch unreachable; - return path; -} - -fn metaTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 { +fn cacheTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 { var path: [HASHED_TMP_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&path, "{s}.meta.tmp", .{hashed_key}) catch unreachable; - return path; -} - -fn bodyTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 { - var path: [HASHED_TMP_PATH_LEN]u8 = undefined; - _ = std.fmt.bufPrint(&path, "{s}.body.tmp", .{hashed_key}) catch unreachable; + _ = std.fmt.bufPrint(&path, "{s}.cache.tmp", .{hashed_key}) catch unreachable; return path; } @@ -98,106 +87,96 @@ pub fn deinit(self: *FsCache) void { pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.CachedResponse { const hashed_key = hashKey(req.url); - const meta_p = metaPath(&hashed_key); - const body_p = bodyPath(&hashed_key); + const cache_p = cachePath(&hashed_key); const lock = self.getLockPtr(&hashed_key); lock.lock(); defer lock.unlock(); - const meta_file = self.dir.openFile(&meta_p, .{ .mode = .read_only }) catch return null; - defer meta_file.close(); + const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch return null; + errdefer file.close(); - const contents = meta_file.readToEndAlloc(arena, 1 * 1024 * 1024) catch return null; - defer arena.free(contents); + var file_buf: [1024]u8 = undefined; + var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined; - const cache_file: CacheMetadataFile = std.json.parseFromSliceLeaky( - CacheMetadataFile, + var file_reader = file.reader(&file_buf); + const file_reader_iface = &file_reader.interface; + + file_reader_iface.readSliceAll(&len_buf) catch return null; + const body_len = std.mem.readInt(u64, &len_buf, .little); + + // Now we read metadata. + file_reader.seekTo(body_len + BODY_LEN_HEADER_LEN) catch return null; + + var json_reader = std.json.Reader.init(arena, file_reader_iface); + const cache_file: CacheMetadataJson = std.json.parseFromTokenSourceLeaky( + CacheMetadataJson, arena, - contents, - .{ .allocate = .alloc_always }, + &json_reader, + .{ + .allocate = .alloc_always, + }, ) catch { - self.dir.deleteFile(&meta_p) catch {}; - self.dir.deleteFile(&body_p) catch {}; + self.dir.deleteFile(&cache_p) catch {}; return null; }; - const metadata = cache_file.metadata; - if (cache_file.version != CACHE_VERSION) { - self.dir.deleteFile(&meta_p) catch {}; - self.dir.deleteFile(&body_p) catch {}; + self.dir.deleteFile(&cache_p) catch {}; return null; } + const metadata = cache_file.metadata; + const now = req.timestamp; const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store)); if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) { - self.dir.deleteFile(&meta_p) catch {}; - self.dir.deleteFile(&body_p) catch {}; + self.dir.deleteFile(&cache_p) catch {}; return null; } - const body_file = self.dir.openFile( - &body_p, - .{ .mode = .read_only }, - ) catch return null; - return .{ .metadata = metadata, - .data = .{ .file = body_file }, + .data = .{ + .file = .{ + .file = file, + .offset = BODY_LEN_HEADER_LEN, + .len = body_len, + }, + }, }; } pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void { const hashed_key = hashKey(meta.url); - const meta_p = metaPath(&hashed_key); - const meta_tmp_p = metaTmpPath(&hashed_key); - const body_p = bodyPath(&hashed_key); - const body_tmp_p = bodyTmpPath(&hashed_key); - var writer_buf: [512]u8 = undefined; + const cache_p = cachePath(&hashed_key); + const cache_tmp_p = cacheTmpPath(&hashed_key); const lock = self.getLockPtr(&hashed_key); lock.lock(); defer lock.unlock(); - { - const meta_file = try self.dir.createFile(&meta_tmp_p, .{}); - errdefer { - meta_file.close(); - self.dir.deleteFile(&meta_tmp_p) catch {}; - } + const file = try self.dir.createFile(&cache_tmp_p, .{}); + defer file.close(); - var meta_file_writer = meta_file.writer(&writer_buf); - const meta_file_writer_iface = &meta_file_writer.interface; - try std.json.Stringify.value( - CacheMetadataFile{ .version = CACHE_VERSION, .metadata = meta }, - .{ .whitespace = .minified }, - meta_file_writer_iface, - ); - try meta_file_writer_iface.flush(); - meta_file.close(); - } - errdefer self.dir.deleteFile(&meta_tmp_p) catch {}; - try self.dir.rename(&meta_tmp_p, &meta_p); + var writer_buf: [1024]u8 = undefined; - { - const body_file = try self.dir.createFile(&body_tmp_p, .{}); - errdefer { - body_file.close(); - self.dir.deleteFile(&body_tmp_p) catch {}; - } + var file_writer = file.writer(&writer_buf); + var file_writer_iface = &file_writer.interface; - var body_file_writer = body_file.writer(&writer_buf); - const body_file_writer_iface = &body_file_writer.interface; - try body_file_writer_iface.writeAll(body); - try body_file_writer_iface.flush(); - body_file.close(); - } - errdefer self.dir.deleteFile(&body_tmp_p) catch {}; + var len_buf: [8]u8 = undefined; + std.mem.writeInt(u64, &len_buf, body.len, .little); + try file_writer_iface.writeAll(&len_buf); + try file_writer_iface.writeAll(body); - errdefer self.dir.deleteFile(&meta_p) catch {}; - try self.dir.rename(&body_tmp_p, &body_p); + try std.json.Stringify.value( + CacheMetadataJson{ .version = CACHE_VERSION, .metadata = meta }, + .{ .whitespace = .minified }, + file_writer_iface, + ); + + try file_writer_iface.flush(); + try self.dir.rename(&cache_tmp_p, &cache_p); } const testing = std.testing; @@ -209,9 +188,9 @@ test "FsCache: basic put and get" { const path = try tmp.dir.realpathAlloc(testing.allocator, "."); defer testing.allocator.free(path); - var fs_cache = try FsCache.init(path); - defer fs_cache.deinit(); + const fs_cache = try FsCache.init(path); var cache = Cache{ .kind = .{ .fs = fs_cache } }; + defer cache.deinit(); var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -233,15 +212,20 @@ test "FsCache: basic put and get" { const body = "hello world"; try cache.put(meta, body); - const result = cache.get(arena.allocator(), .{ .url = "https://example.com", .timestamp = now }) orelse return error.CacheMiss; - defer result.data.file.close(); + const result = cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now }, + ) orelse return error.CacheMiss; + const f = result.data.file; + const file = f.file; + defer file.close(); var buf: [64]u8 = undefined; - var file_reader = result.data.file.reader(&buf); + var file_reader = file.reader(&buf); + try file_reader.seekTo(f.offset); - const read_buf = try file_reader.interface.allocRemaining(testing.allocator, .unlimited); + const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len); defer testing.allocator.free(read_buf); - try testing.expectEqualStrings(body, read_buf); } @@ -252,9 +236,9 @@ test "FsCache: get expiration" { const path = try tmp.dir.realpathAlloc(testing.allocator, "."); defer testing.allocator.free(path); - var fs_cache = try FsCache.init(path); - defer fs_cache.deinit(); + const fs_cache = try FsCache.init(path); var cache = Cache{ .kind = .{ .fs = fs_cache } }; + defer cache.deinit(); var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -282,7 +266,7 @@ test "FsCache: get expiration" { arena.allocator(), .{ .url = "https://example.com", .timestamp = now + 50 }, ) orelse return error.CacheMiss; - result.data.file.close(); + result.data.file.file.close(); try testing.expectEqual(null, cache.get( arena.allocator(), From d65a4b09f39def6e864cc3cb6d755b03abe2065e Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 10:52:20 -0700 Subject: [PATCH 31/48] better logging for cache --- src/log.zig | 1 + src/network/cache/FsCache.zig | 49 +++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/log.zig b/src/log.zig index 201e6c9f..3e1016c5 100644 --- a/src/log.zig +++ b/src/log.zig @@ -39,6 +39,7 @@ pub const Scope = enum { telemetry, unknown_prop, mcp, + cache, }; const Opts = struct { diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 84bd59c4..0624a816 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -17,6 +17,7 @@ // along with this program. If not, see . const std = @import("std"); +const log = @import("../../log.zig"); const Cache = @import("Cache.zig"); const Http = @import("../http.zig"); const CacheRequest = Cache.CacheRequest; @@ -93,8 +94,25 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C lock.lock(); defer lock.unlock(); - const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch return null; - errdefer file.close(); + const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch |e| { + switch (e) { + std.fs.File.OpenError.FileNotFound => { + log.debug(.cache, "miss", .{ .url = req.url, .hash = &hashed_key }); + }, + else => |err| { + log.warn(.cache, "open file err", .{ .url = req.url, .err = err }); + }, + } + return null; + }; + + var cleanup = false; + defer if (cleanup) { + file.close(); + self.dir.deleteFile(&cache_p) catch |e| { + log.err(.cache, "clean fail", .{ .url = req.url, .file = &cache_p, .err = e }); + }; + }; var file_buf: [1024]u8 = undefined; var len_buf: [BODY_LEN_HEADER_LEN]u8 = undefined; @@ -102,27 +120,35 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C var file_reader = file.reader(&file_buf); const file_reader_iface = &file_reader.interface; - file_reader_iface.readSliceAll(&len_buf) catch return null; + file_reader_iface.readSliceAll(&len_buf) catch |e| { + log.warn(.cache, "read header", .{ .url = req.url, .err = e }); + cleanup = true; + return null; + }; const body_len = std.mem.readInt(u64, &len_buf, .little); // Now we read metadata. - file_reader.seekTo(body_len + BODY_LEN_HEADER_LEN) catch return null; + file_reader.seekTo(body_len + BODY_LEN_HEADER_LEN) catch |e| { + log.warn(.cache, "seek metadata", .{ .url = req.url, .err = e }); + cleanup = true; + return null; + }; var json_reader = std.json.Reader.init(arena, file_reader_iface); const cache_file: CacheMetadataJson = std.json.parseFromTokenSourceLeaky( CacheMetadataJson, arena, &json_reader, - .{ - .allocate = .alloc_always, - }, - ) catch { - self.dir.deleteFile(&cache_p) catch {}; + .{ .allocate = .alloc_always }, + ) catch |e| { + log.warn(.cache, "metadata parse", .{ .url = req.url, .err = e }); + cleanup = true; return null; }; if (cache_file.version != CACHE_VERSION) { - self.dir.deleteFile(&cache_p) catch {}; + log.warn(.cache, "version", .{ .url = req.url, .expected = CACHE_VERSION, .got = cache_file.version }); + cleanup = true; return null; } @@ -131,7 +157,8 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C const now = req.timestamp; const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store)); if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) { - self.dir.deleteFile(&cache_p) catch {}; + log.debug(.cache, "expired", .{ .url = req.url }); + cleanup = true; return null; } From 855c3290ffc158c4dcd3a36a81d2738c96017674 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 11:33:19 -0700 Subject: [PATCH 32/48] always close file on serveFromCache --- ...6c6f2dcc1e6022693e854809c72179a7a486b7.cache | Bin 0 -> 10197 bytes src/browser/HttpClient.zig | 9 ++++----- 2 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 lp-net-cache/6ba046812fd9e6b6c7e484ec946c6f2dcc1e6022693e854809c72179a7a486b7.cache diff --git a/lp-net-cache/6ba046812fd9e6b6c7e484ec946c6f2dcc1e6022693e854809c72179a7a486b7.cache b/lp-net-cache/6ba046812fd9e6b6c7e484ec946c6f2dcc1e6022693e854809c72179a7a486b7.cache new file mode 100644 index 0000000000000000000000000000000000000000..a944dbb8e2443049422f18307899943f92913c61 GIT binary patch literal 10197 zcmbVSZFAeW5$;#{S0K8Zqtr;W<-DF;HV-`tCbbhMWi2_NARyXAr?Uw%98F_eIIB#$w-J&wu|d%ym(k-dDKX9CSJ} zsAQ>nrYOKw%`#kUVnRIfsBwC&lQE~og~?2l4LWTG>`K_uM8h^ZI}GCu}x&4Fmr5Z3JwaB;4*}A z&FA0ey{o?ZyBdyU+dV43xVpqe;avD~(4m@nSd-2$ivP37{PX=UXW1&xQXI}%!B4nc zoKWsSZA?%Ywa8!*%4Xn{Mk<@n`JGl<898`GIcO}b;;xB>3u7nLGB&Mk7Uw9Pdif1A z^5KlJBd4rZz_dG^d@R>oLfv*Lws3lSD&8}XQ~1#<9IV+4nP^vE4c)T?j{ZE+%a}CW z#F1SNykl2`OA{}acA$C{ZDg(yTKWbc~2JnP#@!%N3>GRrURApKRcT$g{X0^uVDzVn5mv!m9z|<=gC$j8gUT@A?{$exyj}| z>NJD{@}>fB3qfCjD@?%6?pa~)uHf#~3cF`Tes={9?N{WKh9L-&n^kVN{$Ty;3?%eX z&1-${65FqqFd)@SM7ewEb5Eaxo_;}F?e+b6a1X5`;K?7%i&)QdKiq+vm#{;5AY-+j z{PE=GbH<0l*mdDTn+pZd-PzL25W40{!&8#AB&V+2NLMko(Z}<3k!Q#)*<21xCH4AM z`1kjR&Ac{y*Nm^#X4j(Wbd7KVlXFkZdCqGwRM`dA?3#qCQqsA>nGkN6J(DbheI@ag z!Es@*Cp;da6Q;^+g~2F_X|_6HqJ{A>Xk{s>B)!o-5vT@gy0#pIOnHG6TiDo?u;4VX zYHHNnAWeZ7;O6OnU0J|ikiX;jr zlM+cfjn}KmI^7g-SBQ(9II}U+sEDskfqVsvfDikbTf@7|k1OV>zK$0f=|8L)?Gio# zVRdSpxzh6Q(@($*3?1tN2V-6W5CN7+7Xy51Xb7s@#EAV`y{gnvleRUIO+VFHMbQT? zkoRt-Y0_X&0sy`0s0KxPkAOJY3tIWcK^!uxPeVw+{e7CRp+I4P`AW;EUulIgttM+lg+?p?FJ$zW z_5g0|0|@Mef@-{*Q7gEzBj9Wekd<1+h};rJz{FlbkS1~cK3PgK<$(9++i*CE)2j<0 z|I;o)O_s)R^X}!v66r+{v6$lw*Ca{xo!&AQfgG6_`4zEP02XgPist}?JZhA!beuA< zNMs?vs&@xGfb+gd454{1&@`@2M~B=HbTj$l-2lT`X;UCjAfr6dGt)c#7b=;L;UPdL z5;UW;m-H*NvQNidj>ZPW$(6nLFe+$_JHhh#=rBf~Q6@MHYK^>Kr*%qEdX8wWe*FC% zV?80Yi7E|cDR4BZmX?C%jT6+DG0mcFor>gu^%Szk8zS?!acNCzPz+$7vkXNY#$+qt z?mD?qKXVjZHyHCiZ{Yj9NQ~FV|A!$r@kaT43@xrNfj*S&y@0RDS;i_j(A9JK{p76pdfjE_yXXrUT0(MPRN z5xPo7N*C_qEkxoeDl=SR=*M<=yJF{lj$D+qIW{)8Vjb5zq$%lyR{QOr{~8^?_!#2j zNdA9u{4wl}E}jp+zx?y~;)owl@G-jV%k=k{9zB1_@1U>#ecL~j!0atGsM$Xt_uW%M zu{N7!Nrh%m?vHmbRhF;~;MvN+ITEH`7%y;AJ+pB7#hMjL>8m(j%ql+&>@LqJE`c(T zF(pBMW3$B~G3}zAdsSK$qphu!as8j~-`3$ACiqj+)7Wo_NV{z-WHse1k=UlH+4k5k z^MXJ}E}*l2xuHccRSE1zu11Jq%I!-4l;kTA41JBFt}i6kIa#n;Ru(8pPt`j{?LwpH zu>v}^*}0&H-jnD3uv|}pC3?pL^~~}9!GkpCeLC6bErBHi0Po^V0{{*g1BxsOedW{2 zYV9OPr+1Fw2{_>I+hx}@Yf|SNeW3^n1W~0EzYYPh#x!y%U{t{Y=`sNiEaQ7hY>6EA z>xB`&?YAzP)JLE1JF<1*i11@-W}5A#Z0^8;&{N;e7}}gMBBd-jQn`;X@IP{wOiS=f zpk7rFuxeU@yD#C_AWSwRelQw9rpHa36|#>dy|xuX1EBX@$4RLolr~W8)QB|jxhpW9 zZq?cOAE+YGU(yRC(h6aw)F0TyEBNGaDvdh$jT}qtkCMl$1@5#Cthcf)W}T1XOIQ|! z1vn!4%7ERX^@O5I$`#B7dWvq;nvt6o^=ssJz~Mx1GgK<9D`eYdDasHQ2UvKNf2Mc0 z*h)iyQ=>amt}SqTj`@?)ClOa5_j-Zvrf$lD?Tb|_J$Gs) z5e0;-cG9R}2WDVc`56PER?^1=4rE8p^)T)UXpXQY8F5V$4T*Fa-&?&DAB;u_OF0sD zh5DNcCB~Yg6BVn!0m-5kTG%C?9zPZz;-!@r@g?Hh+iYWsGuS4goo@yTe%qOH^@oFR zN5aAf9br>NMp@_f!U8rEtLyks&=Y6qJ6|d3LeUWS+{F|psQR0?^o-jTudQ{k(<6TE zqy4Yz(n^5>fdn+6AE^*=P)DcuH;aKJd~GehP^AE{r!z`hvZV%8x2q}~_`J8%D~H3h zW0nNZo-`Ue9g%a)b{=h5P(AvO$SyL51~Pc>>sg=4Eg`W9fdE|pfW|V>ZQvSgosP7J zgx$T|KCt=ARl-dc@X67~DRAn~AKtyK*m|Qn8mQx=BL}7H&3yX`-OBrp{=RB3*ZVQA z-fGD^8i2C4B$C4EK4LJP*F~YXp zpPKuv?iK`e0sft;)&sPUw?(*1jD=LPC-4t64|Jar$s#0j@z!zi4t7-~{#$ow-B@i{}CG>San`$Q<`d1<#KHbOVByM?s%a z&PhT}7XcD=!1pD)@QyLm6o+*e2P}Ce>FtE=QC^3PzH%uP=6P%XKCd!tZ}u1}|E_hy z)6ve_l1KrdRsQy_6^f?C1omzXC8L9Ik#x|$!Ua9HTU;_LUk*@!hr)%i z`^BBPMlP9A+G>J~FL}Da{Ueh@QRQ{>omRsMR~z13a6PZ|(u?v44CF*UhdFTFGVw89 z30%?g&qw29zI6w{)>yOu0LQ)hvB znl9e)35l~R3f%7QhPw#Si?aIyk!PiKI4L9aMp_8$YS7&J)n#+7>~ii8uc89yf3+qb z$-98M_SkcU3{gzCz{{2I;#-Qww^T08rM4L7JE%(HZYzGh)9Bd8R^gTRR!U72vkK6* zP}J0QY&L`y)JzSQmbzTJ8hT$j>$K$;{14T#ggUmQbWQ9~*d-jD&|Nl*Y{N;S_t}uHWCt~jY z3K_MUZkelZ>ci1Qd3`&*ocet=ZRNhafa+uvsz2DQfY8z+O3c9JB$RB@4VFX)!sCXJ z=CcL1t`?l2;{}6or^c5OoePxQ^77C%v(O*`?u+Vs7P__(L>#I0iEDi;&grCappo`j`eb@=ZHR!GR zY!5%_Ywdnoa-wh@`1yxDUEBHaCFsd$U3SNS0er{`6HT@y0Bf0 zO=Pw^1|5#I=#JTD7<3a|+Q}+Ii#;}xADpot1P~?tAfA4uKTp7>JAV26DR)|-e3=w{ zVHQI`j7yts(2ZBCwPkOv8o3#Ckr%dPA+K8@((BpGnX7O0INO^A^54ty|d6MYA7%7C`WUu#ed>_A8Bdu#sw|wVV;5a zwzWcX?+|4lUM*$l79z8zCOC0`cgS{1fESo^jNnbOUQjwS&LctpGAiB zDox7qiE=jdASt%Yiq%ld&#dZA1z%=#jb$4B(Hty?Kx9nvUj?lA>qHUYmXE9?0k0GyN2xx^k T#F*l}CCy4)_`G{_d2{n$>p;9@ literal 0 HcmV?d00001 diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 46068a7f..8f4e900f 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -316,6 +316,10 @@ pub fn request(self: *Client, req: Request) !void { fn serveFromCache(req: Request, cached: *const CachedResponse) !void { const response = Response.fromCached(req.ctx, cached); + defer switch (cached.data) { + .buffer => |_| {}, + .file => |f| f.file.close(), + }; if (req.start_callback) |cb| { try cb(response); @@ -323,10 +327,6 @@ fn serveFromCache(req: Request, cached: *const CachedResponse) !void { const proceed = try req.header_callback(response); if (!proceed) { - switch (cached.data) { - .buffer => |_| {}, - .file => |f| f.file.close(), - } req.error_callback(req.ctx, error.Abort); return; } @@ -339,7 +339,6 @@ fn serveFromCache(req: Request, cached: *const CachedResponse) !void { }, .file => |f| { const file = f.file; - defer file.close(); var buf: [1024]u8 = undefined; var file_reader = file.reader(&buf); From 9ffc99d6a259f50b39707497bc5f72b9ea0ddf88 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 13:28:01 -0700 Subject: [PATCH 33/48] add more FsCache tests --- src/network/cache/FsCache.zig | 148 ++++++++++++++++++++++++++++++---- 1 file changed, 134 insertions(+), 14 deletions(-) diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 0624a816..4d70866d 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -183,7 +183,7 @@ pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void { lock.lock(); defer lock.unlock(); - const file = try self.dir.createFile(&cache_tmp_p, .{}); + const file = try self.dir.createFile(&cache_tmp_p, .{ .truncate = true }); defer file.close(); var writer_buf: [1024]u8 = undefined; @@ -208,16 +208,27 @@ pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void { const testing = std.testing; -test "FsCache: basic put and get" { +fn setupCache() !struct { tmp: testing.TmpDir, cache: Cache } { var tmp = testing.tmpDir(.{}); - defer tmp.cleanup(); + errdefer tmp.cleanup(); const path = try tmp.dir.realpathAlloc(testing.allocator, "."); defer testing.allocator.free(path); - const fs_cache = try FsCache.init(path); - var cache = Cache{ .kind = .{ .fs = fs_cache } }; - defer cache.deinit(); + return .{ + .tmp = tmp, + .cache = Cache{ .kind = .{ .fs = try FsCache.init(path) } }, + }; +} + +test "FsCache: basic put and get" { + var setup = try setupCache(); + defer { + setup.cache.deinit(); + setup.tmp.cleanup(); + } + + const cache = &setup.cache; var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -257,15 +268,13 @@ test "FsCache: basic put and get" { } test "FsCache: get expiration" { - var tmp = testing.tmpDir(.{}); - defer tmp.cleanup(); + var setup = try setupCache(); + defer { + setup.cache.deinit(); + setup.tmp.cleanup(); + } - const path = try tmp.dir.realpathAlloc(testing.allocator, "."); - defer testing.allocator.free(path); - - const fs_cache = try FsCache.init(path); - var cache = Cache{ .kind = .{ .fs = fs_cache } }; - defer cache.deinit(); + const cache = &setup.cache; var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); @@ -305,3 +314,114 @@ test "FsCache: get expiration" { .{ .url = "https://example.com", .timestamp = now }, )); } + +test "FsCache: put override" { + var setup = try setupCache(); + defer { + setup.cache.deinit(); + setup.tmp.cleanup(); + } + + const cache = &setup.cache; + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + { + const now = 5000; + const max_age = 1000; + + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 900, + .etag = null, + .last_modified = null, + .cache_control = .{ .max_age = max_age }, + .vary = null, + .headers = &.{}, + }; + + const body = "hello world"; + try cache.put(meta, body); + + const result = cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now }, + ) orelse return error.CacheMiss; + const f = result.data.file; + const file = f.file; + defer file.close(); + + var buf: [64]u8 = undefined; + var file_reader = file.reader(&buf); + try file_reader.seekTo(f.offset); + + const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len); + defer testing.allocator.free(read_buf); + + try testing.expectEqualStrings(body, read_buf); + } + + { + const now = 10000; + const max_age = 2000; + + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .etag = null, + .last_modified = null, + .cache_control = .{ .max_age = max_age }, + .vary = null, + .headers = &.{}, + }; + + const body = "goodbye world"; + try cache.put(meta, body); + + const result = cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now }, + ) orelse return error.CacheMiss; + const f = result.data.file; + const file = f.file; + defer file.close(); + + var buf: [64]u8 = undefined; + var file_reader = file.reader(&buf); + try file_reader.seekTo(f.offset); + + const read_buf = try file_reader.interface.readAlloc(testing.allocator, f.len); + defer testing.allocator.free(read_buf); + + try testing.expectEqualStrings(body, read_buf); + } +} + +test "FsCache: garbage file" { + var setup = try setupCache(); + defer { + setup.cache.deinit(); + setup.tmp.cleanup(); + } + + const hashed_key = hashKey("https://example.com"); + const cache_p = cachePath(&hashed_key); + const file = try setup.cache.kind.fs.dir.createFile(&cache_p, .{}); + try file.writeAll("this is not a valid cache file !@#$%"); + file.close(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + try testing.expectEqual( + null, + setup.cache.get(arena.allocator(), .{ .url = "https://example.com", .timestamp = 5000 }), + ); +} From 7b5e4d6f5234ea28f8c9e0c1b2701c2e045034bf Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 15:19:30 -0700 Subject: [PATCH 34/48] add Vary support --- ...6f2dcc1e6022693e854809c72179a7a486b7.cache | Bin 10197 -> 0 bytes src/browser/HttpClient.zig | 65 +++--- src/network/cache/Cache.zig | 26 +-- src/network/cache/FsCache.zig | 187 ++++++++++++++++-- src/network/http.zig | 43 ++-- 5 files changed, 246 insertions(+), 75 deletions(-) delete mode 100644 lp-net-cache/6ba046812fd9e6b6c7e484ec946c6f2dcc1e6022693e854809c72179a7a486b7.cache diff --git a/lp-net-cache/6ba046812fd9e6b6c7e484ec946c6f2dcc1e6022693e854809c72179a7a486b7.cache b/lp-net-cache/6ba046812fd9e6b6c7e484ec946c6f2dcc1e6022693e854809c72179a7a486b7.cache deleted file mode 100644 index a944dbb8e2443049422f18307899943f92913c61..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10197 zcmbVSZFAeW5$;#{S0K8Zqtr;W<-DF;HV-`tCbbhMWi2_NARyXAr?Uw%98F_eIIB#$w-J&wu|d%ym(k-dDKX9CSJ} zsAQ>nrYOKw%`#kUVnRIfsBwC&lQE~og~?2l4LWTG>`K_uM8h^ZI}GCu}x&4Fmr5Z3JwaB;4*}A z&FA0ey{o?ZyBdyU+dV43xVpqe;avD~(4m@nSd-2$ivP37{PX=UXW1&xQXI}%!B4nc zoKWsSZA?%Ywa8!*%4Xn{Mk<@n`JGl<898`GIcO}b;;xB>3u7nLGB&Mk7Uw9Pdif1A z^5KlJBd4rZz_dG^d@R>oLfv*Lws3lSD&8}XQ~1#<9IV+4nP^vE4c)T?j{ZE+%a}CW z#F1SNykl2`OA{}acA$C{ZDg(yTKWbc~2JnP#@!%N3>GRrURApKRcT$g{X0^uVDzVn5mv!m9z|<=gC$j8gUT@A?{$exyj}| z>NJD{@}>fB3qfCjD@?%6?pa~)uHf#~3cF`Tes={9?N{WKh9L-&n^kVN{$Ty;3?%eX z&1-${65FqqFd)@SM7ewEb5Eaxo_;}F?e+b6a1X5`;K?7%i&)QdKiq+vm#{;5AY-+j z{PE=GbH<0l*mdDTn+pZd-PzL25W40{!&8#AB&V+2NLMko(Z}<3k!Q#)*<21xCH4AM z`1kjR&Ac{y*Nm^#X4j(Wbd7KVlXFkZdCqGwRM`dA?3#qCQqsA>nGkN6J(DbheI@ag z!Es@*Cp;da6Q;^+g~2F_X|_6HqJ{A>Xk{s>B)!o-5vT@gy0#pIOnHG6TiDo?u;4VX zYHHNnAWeZ7;O6OnU0J|ikiX;jr zlM+cfjn}KmI^7g-SBQ(9II}U+sEDskfqVsvfDikbTf@7|k1OV>zK$0f=|8L)?Gio# zVRdSpxzh6Q(@($*3?1tN2V-6W5CN7+7Xy51Xb7s@#EAV`y{gnvleRUIO+VFHMbQT? zkoRt-Y0_X&0sy`0s0KxPkAOJY3tIWcK^!uxPeVw+{e7CRp+I4P`AW;EUulIgttM+lg+?p?FJ$zW z_5g0|0|@Mef@-{*Q7gEzBj9Wekd<1+h};rJz{FlbkS1~cK3PgK<$(9++i*CE)2j<0 z|I;o)O_s)R^X}!v66r+{v6$lw*Ca{xo!&AQfgG6_`4zEP02XgPist}?JZhA!beuA< zNMs?vs&@xGfb+gd454{1&@`@2M~B=HbTj$l-2lT`X;UCjAfr6dGt)c#7b=;L;UPdL z5;UW;m-H*NvQNidj>ZPW$(6nLFe+$_JHhh#=rBf~Q6@MHYK^>Kr*%qEdX8wWe*FC% zV?80Yi7E|cDR4BZmX?C%jT6+DG0mcFor>gu^%Szk8zS?!acNCzPz+$7vkXNY#$+qt z?mD?qKXVjZHyHCiZ{Yj9NQ~FV|A!$r@kaT43@xrNfj*S&y@0RDS;i_j(A9JK{p76pdfjE_yXXrUT0(MPRN z5xPo7N*C_qEkxoeDl=SR=*M<=yJF{lj$D+qIW{)8Vjb5zq$%lyR{QOr{~8^?_!#2j zNdA9u{4wl}E}jp+zx?y~;)owl@G-jV%k=k{9zB1_@1U>#ecL~j!0atGsM$Xt_uW%M zu{N7!Nrh%m?vHmbRhF;~;MvN+ITEH`7%y;AJ+pB7#hMjL>8m(j%ql+&>@LqJE`c(T zF(pBMW3$B~G3}zAdsSK$qphu!as8j~-`3$ACiqj+)7Wo_NV{z-WHse1k=UlH+4k5k z^MXJ}E}*l2xuHccRSE1zu11Jq%I!-4l;kTA41JBFt}i6kIa#n;Ru(8pPt`j{?LwpH zu>v}^*}0&H-jnD3uv|}pC3?pL^~~}9!GkpCeLC6bErBHi0Po^V0{{*g1BxsOedW{2 zYV9OPr+1Fw2{_>I+hx}@Yf|SNeW3^n1W~0EzYYPh#x!y%U{t{Y=`sNiEaQ7hY>6EA z>xB`&?YAzP)JLE1JF<1*i11@-W}5A#Z0^8;&{N;e7}}gMBBd-jQn`;X@IP{wOiS=f zpk7rFuxeU@yD#C_AWSwRelQw9rpHa36|#>dy|xuX1EBX@$4RLolr~W8)QB|jxhpW9 zZq?cOAE+YGU(yRC(h6aw)F0TyEBNGaDvdh$jT}qtkCMl$1@5#Cthcf)W}T1XOIQ|! z1vn!4%7ERX^@O5I$`#B7dWvq;nvt6o^=ssJz~Mx1GgK<9D`eYdDasHQ2UvKNf2Mc0 z*h)iyQ=>amt}SqTj`@?)ClOa5_j-Zvrf$lD?Tb|_J$Gs) z5e0;-cG9R}2WDVc`56PER?^1=4rE8p^)T)UXpXQY8F5V$4T*Fa-&?&DAB;u_OF0sD zh5DNcCB~Yg6BVn!0m-5kTG%C?9zPZz;-!@r@g?Hh+iYWsGuS4goo@yTe%qOH^@oFR zN5aAf9br>NMp@_f!U8rEtLyks&=Y6qJ6|d3LeUWS+{F|psQR0?^o-jTudQ{k(<6TE zqy4Yz(n^5>fdn+6AE^*=P)DcuH;aKJd~GehP^AE{r!z`hvZV%8x2q}~_`J8%D~H3h zW0nNZo-`Ue9g%a)b{=h5P(AvO$SyL51~Pc>>sg=4Eg`W9fdE|pfW|V>ZQvSgosP7J zgx$T|KCt=ARl-dc@X67~DRAn~AKtyK*m|Qn8mQx=BL}7H&3yX`-OBrp{=RB3*ZVQA z-fGD^8i2C4B$C4EK4LJP*F~YXp zpPKuv?iK`e0sft;)&sPUw?(*1jD=LPC-4t64|Jar$s#0j@z!zi4t7-~{#$ow-B@i{}CG>San`$Q<`d1<#KHbOVByM?s%a z&PhT}7XcD=!1pD)@QyLm6o+*e2P}Ce>FtE=QC^3PzH%uP=6P%XKCd!tZ}u1}|E_hy z)6ve_l1KrdRsQy_6^f?C1omzXC8L9Ik#x|$!Ua9HTU;_LUk*@!hr)%i z`^BBPMlP9A+G>J~FL}Da{Ueh@QRQ{>omRsMR~z13a6PZ|(u?v44CF*UhdFTFGVw89 z30%?g&qw29zI6w{)>yOu0LQ)hvB znl9e)35l~R3f%7QhPw#Si?aIyk!PiKI4L9aMp_8$YS7&J)n#+7>~ii8uc89yf3+qb z$-98M_SkcU3{gzCz{{2I;#-Qww^T08rM4L7JE%(HZYzGh)9Bd8R^gTRR!U72vkK6* zP}J0QY&L`y)JzSQmbzTJ8hT$j>$K$;{14T#ggUmQbWQ9~*d-jD&|Nl*Y{N;S_t}uHWCt~jY z3K_MUZkelZ>ci1Qd3`&*ocet=ZRNhafa+uvsz2DQfY8z+O3c9JB$RB@4VFX)!sCXJ z=CcL1t`?l2;{}6or^c5OoePxQ^77C%v(O*`?u+Vs7P__(L>#I0iEDi;&grCappo`j`eb@=ZHR!GR zY!5%_Ywdnoa-wh@`1yxDUEBHaCFsd$U3SNS0er{`6HT@y0Bf0 zO=Pw^1|5#I=#JTD7<3a|+Q}+Ii#;}xADpot1P~?tAfA4uKTp7>JAV26DR)|-e3=w{ zVHQI`j7yts(2ZBCwPkOv8o3#Ckr%dPA+K8@((BpGnX7O0INO^A^54ty|d6MYA7%7C`WUu#ed>_A8Bdu#sw|wVV;5a zwzWcX?+|4lUM*$l79z8zCOC0`cgS{1fESo^jNnbOUQjwS&LctpGAiB zDox7qiE=jdASt%Yiq%ld&#dZA1z%=#jb$4B(Hty?Kx9nvUj?lA>qHUYmXE9?0k0GyN2xx^k T#F*l}CCy4)_`G{_d2{n$>p;9@ diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 8f4e900f..b16b6402 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -367,7 +367,14 @@ fn processRequest(self: *Client, req: Request) !void { const arena = try self.network.app.arena_pool.acquire(.{ .debug = "HttpClient.processRequest.cache" }); defer self.network.app.arena_pool.release(arena); - if (cache.get(arena, .{ .url = req.url, .timestamp = std.time.timestamp() })) |cached| { + var iter = req.headers.iterator(); + const req_header_list = try iter.collect(arena); + + if (cache.get(arena, .{ + .url = req.url, + .timestamp = std.time.timestamp(), + .request_headers = req_header_list.items, + })) |cached| { log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = true, @@ -963,23 +970,6 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T } } - const allocator = transfer.arena.allocator(); - var header_list: std.ArrayList(http.Header) = .empty; - - var it = transfer.responseHeaderIterator(); - while (it.next()) |hdr| { - header_list.append( - allocator, - .{ - .name = try allocator.dupe(u8, hdr.name), - .value = try allocator.dupe(u8, hdr.value), - }, - ) catch |err| { - log.warn(.http, "cache header collect failed", .{ .err = err }); - break; - }; - } - // release conn ASAP so that it's available; some done_callbacks // will load more resources. transfer.releaseConn(); @@ -1566,6 +1556,8 @@ pub const Transfer = struct { const rh = &transfer.response_header.?; const allocator = transfer.arena.allocator(); + const vary = if (conn.getResponseHeader("vary", 0)) |h| h.value else null; + const maybe_cm = try Cache.tryCache( allocator, std.time.timestamp(), @@ -1573,7 +1565,7 @@ pub const Transfer = struct { rh.status, rh.contentType(), if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null, - if (conn.getResponseHeader("vary", 0)) |h| h.value else null, + vary, if (conn.getResponseHeader("etag", 0)) |h| h.value else null, if (conn.getResponseHeader("last-modified", 0)) |h| h.value else null, if (conn.getResponseHeader("age", 0)) |h| h.value else null, @@ -1582,17 +1574,32 @@ pub const Transfer = struct { ); if (maybe_cm) |cm| { - var header_list: std.ArrayList(http.Header) = .empty; - var it = transfer.responseHeaderIterator(); - while (it.next()) |hdr| { - try header_list.append(allocator, .{ - .name = try allocator.dupe(u8, hdr.name), - .value = try allocator.dupe(u8, hdr.value), - }); - } - transfer._pending_cache_metadata = cm; - transfer._pending_cache_metadata.?.headers = header_list.items; + + var iter = transfer.responseHeaderIterator(); + var header_list = try iter.collect(allocator); + const end_of_response = header_list.items.len; + transfer._pending_cache_metadata.?.headers = header_list.items[0..end_of_response]; + + if (vary) |vary_str| { + var req_it = transfer.req.headers.iterator(); + + while (req_it.next()) |hdr| { + var vary_iter = std.mem.splitScalar(u8, vary_str, ','); + + while (vary_iter.next()) |part| { + const name = std.mem.trim(u8, part, &std.ascii.whitespace); + if (std.ascii.eqlIgnoreCase(hdr.name, name)) { + try header_list.append(allocator, .{ + .name = try allocator.dupe(u8, hdr.name), + .value = try allocator.dupe(u8, hdr.value), + }); + } + } + } + + transfer._pending_cache_metadata.?.vary_headers = header_list.items[end_of_response..]; + } } } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index c5d9af56..fd20e967 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -95,23 +95,6 @@ pub const CacheControl = struct { } }; -pub const Vary = union(enum) { - wildcard: void, - value: []const u8, - - pub fn parse(value: []const u8) Vary { - if (std.mem.eql(u8, value, "*")) return .wildcard; - return .{ .value = value }; - } - - pub fn toString(self: Vary) []const u8 { - return switch (self) { - .wildcard => "*", - .value => |v| v, - }; - } -}; - pub const CachedMetadata = struct { url: [:0]const u8, content_type: []const u8, @@ -126,13 +109,17 @@ pub const CachedMetadata = struct { last_modified: ?[]const u8, cache_control: CacheControl, - vary: ?Vary, + /// Response Headers headers: []const Http.Header, + + /// These are Request Headers used by Vary. + vary_headers: []const Http.Header, }; pub const CacheRequest = struct { url: []const u8, timestamp: i64, + request_headers: []const Http.Header, }; pub const CachedData = union(enum) { @@ -166,6 +153,7 @@ pub fn tryCache( if (status != 200) return null; if (has_set_cookie) return null; if (has_authorization) return null; + if (vary) |v| if (std.mem.eql(u8, v, "*")) return null; const cc = CacheControl.parse(cache_control orelse return null) orelse return null; return .{ @@ -175,9 +163,9 @@ pub fn tryCache( .stored_at = timestamp, .age_at_store = if (age) |a| std.fmt.parseInt(u64, a, 10) catch 0 else 0, .cache_control = cc, - .vary = if (vary) |v| Vary.parse(v) else null, .etag = if (etag) |e| try arena.dupe(u8, e) else null, .last_modified = if (last_modified) |lm| try arena.dupe(u8, lm) else null, .headers = &.{}, + .vary_headers = &.{}, }; } diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 4d70866d..fef6a4b4 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -154,6 +154,7 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C const metadata = cache_file.metadata; + // Check entry expiration. const now = req.timestamp; const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store)); if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) { @@ -162,6 +163,28 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C return null; } + // If we have Vary headers, ensure they are present & matching. + for (metadata.vary_headers) |vary_hdr| { + const name = vary_hdr.name; + const value = vary_hdr.value; + + const incoming = for (req.request_headers) |h| { + if (std.ascii.eqlIgnoreCase(h.name, name)) break h.value; + } else ""; + + if (!std.ascii.eqlIgnoreCase(value, incoming)) { + log.debug(.cache, "vary mismatch", .{ .url = req.url, .header = name }); + return null; + } + } + + // On the case of a hash collision. + if (!std.ascii.eqlIgnoreCase(metadata.url, req.url)) { + log.warn(.cache, "collision", .{ .url = req.url, .expected = metadata.url, .got = req.url }); + cleanup = true; + return null; + } + return .{ .metadata = metadata, .data = .{ @@ -243,8 +266,8 @@ test "FsCache: basic put and get" { .etag = null, .last_modified = null, .cache_control = .{ .max_age = 600 }, - .vary = null, .headers = &.{}, + .vary_headers = &.{}, }; const body = "hello world"; @@ -252,7 +275,11 @@ test "FsCache: basic put and get" { const result = cache.get( arena.allocator(), - .{ .url = "https://example.com", .timestamp = now }, + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, ) orelse return error.CacheMiss; const f = result.data.file; const file = f.file; @@ -291,8 +318,8 @@ test "FsCache: get expiration" { .etag = null, .last_modified = null, .cache_control = .{ .max_age = max_age }, - .vary = null, .headers = &.{}, + .vary_headers = &.{}, }; const body = "hello world"; @@ -300,18 +327,30 @@ test "FsCache: get expiration" { const result = cache.get( arena.allocator(), - .{ .url = "https://example.com", .timestamp = now + 50 }, + .{ + .url = "https://example.com", + .timestamp = now + 50, + .request_headers = &.{}, + }, ) orelse return error.CacheMiss; result.data.file.file.close(); try testing.expectEqual(null, cache.get( arena.allocator(), - .{ .url = "https://example.com", .timestamp = now + 200 }, + .{ + .url = "https://example.com", + .timestamp = now + 200, + .request_headers = &.{}, + }, )); try testing.expectEqual(null, cache.get( arena.allocator(), - .{ .url = "https://example.com", .timestamp = now }, + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, )); } @@ -340,8 +379,8 @@ test "FsCache: put override" { .etag = null, .last_modified = null, .cache_control = .{ .max_age = max_age }, - .vary = null, .headers = &.{}, + .vary_headers = &.{}, }; const body = "hello world"; @@ -349,7 +388,11 @@ test "FsCache: put override" { const result = cache.get( arena.allocator(), - .{ .url = "https://example.com", .timestamp = now }, + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, ) orelse return error.CacheMiss; const f = result.data.file; const file = f.file; @@ -378,8 +421,8 @@ test "FsCache: put override" { .etag = null, .last_modified = null, .cache_control = .{ .max_age = max_age }, - .vary = null, .headers = &.{}, + .vary_headers = &.{}, }; const body = "goodbye world"; @@ -387,7 +430,11 @@ test "FsCache: put override" { const result = cache.get( arena.allocator(), - .{ .url = "https://example.com", .timestamp = now }, + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, ) orelse return error.CacheMiss; const f = result.data.file; const file = f.file; @@ -422,6 +469,124 @@ test "FsCache: garbage file" { try testing.expectEqual( null, - setup.cache.get(arena.allocator(), .{ .url = "https://example.com", .timestamp = 5000 }), + setup.cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = 5000, + .request_headers = &.{}, + }), ); } + +test "FsCache: vary hit and miss" { + var setup = try setupCache(); + defer { + setup.cache.deinit(); + setup.tmp.cleanup(); + } + + const cache = &setup.cache; + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .etag = null, + .last_modified = null, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + }, + }; + + try cache.put(meta, "hello world"); + + const result = cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + }, + }) orelse return error.CacheMiss; + result.data.file.file.close(); + + try testing.expectEqual(null, cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "br" }, + }, + })); + + try testing.expectEqual(null, cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + })); + + const result2 = cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + }, + }) orelse return error.CacheMiss; + result2.data.file.file.close(); +} + +test "FsCache: vary multiple headers" { + var setup = try setupCache(); + defer { + setup.cache.deinit(); + setup.tmp.cleanup(); + } + + const cache = &setup.cache; + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .etag = null, + .last_modified = null, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }; + + try cache.put(meta, "hello world"); + + const result = cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }) orelse return error.CacheMiss; + result.data.file.file.close(); + + try testing.expectEqual(null, cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "fr" }, + }, + })); +} diff --git a/src/network/http.zig b/src/network/http.zig index 2bfabac0..6dc217ea 100644 --- a/src/network/http.zig +++ b/src/network/http.zig @@ -79,7 +79,7 @@ pub const Headers = struct { self.headers = updated_headers; } - fn parseHeader(header_str: []const u8) ?Header { + pub fn parseHeader(header_str: []const u8) ?Header { const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null; const name = std.mem.trim(u8, header_str[0..colon_pos], " \t"); @@ -88,22 +88,9 @@ pub const Headers = struct { return .{ .name = name, .value = value }; } - pub fn iterator(self: *Headers) Iterator { - return .{ - .header = self.headers, - }; + pub fn iterator(self: Headers) HeaderIterator { + return .{ .curl_slist = .{ .header = self.headers } }; } - - const Iterator = struct { - header: [*c]libcurl.CurlSList, - - pub fn next(self: *Iterator) ?Header { - const h = self.header orelse return null; - - self.header = h.*.next; - return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data)))); - } - }; }; // In normal cases, the header iterator comes from the curl linked list. @@ -112,6 +99,7 @@ pub const Headers = struct { // This union, is an iterator that exposes the same API for either case. pub const HeaderIterator = union(enum) { curl: CurlHeaderIterator, + curl_slist: CurlSListIterator, list: ListHeaderIterator, pub fn next(self: *HeaderIterator) ?Header { @@ -120,6 +108,19 @@ pub const HeaderIterator = union(enum) { } } + pub fn collect(self: *HeaderIterator, allocator: std.mem.Allocator) !std.ArrayList(Header) { + var list: std.ArrayList(Header) = .empty; + + while (self.next()) |hdr| { + try list.append(allocator, .{ + .name = try allocator.dupe(u8, hdr.name), + .value = try allocator.dupe(u8, hdr.value), + }); + } + + return list; + } + const CurlHeaderIterator = struct { conn: *const Connection, prev: ?*libcurl.CurlHeader = null, @@ -136,6 +137,16 @@ pub const HeaderIterator = union(enum) { } }; + const CurlSListIterator = struct { + header: [*c]libcurl.CurlSList, + + pub fn next(self: *CurlSListIterator) ?Header { + const h = self.header orelse return null; + self.header = h.*.next; + return Headers.parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data)))); + } + }; + const ListHeaderIterator = struct { index: usize = 0, list: []const Header, From f098a991a83354b7080d3e0475f9ecfca7f2cce6 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 30 Mar 2026 15:24:17 -0700 Subject: [PATCH 35/48] remove cache revalidation stubs --- src/browser/HttpClient.zig | 2 -- src/network/cache/Cache.zig | 15 --------------- src/network/cache/FsCache.zig | 12 ------------ 3 files changed, 29 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index b16b6402..08fc60c5 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -1566,8 +1566,6 @@ pub const Transfer = struct { rh.contentType(), if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null, vary, - if (conn.getResponseHeader("etag", 0)) |h| h.value else null, - if (conn.getResponseHeader("last-modified", 0)) |h| h.value else null, if (conn.getResponseHeader("age", 0)) |h| h.value else null, conn.getResponseHeader("set-cookie", 0) != null, conn.getResponseHeader("authorization", 0) != null, diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index fd20e967..c941164f 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -48,8 +48,6 @@ pub fn put(self: *Cache, metadata: CachedMetadata, body: []const u8) !void { pub const CacheControl = struct { max_age: u64, - must_revalidate: bool = false, - immutable: bool = false, pub fn parse(value: []const u8) ?CacheControl { var cc: CacheControl = .{ .max_age = undefined }; @@ -65,10 +63,6 @@ pub const CacheControl = struct { return null; } else if (std.ascii.eqlIgnoreCase(directive, "no-cache")) { return null; - } else if (std.ascii.eqlIgnoreCase(directive, "must-revalidate")) { - cc.must_revalidate = true; - } else if (std.ascii.eqlIgnoreCase(directive, "immutable")) { - cc.immutable = true; } else if (std.ascii.eqlIgnoreCase(directive, "public")) { is_public = true; } else if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) { @@ -103,11 +97,6 @@ pub const CachedMetadata = struct { stored_at: i64, age_at_store: u64, - // for If-None-Match - etag: ?[]const u8, - // for If-Modified-Since - last_modified: ?[]const u8, - cache_control: CacheControl, /// Response Headers headers: []const Http.Header, @@ -144,8 +133,6 @@ pub fn tryCache( content_type: ?[]const u8, cache_control: ?[]const u8, vary: ?[]const u8, - etag: ?[]const u8, - last_modified: ?[]const u8, age: ?[]const u8, has_set_cookie: bool, has_authorization: bool, @@ -163,8 +150,6 @@ pub fn tryCache( .stored_at = timestamp, .age_at_store = if (age) |a| std.fmt.parseInt(u64, a, 10) catch 0 else 0, .cache_control = cc, - .etag = if (etag) |e| try arena.dupe(u8, e) else null, - .last_modified = if (last_modified) |lm| try arena.dupe(u8, lm) else null, .headers = &.{}, .vary_headers = &.{}, }; diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index fef6a4b4..685e5ba1 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -263,8 +263,6 @@ test "FsCache: basic put and get" { .status = 200, .stored_at = now, .age_at_store = 0, - .etag = null, - .last_modified = null, .cache_control = .{ .max_age = 600 }, .headers = &.{}, .vary_headers = &.{}, @@ -315,8 +313,6 @@ test "FsCache: get expiration" { .status = 200, .stored_at = now, .age_at_store = 900, - .etag = null, - .last_modified = null, .cache_control = .{ .max_age = max_age }, .headers = &.{}, .vary_headers = &.{}, @@ -376,8 +372,6 @@ test "FsCache: put override" { .status = 200, .stored_at = now, .age_at_store = 900, - .etag = null, - .last_modified = null, .cache_control = .{ .max_age = max_age }, .headers = &.{}, .vary_headers = &.{}, @@ -418,8 +412,6 @@ test "FsCache: put override" { .status = 200, .stored_at = now, .age_at_store = 0, - .etag = null, - .last_modified = null, .cache_control = .{ .max_age = max_age }, .headers = &.{}, .vary_headers = &.{}, @@ -496,8 +488,6 @@ test "FsCache: vary hit and miss" { .status = 200, .stored_at = now, .age_at_store = 0, - .etag = null, - .last_modified = null, .cache_control = .{ .max_age = 600 }, .headers = &.{}, .vary_headers = &.{ @@ -559,8 +549,6 @@ test "FsCache: vary multiple headers" { .status = 200, .stored_at = now, .age_at_store = 0, - .etag = null, - .last_modified = null, .cache_control = .{ .max_age = 600 }, .headers = &.{}, .vary_headers = &.{ From 0b9cae535486f9ed160a695816de8fb3d6107314 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Tue, 31 Mar 2026 12:11:33 -0700 Subject: [PATCH 36/48] fix self.req.ctx in HttpClient --- src/browser/HttpClient.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 08fc60c5..fac82d57 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -1298,7 +1298,7 @@ pub const Transfer = struct { if (execute_callback) { self.req.error_callback(self.req.ctx, err); } else if (self.req.shutdown_callback) |cb| { - cb(self.ctx); + cb(self.req.ctx); } } From 619a2653d14b5935d5fe84f1c79ec103fe27d4fe Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 1 Apr 2026 08:16:21 -0700 Subject: [PATCH 37/48] update cacheDir config option --- src/Config.zig | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Config.zig b/src/Config.zig index 9ad6e8bc..38c03b26 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -400,6 +400,11 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void { \\ \\--web-bot-auth-domain \\ Your domain e.g. yourdomain.com + \\ + \\--cache-dir + \\ Path to a directory to use as a Filesystem Cache for network resources. + \\ Omitting this will result is no caching. + \\ Defaults to no caching. ; // MAX_HELP_LEN| @@ -1074,9 +1079,9 @@ fn parseCommonArg( return true; } - if (std.mem.eql(u8, "--cache_dir", opt)) { + if (std.mem.eql(u8, "--cache-dir", opt)) { const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--cache_dir" }); + log.fatal(.app, "missing argument value", .{ .arg = "--cache-dir" }); return error.InvalidArgument; }; common.cache_dir = try allocator.dupe(u8, str); From a1a301666fcb333444282d1b523ddd8157a50d16 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 1 Apr 2026 08:20:01 -0700 Subject: [PATCH 38/48] dupe url in tryCache --- src/network/cache/Cache.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index c941164f..71e7d0bd 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -144,7 +144,7 @@ pub fn tryCache( const cc = CacheControl.parse(cache_control orelse return null) orelse return null; return .{ - .url = url, + .url = try arena.dupeZ(u8, url), .content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream", .status = status, .stored_at = timestamp, From 31bab4cc051acb0273191f7dcce66037732fc4d2 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 1 Apr 2026 08:52:26 -0700 Subject: [PATCH 39/48] put in cache before releasing conn --- src/browser/HttpClient.zig | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index fac82d57..b188241a 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -970,25 +970,22 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T } } - // release conn ASAP so that it's available; some done_callbacks - // will load more resources. - transfer.releaseConn(); - - try transfer.req.done_callback(transfer.req.ctx); - if (transfer._pending_cache_metadata) |metadata| { const cache = &self.network.cache.?; + log.debug(.browser, "http cache", .{ .url = transfer.req.url, .metadata = metadata }); - // TODO: Support Vary Keying - const cache_key = transfer.req.url; - - log.debug(.browser, "http cache", .{ .key = cache_key, .metadata = metadata }); cache.put(metadata, body) catch |err| { log.warn(.http, "cache put failed", .{ .err = err }); }; log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); } + // release conn ASAP so that it's available; some done_callbacks + // will load more resources. + transfer.releaseConn(); + + try transfer.req.done_callback(transfer.req.ctx); + transfer.req.notification.dispatch(.http_request_done, &.{ .transfer = transfer, }); From 1e8bdd7e2853c4dcc0b9f60ccaa8c2cf21a45516 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 1 Apr 2026 08:52:47 -0700 Subject: [PATCH 40/48] assign headers and vary headers before possible move --- src/browser/HttpClient.zig | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index b188241a..67fdc1c9 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -1569,12 +1569,9 @@ pub const Transfer = struct { ); if (maybe_cm) |cm| { - transfer._pending_cache_metadata = cm; - var iter = transfer.responseHeaderIterator(); var header_list = try iter.collect(allocator); const end_of_response = header_list.items.len; - transfer._pending_cache_metadata.?.headers = header_list.items[0..end_of_response]; if (vary) |vary_str| { var req_it = transfer.req.headers.iterator(); @@ -1592,9 +1589,11 @@ pub const Transfer = struct { } } } - - transfer._pending_cache_metadata.?.vary_headers = header_list.items[end_of_response..]; } + + transfer._pending_cache_metadata = cm; + transfer._pending_cache_metadata.?.headers = header_list.items[0..end_of_response]; + transfer._pending_cache_metadata.?.vary_headers = header_list.items[end_of_response..]; } } From 3d760e457792be3153309868b56bd156a75feb4d Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 1 Apr 2026 08:52:55 -0700 Subject: [PATCH 41/48] add format to CachedMetadata --- src/network/cache/Cache.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 71e7d0bd..3a58fa22 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -103,6 +103,26 @@ pub const CachedMetadata = struct { /// These are Request Headers used by Vary. vary_headers: []const Http.Header, + + pub fn format(self: CachedMetadata, writer: *std.Io.Writer) !void { + try writer.print("url={s} | status={d} | content_type={s} | max_age={d} | vary=[", .{ + self.url, + self.status, + self.content_type, + self.cache_control.max_age, + }); + + // Logging all headers gets pretty verbose... + // so we just log the Vary ones that matter for caching. + + if (self.vary_headers.len > 0) { + for (self.vary_headers, 0..) |hdr, i| { + if (i > 0) try writer.print(", ", .{}); + try writer.print("{s}: {s}", .{ hdr.name, hdr.value }); + } + } + try writer.print("]", .{}); + } }; pub const CacheRequest = struct { From e00d5697541c8a714bc41931ba84436b482121b3 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 1 Apr 2026 14:12:25 -0700 Subject: [PATCH 42/48] fix crashes on cached file from script manager --- src/browser/ScriptManager.zig | 42 +++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 4bd4148c..c7901e9e 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -273,23 +273,31 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e // Let the outer errdefer handle releasing the arena if client.request fails } - try self.client.request(.{ - .url = url, - .ctx = script, - .method = .GET, - .frame_id = page._frame_id, - .headers = try self.getHeaders(), - .blocking = is_blocking, - .cookie_jar = &page._session.cookie_jar, - .cookie_origin = page.url, - .resource_type = .script, - .notification = page._session.notification, - .start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null, - .header_callback = Script.headerCallback, - .data_callback = Script.dataCallback, - .done_callback = Script.doneCallback, - .error_callback = Script.errorCallback, - }); + // If we return synchronously (like from cache), we would call evaluate() immediately. + { + const was_evaluating = self.is_evaluating; + self.is_evaluating = true; + defer self.is_evaluating = was_evaluating; + + try self.client.request(.{ + .url = url, + .ctx = script, + .method = .GET, + .frame_id = page._frame_id, + .headers = try self.getHeaders(), + .blocking = is_blocking, + .cookie_jar = &page._session.cookie_jar, + .cookie_origin = page.url, + .resource_type = .script, + .notification = page._session.notification, + .start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null, + .header_callback = Script.headerCallback, + .data_callback = Script.dataCallback, + .done_callback = Script.doneCallback, + .error_callback = Script.errorCallback, + }); + } + handover = true; if (comptime IS_DEBUG) { From dc600c953f8267e5a13d55275ef44c64ac819429 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 2 Apr 2026 03:26:02 -0700 Subject: [PATCH 43/48] move script queue log before request --- src/browser/ScriptManager.zig | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index c7901e9e..546a05c5 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -273,7 +273,19 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e // Let the outer errdefer handle releasing the arena if client.request fails } - // If we return synchronously (like from cache), we would call evaluate() immediately. + if (comptime IS_DEBUG) { + var ls: js.Local.Scope = undefined; + page.js.localScope(&ls); + defer ls.deinit(); + + log.debug(.http, "script queue", .{ + .ctx = ctx, + .url = remote_url.?, + .element = element, + .stack = ls.local.stackTrace() catch "???", + }); + } + { const was_evaluating = self.is_evaluating; self.is_evaluating = true; @@ -299,19 +311,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e } handover = true; - - if (comptime IS_DEBUG) { - var ls: js.Local.Scope = undefined; - page.js.localScope(&ls); - defer ls.deinit(); - - log.debug(.http, "script queue", .{ - .ctx = ctx, - .url = remote_url.?, - .element = element, - .stack = ls.local.stackTrace() catch "???", - }); - } } if (is_blocking == false) { From 13ea4d1ee32ceb153d69a82adf88e9d61d624073 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 2 Apr 2026 13:21:12 -0700 Subject: [PATCH 44/48] more expressive cache logging --- src/browser/HttpClient.zig | 20 +++++------- src/network/cache/Cache.zig | 14 +++++++++ src/network/cache/FsCache.zig | 58 +++++++++++++++++++++++++++-------- 3 files changed, 67 insertions(+), 25 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 67fdc1c9..36620982 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -375,16 +375,8 @@ fn processRequest(self: *Client, req: Request) !void { .timestamp = std.time.timestamp(), .request_headers = req_header_list.items, })) |cached| { - log.debug(.browser, "http.cache.get", .{ - .url = req.url, - .found = true, - .metadata = cached.metadata, - }); - defer req.headers.deinit(); return serveFromCache(req, &cached); - } else { - log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = false }); } } } @@ -972,12 +964,9 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T if (transfer._pending_cache_metadata) |metadata| { const cache = &self.network.cache.?; - log.debug(.browser, "http cache", .{ .url = transfer.req.url, .metadata = metadata }); - cache.put(metadata, body) catch |err| { - log.warn(.http, "cache put failed", .{ .err = err }); + log.warn(.cache, "cache put failed", .{ .err = err }); }; - log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url }); } // release conn ASAP so that it's available; some done_callbacks @@ -1157,6 +1146,13 @@ pub const Response = struct { .cached => {}, } } + + pub fn format(self: Response, writer: *std.Io.Writer) !void { + return switch (self.inner) { + .transfer => |t| try t.format(writer), + .cached => |c| try c.format(writer), + }; + } }; pub const Transfer = struct { diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 3a58fa22..f8b2a9d4 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -138,11 +138,25 @@ pub const CachedData = union(enum) { offset: usize, len: usize, }, + + pub fn format(self: CachedData, writer: *std.Io.Writer) !void { + switch (self) { + .buffer => |buf| try writer.print("buffer({d} bytes)", .{buf.len}), + .file => |f| try writer.print("file(offset={d}, len={d} bytes)", .{ f.offset, f.len }), + } + } }; pub const CachedResponse = struct { metadata: CachedMetadata, data: CachedData, + + pub fn format(self: *const CachedResponse, writer: *std.Io.Writer) !void { + try writer.print("metadata=(", .{}); + try self.metadata.format(writer); + try writer.print("), data=", .{}); + try self.data.format(writer); + } }; pub fn tryCache( diff --git a/src/network/cache/FsCache.zig b/src/network/cache/FsCache.zig index 685e5ba1..3d67a945 100644 --- a/src/network/cache/FsCache.zig +++ b/src/network/cache/FsCache.zig @@ -97,7 +97,7 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C const file = self.dir.openFile(&cache_p, .{ .mode = .read_only }) catch |e| { switch (e) { std.fs.File.OpenError.FileNotFound => { - log.debug(.cache, "miss", .{ .url = req.url, .hash = &hashed_key }); + log.debug(.cache, "miss", .{ .url = req.url, .hash = &hashed_key, .reason = "missing" }); }, else => |err| { log.warn(.cache, "open file err", .{ .url = req.url, .err = err }); @@ -141,13 +141,19 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C &json_reader, .{ .allocate = .alloc_always }, ) catch |e| { - log.warn(.cache, "metadata parse", .{ .url = req.url, .err = e }); + // Warn because malformed metadata can be a deeper symptom. + log.warn(.cache, "miss", .{ .url = req.url, .err = e, .reason = "malformed metadata" }); cleanup = true; return null; }; if (cache_file.version != CACHE_VERSION) { - log.warn(.cache, "version", .{ .url = req.url, .expected = CACHE_VERSION, .got = cache_file.version }); + log.debug(.cache, "miss", .{ + .url = req.url, + .reason = "version mismatch", + .expected = CACHE_VERSION, + .got = cache_file.version, + }); cleanup = true; return null; } @@ -158,7 +164,7 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C const now = req.timestamp; const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store)); if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) { - log.debug(.cache, "expired", .{ .url = req.url }); + log.debug(.cache, "miss", .{ .url = req.url, .reason = "expired" }); cleanup = true; return null; } @@ -173,7 +179,13 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C } else ""; if (!std.ascii.eqlIgnoreCase(value, incoming)) { - log.debug(.cache, "vary mismatch", .{ .url = req.url, .header = name }); + log.debug(.cache, "miss", .{ + .url = req.url, + .reason = "vary mismatch", + .header = name, + .expected = value, + .got = incoming, + }); return null; } } @@ -185,6 +197,8 @@ pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.C return null; } + log.debug(.cache, "hit", .{ .url = req.url, .hash = &hashed_key }); + return .{ .metadata = metadata, .data = .{ @@ -206,27 +220,45 @@ pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void { lock.lock(); defer lock.unlock(); - const file = try self.dir.createFile(&cache_tmp_p, .{ .truncate = true }); + const file = self.dir.createFile(&cache_tmp_p, .{ .truncate = true }) catch |e| { + log.err(.cache, "create file", .{ .url = meta.url, .file = &cache_tmp_p, .err = e }); + return e; + }; defer file.close(); var writer_buf: [1024]u8 = undefined; - var file_writer = file.writer(&writer_buf); var file_writer_iface = &file_writer.interface; var len_buf: [8]u8 = undefined; std.mem.writeInt(u64, &len_buf, body.len, .little); - try file_writer_iface.writeAll(&len_buf); - try file_writer_iface.writeAll(body); - try std.json.Stringify.value( + file_writer_iface.writeAll(&len_buf) catch |e| { + log.err(.cache, "write body len", .{ .url = meta.url, .err = e }); + return e; + }; + file_writer_iface.writeAll(body) catch |e| { + log.err(.cache, "write body", .{ .url = meta.url, .err = e }); + return e; + }; + std.json.Stringify.value( CacheMetadataJson{ .version = CACHE_VERSION, .metadata = meta }, .{ .whitespace = .minified }, file_writer_iface, - ); + ) catch |e| { + log.err(.cache, "write metadata", .{ .url = meta.url, .err = e }); + return e; + }; + file_writer_iface.flush() catch |e| { + log.err(.cache, "flush", .{ .url = meta.url, .err = e }); + return e; + }; + self.dir.rename(&cache_tmp_p, &cache_p) catch |e| { + log.err(.cache, "rename", .{ .url = meta.url, .from = &cache_tmp_p, .to = &cache_p, .err = e }); + return e; + }; - try file_writer_iface.flush(); - try self.dir.rename(&cache_tmp_p, &cache_p); + log.debug(.cache, "put", .{ .url = meta.url, .hash = &hashed_key, .body_len = body.len }); } const testing = std.testing; From 5a551607c2632f6de610c0a967f9d88217673197 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 2 Apr 2026 13:32:02 -0700 Subject: [PATCH 45/48] better logging on FsCache init failure --- src/network/Network.zig | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/network/Network.zig b/src/network/Network.zig index 0cfc7a20..abb64fce 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -17,6 +17,7 @@ // along with this program. If not, see . const std = @import("std"); +const log = @import("../log.zig"); const builtin = @import("builtin"); const net = std.net; const posix = std.posix; @@ -29,7 +30,9 @@ const libcurl = @import("../sys/libcurl.zig"); const http = @import("http.zig"); const RobotStore = @import("Robots.zig").RobotStore; const WebBotAuth = @import("WebBotAuth.zig"); + const Cache = @import("cache/Cache.zig"); +const FsCache = @import("cache/FsCache.zig"); const App = @import("../App.zig"); const Network = @This(); @@ -238,7 +241,18 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { null; const cache = if (config.cacheDir()) |cache_dir_path| - Cache{ .kind = .{ .fs = try .init(cache_dir_path) } } + Cache{ + .kind = .{ + .fs = FsCache.init(cache_dir_path) catch |e| { + log.err(.cache, "failed to init", .{ + .kind = "FsCache", + .path = cache_dir_path, + .err = e, + }); + return e; + }, + }, + } else null; From a71ff521aae3ad606718e5aeefc99247e5c894fc Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Fri, 3 Apr 2026 09:56:39 +0200 Subject: [PATCH 46/48] cache: add debug log with no store reason --- src/network/cache/Cache.zig | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index f8b2a9d4..d270310e 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -17,6 +17,7 @@ // along with this program. If not, see . const std = @import("std"); +const log = @import("../../log.zig"); const Http = @import("../http.zig"); const FsCache = @import("FsCache.zig"); @@ -171,11 +172,33 @@ pub fn tryCache( has_set_cookie: bool, has_authorization: bool, ) !?CachedMetadata { - if (status != 200) return null; - if (has_set_cookie) return null; - if (has_authorization) return null; - if (vary) |v| if (std.mem.eql(u8, v, "*")) return null; - const cc = CacheControl.parse(cache_control orelse return null) orelse return null; + if (status != 200) { + log.debug(.cache, "no store", .{ .url = url, .code = status, .reason = "status" }); + return null; + } + if (has_set_cookie) { + log.debug(.cache, "no store", .{ .url = url, .reason = "has_cookies" }); + return null; + } + if (has_authorization) { + log.debug(.cache, "no store", .{ .url = url, .reason = "has_authorization" }); + return null; + } + if (vary) |v| if (std.mem.eql(u8, v, "*")) { + log.debug(.cache, "no store", .{ .url = url, .vary = v, .reason = "vary" }); + return null; + }; + const cc = blk: { + if (cache_control == null) { + log.debug(.cache, "no store", .{ .url = url, .reason = "no cache control" }); + return null; + } + if (CacheControl.parse(cache_control.?)) |cc| { + break :blk cc; + } + log.debug(.cache, "no store", .{ .url = url, .cache_control = cache_control.?, .reason = "cache control" }); + return null; + }; return .{ .url = try arena.dupeZ(u8, url), From ca5fa2b866d7beaeae1108970d50c88ee8e34c07 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Fri, 3 Apr 2026 07:21:17 -0700 Subject: [PATCH 47/48] change --cache-dir -> --http-cache-dir --- src/Config.zig | 14 +++++++------- src/network/Network.zig | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Config.zig b/src/Config.zig index 38c03b26..6788db1d 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -156,9 +156,9 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 { }; } -pub fn cacheDir(self: *const Config) ?[]const u8 { +pub fn httpCacheDir(self: *const Config) ?[]const u8 { return switch (self.mode) { - inline .serve, .fetch, .mcp => |opts| opts.common.cache_dir, + inline .serve, .fetch, .mcp => |opts| opts.common.http_cache_dir, else => null, }; } @@ -280,7 +280,7 @@ pub const Common = struct { log_format: ?log.Format = null, log_filter_scopes: ?[]log.Scope = null, user_agent_suffix: ?[]const u8 = null, - cache_dir: ?[]const u8 = null, + http_cache_dir: ?[]const u8 = null, web_bot_auth_key_file: ?[]const u8 = null, web_bot_auth_keyid: ?[]const u8 = null, @@ -401,7 +401,7 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void { \\--web-bot-auth-domain \\ Your domain e.g. yourdomain.com \\ - \\--cache-dir + \\--http-cache-dir \\ Path to a directory to use as a Filesystem Cache for network resources. \\ Omitting this will result is no caching. \\ Defaults to no caching. @@ -1079,12 +1079,12 @@ fn parseCommonArg( return true; } - if (std.mem.eql(u8, "--cache-dir", opt)) { + if (std.mem.eql(u8, "--http-cache-dir", opt)) { const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--cache-dir" }); + log.fatal(.app, "missing argument value", .{ .arg = "--http-cache-dir" }); return error.InvalidArgument; }; - common.cache_dir = try allocator.dupe(u8, str); + common.http_cache_dir = try allocator.dupe(u8, str); return true; } diff --git a/src/network/Network.zig b/src/network/Network.zig index abb64fce..ab11e5ce 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -240,7 +240,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { else null; - const cache = if (config.cacheDir()) |cache_dir_path| + const cache = if (config.httpCacheDir()) |cache_dir_path| Cache{ .kind = .{ .fs = FsCache.init(cache_dir_path) catch |e| { From 778b7eb8c251ec0a38ae18f18231741d346265ea Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Fri, 3 Apr 2026 07:34:31 -0700 Subject: [PATCH 48/48] allocate CacheMetadata on use --- src/browser/HttpClient.zig | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 36620982..6247e323 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -964,7 +964,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T if (transfer._pending_cache_metadata) |metadata| { const cache = &self.network.cache.?; - cache.put(metadata, body) catch |err| { + cache.put(metadata.*, body) catch |err| { log.warn(.cache, "cache put failed", .{ .err = err }); }; } @@ -1164,7 +1164,7 @@ pub const Transfer = struct { // total bytes received in the response, including the response status line, // the headers, and the [encoded] body. bytes_received: usize = 0, - _pending_cache_metadata: ?CacheMetadata = null, + _pending_cache_metadata: ?*CacheMetadata = null, aborted: bool = false, @@ -1587,9 +1587,11 @@ pub const Transfer = struct { } } - transfer._pending_cache_metadata = cm; - transfer._pending_cache_metadata.?.headers = header_list.items[0..end_of_response]; - transfer._pending_cache_metadata.?.vary_headers = header_list.items[end_of_response..]; + const metadata = try transfer.arena.allocator().create(CacheMetadata); + metadata.* = cm; + metadata.headers = header_list.items[0..end_of_response]; + metadata.vary_headers = header_list.items[end_of_response..]; + transfer._pending_cache_metadata = metadata; } }