29 Commits

Author SHA1 Message Date
Muki Kiboigo
c9b8632b50 require timestamp passed in with cache request 2026-03-27 09:53:18 -07:00
Muki Kiboigo
41f154b042 remove unused cache method on fs cache 2026-03-27 09:53:18 -07:00
Muki Kiboigo
666b1d1016 add basic fs cache get/put test 2026-03-27 09:53:17 -07:00
Muki Kiboigo
1559346a67 check age on fs cache get 2026-03-27 09:53:17 -07:00
Muki Kiboigo
0c97ff2cab ensure capacity of pending cache body 2026-03-27 09:53:17 -07:00
Muki Kiboigo
2a085dc0c2 only store stuff when we know we will cache 2026-03-27 09:53:17 -07:00
Muki Kiboigo
26ba3b52ca shortcircuit a lot of caching checks 2026-03-27 09:53:17 -07:00
Muki Kiboigo
076c1942a4 ensure fs cache file is closed after use 2026-03-27 09:53:17 -07:00
Muki Kiboigo
def6e564d2 use CacheRequest instead of key 2026-03-27 09:53:17 -07:00
Muki Kiboigo
368ac00573 add striped lock to FsCache 2026-03-27 09:53:16 -07:00
Muki Kiboigo
756373e0ba use arena_pool for cache get 2026-03-27 09:53:16 -07:00
Muki Kiboigo
e596e0c310 use writer for fs cache body file 2026-03-27 09:53:16 -07:00
Muki Kiboigo
4ccf150ed0 use json for fs cache metadata file 2026-03-27 09:53:16 -07:00
Muki Kiboigo
dcb08b97ad use sha256 instead of wyhash 2026-03-27 09:53:16 -07:00
Muki Kiboigo
c2afb2fb17 store type_buf and sub_type_buf in Mime 2026-03-27 09:53:16 -07:00
Muki Kiboigo
8a9007b939 add more http caching rules 2026-03-27 09:53:16 -07:00
Muki Kiboigo
cd31e68212 use CacheControl and Vary 2026-03-27 09:53:15 -07:00
Muki Kiboigo
46d25652e0 cache headers along with response 2026-03-27 09:53:15 -07:00
Muki Kiboigo
6972a3e130 add basic caching support 2026-03-27 09:53:15 -07:00
Muki Kiboigo
8f13ace8a2 create cache owned by the network struct 2026-03-27 09:53:15 -07:00
Muki Kiboigo
31921e4890 use enum approach instead of vtable 2026-03-27 09:53:15 -07:00
Muki Kiboigo
3abf026f77 add basic FsCache impl 2026-03-27 09:53:15 -07:00
Muki Kiboigo
3f0b17c755 add CachedResponse variant to Response 2026-03-27 09:53:14 -07:00
Muki Kiboigo
a97ce89362 add headerIterator to Http Response 2026-03-27 09:52:17 -07:00
Muki Kiboigo
9da1b96a14 use Response instead of Transfer in callbacks 2026-03-27 09:52:11 -07:00
Muki Kiboigo
6bd5f1d013 allow Mime parse to use []const u8 2026-03-27 09:22:40 -07:00
Muki Kiboigo
3f2a4ada53 add cache dir to configuration opts 2026-03-27 09:22:40 -07:00
Muki Kiboigo
edd2c661f4 create Cache interface file 2026-03-27 09:22:39 -07:00
Karl Seguin
9e902d176e Add adoptedStyleSheets property to ShadowRoot, just like Document
Used in github.
2026-03-27 09:22:39 -07:00
14 changed files with 903 additions and 161 deletions

View File

@@ -55,7 +55,7 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
.arena_pool = undefined, .arena_pool = undefined,
}; };
app.network = try Network.init(allocator, config); app.network = try Network.init(allocator, app, config);
errdefer app.network.deinit(); errdefer app.network.deinit();
app.platform = try Platform.init(); app.platform = try Platform.init();

View File

@@ -156,6 +156,13 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
}; };
} }
pub fn cacheDir(self: *const Config) ?[]const u8 {
return switch (self.mode) {
inline .serve, .fetch, .mcp => |opts| opts.common.cache_dir,
else => null,
};
}
pub fn cdpTimeout(self: *const Config) usize { pub fn cdpTimeout(self: *const Config) usize {
return switch (self.mode) { return switch (self.mode) {
.serve => |opts| if (opts.timeout > 604_800) 604_800_000 else @as(usize, opts.timeout) * 1000, .serve => |opts| if (opts.timeout > 604_800) 604_800_000 else @as(usize, opts.timeout) * 1000,
@@ -240,6 +247,7 @@ pub const Common = struct {
log_format: ?log.Format = null, log_format: ?log.Format = null,
log_filter_scopes: ?[]log.Scope = null, log_filter_scopes: ?[]log.Scope = null,
user_agent_suffix: ?[]const u8 = null, user_agent_suffix: ?[]const u8 = null,
cache_dir: ?[]const u8 = null,
web_bot_auth_key_file: ?[]const u8 = null, web_bot_auth_key_file: ?[]const u8 = null,
web_bot_auth_keyid: ?[]const u8 = null, web_bot_auth_keyid: ?[]const u8 = null,
@@ -907,5 +915,14 @@ fn parseCommonArg(
return true; return true;
} }
if (std.mem.eql(u8, "--cache_dir", opt)) {
const str = args.next() orelse {
log.fatal(.app, "missing argument value", .{ .arg = "--cache_dir" });
return error.InvalidArgument;
};
common.cache_dir = try allocator.dupe(u8, str);
return true;
}
return false; return false;
} }

View File

@@ -32,6 +32,10 @@ const Robots = @import("../network/Robots.zig");
const RobotStore = Robots.RobotStore; const RobotStore = Robots.RobotStore;
const WebBotAuth = @import("../network/WebBotAuth.zig"); const WebBotAuth = @import("../network/WebBotAuth.zig");
const Cache = @import("../network/cache/Cache.zig");
const CacheMetadata = Cache.CachedMetadata;
const CachedResponse = Cache.CachedResponse;
const Allocator = std.mem.Allocator; const Allocator = std.mem.Allocator;
const ArenaAllocator = std.heap.ArenaAllocator; const ArenaAllocator = std.heap.ArenaAllocator;
@@ -289,7 +293,69 @@ pub fn request(self: *Client, req: Request) !void {
return self.fetchRobotsThenProcessRequest(robots_url, req); return self.fetchRobotsThenProcessRequest(robots_url, req);
} }
fn serveFromCache(req: Request, cached: *const CachedResponse) !void {
const response = Response.fromCached(req.ctx, cached);
if (req.start_callback) |cb| {
try cb(response);
}
const proceed = try req.header_callback(response);
if (!proceed) {
switch (cached.data) {
.buffer => |_| {},
.file => |file| file.close(),
}
req.error_callback(req.ctx, error.Abort);
return;
}
switch (cached.data) {
.buffer => |data| {
if (data.len > 0) {
try req.data_callback(response, data);
}
},
.file => |file| {
defer file.close();
var buf: [1024]u8 = undefined;
var file_reader = file.reader(&buf);
const reader = &file_reader.interface;
var read_buf: [1024]u8 = undefined;
while (true) {
const curr = try reader.readSliceShort(&read_buf);
if (curr == 0) break;
try req.data_callback(response, read_buf[0..curr]);
}
},
}
try req.done_callback(req.ctx);
}
fn processRequest(self: *Client, req: Request) !void { fn processRequest(self: *Client, req: Request) !void {
if (self.network.cache) |*cache| {
if (req.method == .GET) {
const arena = try self.network.app.arena_pool.acquire();
defer self.network.app.arena_pool.release(arena);
if (cache.get(arena, .{ .url = req.url, .timestamp = std.time.timestamp() })) |cached| {
log.debug(.browser, "http.cache.get", .{
.url = req.url,
.found = true,
.metadata = cached.metadata,
});
defer req.headers.deinit();
return serveFromCache(req, &cached);
} else {
log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = false });
}
}
}
const transfer = try self.makeTransfer(req); const transfer = try self.makeTransfer(req);
transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer }); transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
@@ -376,8 +442,10 @@ fn fetchRobotsThenProcessRequest(self: *Client, robots_url: [:0]const u8, req: R
try entry.value_ptr.append(self.allocator, req); try entry.value_ptr.append(self.allocator, req);
} }
fn robotsHeaderCallback(transfer: *Transfer) !bool { fn robotsHeaderCallback(response: Response) !bool {
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx)); const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx));
// Robots callbacks only happen on real live requests.
const transfer = response.inner.live;
if (transfer.response_header) |hdr| { if (transfer.response_header) |hdr| {
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = ctx.robots_url }); log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = ctx.robots_url });
@@ -391,8 +459,8 @@ fn robotsHeaderCallback(transfer: *Transfer) !bool {
return true; return true;
} }
fn robotsDataCallback(transfer: *Transfer, data: []const u8) !void { fn robotsDataCallback(response: Response, data: []const u8) !void {
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx)); const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx));
try ctx.buffer.appendSlice(ctx.client.allocator, data); try ctx.buffer.appendSlice(ctx.client.allocator, data);
} }
@@ -611,7 +679,6 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer {
.id = id, .id = id,
.url = req.url, .url = req.url,
.req = req, .req = req,
.ctx = req.ctx,
.client = self, .client = self,
.max_response_size = self.network.config.httpMaxResponseSize(), .max_response_size = self.network.config.httpMaxResponseSize(),
}; };
@@ -634,9 +701,9 @@ fn requestFailed(transfer: *Transfer, err: anyerror, comptime execute_callback:
}); });
if (execute_callback) { if (execute_callback) {
transfer.req.error_callback(transfer.ctx, err); transfer.req.error_callback(transfer.req.ctx, err);
} else if (transfer.req.shutdown_callback) |cb| { } else if (transfer.req.shutdown_callback) |cb| {
cb(transfer.ctx); cb(transfer.req.ctx);
} }
} }
@@ -743,7 +810,7 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
}; };
if (req.start_callback) |cb| { if (req.start_callback) |cb| {
cb(transfer) catch |err| { cb(Response.fromLive(transfer)) catch |err| {
transfer.deinit(); transfer.deinit();
return err; return err;
}; };
@@ -875,18 +942,31 @@ fn processMessages(self: *Client) !bool {
break :blk; break :blk;
} }
} }
transfer.req.done_callback(transfer.ctx) catch |err| { transfer.req.done_callback(transfer.req.ctx) catch |err| {
// transfer isn't valid at this point, don't use it. // transfer isn't valid at this point, don't use it.
log.err(.http, "done_callback", .{ .err = err }); log.err(.http, "done_callback", .{ .err = err });
requestFailed(transfer, err, true); requestFailed(transfer, err, true);
continue; continue;
}; };
transfer.req.notification.dispatch(.http_request_done, &.{ if (transfer.pending_cache_metadata) |metadata| {
.transfer = transfer, const cache = &self.network.cache.?;
});
processed = true; // TODO: Support Vary Keying
const cache_key = transfer.req.url;
log.debug(.browser, "http cache", .{ .key = cache_key, .metadata = metadata });
cache.put(metadata, transfer.pending_cache_body.items) catch |err| {
log.warn(.http, "cache put failed", .{ .err = err });
};
log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url });
}
} }
transfer.req.notification.dispatch(.http_request_done, &.{
.transfer = transfer,
});
processed = true;
} }
return processed; return processed;
} }
@@ -962,9 +1042,9 @@ pub const Request = struct {
// arbitrary data that can be associated with this request // arbitrary data that can be associated with this request
ctx: *anyopaque = undefined, ctx: *anyopaque = undefined,
start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null, start_callback: ?*const fn (response: Response) anyerror!void = null,
header_callback: *const fn (transfer: *Transfer) anyerror!bool, header_callback: *const fn (response: Response) anyerror!bool,
data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void, data_callback: *const fn (response: Response, data: []const u8) anyerror!void,
done_callback: *const fn (ctx: *anyopaque) anyerror!void, done_callback: *const fn (ctx: *anyopaque) anyerror!void,
error_callback: *const fn (ctx: *anyopaque, err: anyerror) void, error_callback: *const fn (ctx: *anyopaque, err: anyerror) void,
shutdown_callback: ?*const fn (ctx: *anyopaque) void = null, shutdown_callback: ?*const fn (ctx: *anyopaque) void = null,
@@ -992,16 +1072,92 @@ pub const Request = struct {
const AuthChallenge = Net.AuthChallenge; const AuthChallenge = Net.AuthChallenge;
pub const Response = struct {
ctx: *anyopaque,
inner: union(enum) {
live: *Transfer,
cached: *const CachedResponse,
},
pub fn fromLive(transfer: *Transfer) Response {
return .{ .ctx = transfer.req.ctx, .inner = .{ .live = transfer } };
}
pub fn fromCached(ctx: *anyopaque, resp: *const CachedResponse) Response {
return .{ .ctx = ctx, .inner = .{ .cached = resp } };
}
pub fn status(self: Response) ?u16 {
return switch (self.inner) {
.live => |live| if (live.response_header) |rh| rh.status else null,
.cached => |c| c.metadata.status,
};
}
pub fn contentType(self: Response) ?[]const u8 {
return switch (self.inner) {
.live => |live| if (live.response_header) |*rh| rh.contentType() else null,
.cached => |c| c.metadata.content_type,
};
}
pub fn contentLength(self: Response) ?u32 {
return switch (self.inner) {
.live => |live| live.getContentLength(),
.cached => |c| switch (c.data) {
.buffer => |buf| @intCast(buf.len),
.file => |f| @intCast(f.getEndPos() catch 0),
},
};
}
pub fn redirectCount(self: Response) ?u32 {
return switch (self.inner) {
.live => |live| if (live.response_header) |rh| rh.redirect_count else null,
.cached => 0,
};
}
pub fn url(self: Response) [:0]const u8 {
return switch (self.inner) {
.live => |live| live.url,
.cached => |c| c.metadata.url,
};
}
pub fn headerIterator(self: Response) HeaderIterator {
return switch (self.inner) {
.live => |live| live.responseHeaderIterator(),
.cached => |c| HeaderIterator{ .list = .{ .list = c.metadata.headers } },
};
}
pub fn abort(self: Response, err: anyerror) void {
switch (self.inner) {
.live => |live| live.abort(err),
.cached => {},
}
}
pub fn terminate(self: Response) void {
switch (self.inner) {
.live => |live| live.terminate(),
.cached => {},
}
}
};
pub const Transfer = struct { pub const Transfer = struct {
arena: ArenaAllocator, arena: ArenaAllocator,
id: u32 = 0, id: u32 = 0,
req: Request, req: Request,
url: [:0]const u8, url: [:0]const u8,
ctx: *anyopaque, // copied from req.ctx to make it easier for callback handlers
client: *Client, client: *Client,
// total bytes received in the response, including the response status line, // total bytes received in the response, including the response status line,
// the headers, and the [encoded] body. // the headers, and the [encoded] body.
bytes_received: usize = 0, bytes_received: usize = 0,
pending_cache_body: std.ArrayList(u8) = .empty,
pending_cache_metadata: ?CacheMetadata = null,
aborted: bool = false, aborted: bool = false,
@@ -1053,6 +1209,8 @@ pub const Transfer = struct {
self._notified_fail = false; self._notified_fail = false;
self.response_header = null; self.response_header = null;
self.bytes_received = 0; self.bytes_received = 0;
self.pending_cache_metadata = null;
self.pending_cache_body = .empty;
self._tries += 1; self._tries += 1;
} }
@@ -1161,7 +1319,7 @@ pub const Transfer = struct {
self.client.endTransfer(self); self.client.endTransfer(self);
} }
if (self.req.shutdown_callback) |cb| { if (self.req.shutdown_callback) |cb| {
cb(self.ctx); cb(self.req.ctx);
} }
self.deinit(); self.deinit();
} }
@@ -1263,11 +1421,48 @@ pub const Transfer = struct {
} }
} }
const proceed = transfer.req.header_callback(transfer) catch |err| { const proceed = transfer.req.header_callback(Response.fromLive(transfer)) catch |err| {
log.err(.http, "header_callback", .{ .err = err, .req = transfer }); log.err(.http, "header_callback", .{ .err = err, .req = transfer });
return err; return err;
}; };
if (transfer.client.network.cache != null and transfer.req.method == .GET) {
const rh = &transfer.response_header.?;
const allocator = transfer.arena.allocator();
const maybe_cm = try Cache.tryCache(
allocator,
std.time.timestamp(),
transfer.url,
rh.status,
rh.contentType(),
if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,
if (conn.getResponseHeader("vary", 0)) |h| h.value else null,
if (conn.getResponseHeader("etag", 0)) |h| h.value else null,
if (conn.getResponseHeader("last-modified", 0)) |h| h.value else null,
if (conn.getResponseHeader("age", 0)) |h| h.value else null,
conn.getResponseHeader("set-cookie", 0) != null,
conn.getResponseHeader("authorization", 0) != null,
);
if (maybe_cm) |cm| {
var header_list: std.ArrayList(Net.Header) = .empty;
var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
try header_list.append(allocator, .{
.name = try allocator.dupe(u8, hdr.name),
.value = try allocator.dupe(u8, hdr.value),
});
}
transfer.pending_cache_metadata = cm;
transfer.pending_cache_metadata.?.headers = header_list.items;
if (transfer.getContentLength()) |cl| {
try transfer.pending_cache_body.ensureTotalCapacity(allocator, cl);
}
}
}
transfer.req.notification.dispatch(.http_response_header_done, &.{ transfer.req.notification.dispatch(.http_response_header_done, &.{
.transfer = transfer, .transfer = transfer,
}); });
@@ -1428,7 +1623,14 @@ pub const Transfer = struct {
} }
const chunk = buffer[0..chunk_len]; const chunk = buffer[0..chunk_len];
transfer.req.data_callback(transfer, chunk) catch |err| { if (transfer.pending_cache_metadata != null) {
transfer.pending_cache_body.appendSlice(transfer.arena.allocator(), chunk) catch |err| {
log.err(.http, "cache body append", .{ .err = err, .req = transfer });
return Net.writefunc_error;
};
}
transfer.req.data_callback(Response.fromLive(transfer), chunk) catch |err| {
log.err(.http, "data_callback", .{ .err = err, .req = transfer }); log.err(.http, "data_callback", .{ .err = err, .req = transfer });
return Net.writefunc_error; return Net.writefunc_error;
}; };
@@ -1480,7 +1682,7 @@ pub const Transfer = struct {
fn _fulfill(transfer: *Transfer, status: u16, headers: []const Net.Header, body: ?[]const u8) !void { fn _fulfill(transfer: *Transfer, status: u16, headers: []const Net.Header, body: ?[]const u8) !void {
const req = &transfer.req; const req = &transfer.req;
if (req.start_callback) |cb| { if (req.start_callback) |cb| {
try cb(transfer); try cb(Response.fromLive(transfer));
} }
transfer.response_header = .{ transfer.response_header = .{
@@ -1499,13 +1701,13 @@ pub const Transfer = struct {
} }
lp.assert(transfer._header_done_called == false, "Transfer.fulfill header_done_called", .{}); lp.assert(transfer._header_done_called == false, "Transfer.fulfill header_done_called", .{});
if (try req.header_callback(transfer) == false) { if (try req.header_callback(Response.fromLive(transfer)) == false) {
transfer.abort(error.Abort); transfer.abort(error.Abort);
return; return;
} }
if (body) |b| { if (body) |b| {
try req.data_callback(transfer, b); try req.data_callback(Response.fromLive(transfer), b);
} }
try req.done_callback(req.ctx); try req.done_callback(req.ctx);

View File

@@ -27,6 +27,9 @@ charset: [41]u8 = default_charset,
charset_len: usize = default_charset_len, charset_len: usize = default_charset_len,
is_default_charset: bool = true, is_default_charset: bool = true,
type_buf: [127]u8 = @splat(0),
sub_type_buf: [127]u8 = @splat(0),
/// String "UTF-8" continued by null characters. /// String "UTF-8" continued by null characters.
const default_charset = .{ 'U', 'T', 'F', '-', '8' } ++ .{0} ** 36; const default_charset = .{ 'U', 'T', 'F', '-', '8' } ++ .{0} ** 36;
const default_charset_len = 5; const default_charset_len = 5;
@@ -61,7 +64,10 @@ pub const ContentType = union(ContentTypeEnum) {
image_webp: void, image_webp: void,
application_json: void, application_json: void,
unknown: void, unknown: void,
other: struct { type: []const u8, sub_type: []const u8 }, other: struct {
type: []const u8,
sub_type: []const u8,
},
}; };
pub fn contentTypeString(mime: *const Mime) []const u8 { pub fn contentTypeString(mime: *const Mime) []const u8 {
@@ -112,17 +118,18 @@ fn parseCharset(value: []const u8) error{ CharsetTooBig, Invalid }![]const u8 {
return value; return value;
} }
pub fn parse(input: []u8) !Mime { pub fn parse(input: []const u8) !Mime {
if (input.len > 255) { if (input.len > 255) {
return error.TooBig; return error.TooBig;
} }
// Zig's trim API is broken. The return type is always `[]const u8`, var buf: [255]u8 = undefined;
// even if the input type is `[]u8`. @constCast is safe here. const normalized = std.ascii.lowerString(&buf, std.mem.trim(u8, input, &std.ascii.whitespace));
var normalized = @constCast(std.mem.trim(u8, input, &std.ascii.whitespace));
_ = std.ascii.lowerString(normalized, normalized); _ = std.ascii.lowerString(normalized, normalized);
const content_type, const type_len = try parseContentType(normalized); var mime = Mime{ .content_type = undefined };
const content_type, const type_len = try parseContentType(normalized, &mime.type_buf, &mime.sub_type_buf);
if (type_len >= normalized.len) { if (type_len >= normalized.len) {
return .{ .content_type = content_type }; return .{ .content_type = content_type };
} }
@@ -163,13 +170,12 @@ pub fn parse(input: []u8) !Mime {
} }
} }
return .{ mime.params = params;
.params = params, mime.charset = charset;
.charset = charset, mime.charset_len = charset_len;
.charset_len = charset_len, mime.content_type = content_type;
.content_type = content_type, mime.is_default_charset = !has_explicit_charset;
.is_default_charset = !has_explicit_charset, return mime;
};
} }
/// Prescan the first 1024 bytes of an HTML document for a charset declaration. /// Prescan the first 1024 bytes of an HTML document for a charset declaration.
@@ -387,7 +393,7 @@ pub fn isHTML(self: *const Mime) bool {
} }
// we expect value to be lowercase // we expect value to be lowercase
fn parseContentType(value: []const u8) !struct { ContentType, usize } { fn parseContentType(value: []const u8, type_buf: []u8, sub_type_buf: []u8) !struct { ContentType, usize } {
const end = std.mem.indexOfScalarPos(u8, value, 0, ';') orelse value.len; const end = std.mem.indexOfScalarPos(u8, value, 0, ';') orelse value.len;
const type_name = trimRight(value[0..end]); const type_name = trimRight(value[0..end]);
const attribute_start = end + 1; const attribute_start = end + 1;
@@ -436,10 +442,18 @@ fn parseContentType(value: []const u8) !struct { ContentType, usize } {
return error.Invalid; return error.Invalid;
} }
return .{ .{ .other = .{ @memcpy(type_buf[0..main_type.len], main_type);
.type = main_type, @memcpy(sub_type_buf[0..sub_type.len], sub_type);
.sub_type = sub_type,
} }, attribute_start }; return .{
.{
.other = .{
.type = type_buf[0..main_type.len],
.sub_type = sub_type_buf[0..sub_type.len],
},
},
attribute_start,
};
} }
const VALID_CODEPOINTS = blk: { const VALID_CODEPOINTS = blk: {
@@ -453,6 +467,13 @@ const VALID_CODEPOINTS = blk: {
break :blk v; break :blk v;
}; };
pub fn typeString(self: *const Mime) []const u8 {
return switch (self.content_type) {
.other => |o| o.type[0..o.type_len],
else => "",
};
}
fn validType(value: []const u8) bool { fn validType(value: []const u8) bool {
for (value) |b| { for (value) |b| {
if (VALID_CODEPOINTS[b] == false) { if (VALID_CODEPOINTS[b] == false) {

View File

@@ -821,12 +821,10 @@ fn notifyParentLoadComplete(self: *Page) void {
parent.iframeCompletedLoading(self.iframe.?); parent.iframeCompletedLoading(self.iframe.?);
} }
fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { fn pageHeaderDoneCallback(response: HttpClient.Response) !bool {
var self: *Page = @ptrCast(@alignCast(transfer.ctx)); var self: *Page = @ptrCast(@alignCast(response.ctx));
const header = &transfer.response_header.?; const response_url = response.url();
const response_url = std.mem.span(header.url);
if (std.mem.eql(u8, response_url, self.url) == false) { if (std.mem.eql(u8, response_url, self.url) == false) {
// would be different than self.url in the case of a redirect // would be different than self.url in the case of a redirect
self.url = try self.arena.dupeZ(u8, response_url); self.url = try self.arena.dupeZ(u8, response_url);
@@ -840,8 +838,8 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
log.debug(.page, "navigate header", .{ log.debug(.page, "navigate header", .{
.url = self.url, .url = self.url,
.status = header.status, .status = response.status(),
.content_type = header.contentType(), .content_type = response.contentType(),
.type = self._type, .type = self._type,
}); });
} }
@@ -849,14 +847,14 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
return true; return true;
} }
fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { fn pageDataCallback(response: HttpClient.Response, data: []const u8) !void {
var self: *Page = @ptrCast(@alignCast(transfer.ctx)); var self: *Page = @ptrCast(@alignCast(response.ctx));
if (self._parse_state == .pre) { if (self._parse_state == .pre) {
// we lazily do this, because we might need the first chunk of data // we lazily do this, because we might need the first chunk of data
// to sniff the content type // to sniff the content type
var mime: Mime = blk: { var mime: Mime = blk: {
if (transfer.response_header.?.contentType()) |ct| { if (response.contentType()) |ct| {
break :blk try Mime.parse(ct); break :blk try Mime.parse(ct);
} }
break :blk Mime.sniff(data); break :blk Mime.sniff(data);

View File

@@ -694,85 +694,86 @@ pub const Script = struct {
self.manager.page.releaseArena(self.arena); self.manager.page.releaseArena(self.arena);
} }
fn startCallback(transfer: *HttpClient.Transfer) !void { fn startCallback(response: HttpClient.Response) !void {
log.debug(.http, "script fetch start", .{ .req = transfer }); log.debug(.http, "script fetch start", .{ .req = response });
} }
fn headerCallback(transfer: *HttpClient.Transfer) !bool { fn headerCallback(response: HttpClient.Response) !bool {
const self: *Script = @ptrCast(@alignCast(transfer.ctx)); const self: *Script = @ptrCast(@alignCast(response.ctx));
const header = &transfer.response_header.?;
self.status = header.status; self.status = response.status().?;
if (header.status != 200) { if (response.status() != 200) {
log.info(.http, "script header", .{ log.info(.http, "script header", .{
.req = transfer, .req = response,
.status = header.status, .status = response.status(),
.content_type = header.contentType(), .content_type = response.contentType(),
}); });
return false; return false;
} }
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
log.debug(.http, "script header", .{ log.debug(.http, "script header", .{
.req = transfer, .req = response,
.status = header.status, .status = response.status(),
.content_type = header.contentType(), .content_type = response.contentType(),
}); });
} }
{ // {
// temp debug, trying to figure out why the next assert sometimes // // temp debug, trying to figure out why the next assert sometimes
// fails. Is the buffer just corrupt or is headerCallback really // // fails. Is the buffer just corrupt or is headerCallback really
// being called twice? // // being called twice?
lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{ // lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{
.m = @tagName(std.meta.activeTag(self.mode)), // .m = @tagName(std.meta.activeTag(self.mode)),
.a1 = self.debug_transfer_id, // .a1 = self.debug_transfer_id,
.a2 = self.debug_transfer_tries, // .a2 = self.debug_transfer_tries,
.a3 = self.debug_transfer_aborted, // .a3 = self.debug_transfer_aborted,
.a4 = self.debug_transfer_bytes_received, // .a4 = self.debug_transfer_bytes_received,
.a5 = self.debug_transfer_notified_fail, // .a5 = self.debug_transfer_notified_fail,
.a6 = self.debug_transfer_redirecting, // .a6 = self.debug_transfer_redirecting,
.a7 = self.debug_transfer_intercept_state, // .a7 = self.debug_transfer_intercept_state,
.a8 = self.debug_transfer_auth_challenge, // .a8 = self.debug_transfer_auth_challenge,
.a9 = self.debug_transfer_easy_id, // .a9 = self.debug_transfer_easy_id,
.b1 = transfer.id, // .b1 = transfer.id,
.b2 = transfer._tries, // .b2 = transfer._tries,
.b3 = transfer.aborted, // .b3 = transfer.aborted,
.b4 = transfer.bytes_received, // .b4 = transfer.bytes_received,
.b5 = transfer._notified_fail, // .b5 = transfer._notified_fail,
.b6 = transfer._redirecting, // .b6 = transfer._redirecting,
.b7 = @intFromEnum(transfer._intercept_state), // .b7 = @intFromEnum(transfer._intercept_state),
.b8 = transfer._auth_challenge != null, // .b8 = transfer._auth_challenge != null,
.b9 = if (transfer._conn) |c| @intFromPtr(c.easy) else 0, // .b9 = if (transfer._conn) |c| @intFromPtr(c.easy) else 0,
}); // });
self.header_callback_called = true; // self.header_callback_called = true;
self.debug_transfer_id = transfer.id; // self.debug_transfer_id = transfer.id;
self.debug_transfer_tries = transfer._tries; // self.debug_transfer_tries = transfer._tries;
self.debug_transfer_aborted = transfer.aborted; // self.debug_transfer_aborted = transfer.aborted;
self.debug_transfer_bytes_received = transfer.bytes_received; // self.debug_transfer_bytes_received = transfer.bytes_received;
self.debug_transfer_notified_fail = transfer._notified_fail; // self.debug_transfer_notified_fail = transfer._notified_fail;
self.debug_transfer_redirecting = transfer._redirecting; // self.debug_transfer_redirecting = transfer._redirecting;
self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state); // self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
self.debug_transfer_auth_challenge = transfer._auth_challenge != null; // self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c.easy) else 0; // self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c.easy) else 0;
} // }
lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity }); lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity });
var buffer: std.ArrayList(u8) = .empty; var buffer: std.ArrayList(u8) = .empty;
if (transfer.getContentLength()) |cl| { if (response.contentLength()) |cl| {
try buffer.ensureTotalCapacity(self.arena, cl); try buffer.ensureTotalCapacity(self.arena, cl);
} }
self.source = .{ .remote = buffer }; self.source = .{ .remote = buffer };
return true; return true;
} }
fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { fn dataCallback(response: HttpClient.Response, data: []const u8) !void {
const self: *Script = @ptrCast(@alignCast(transfer.ctx)); const self: *Script = @ptrCast(@alignCast(response.ctx));
self._dataCallback(transfer, data) catch |err| { self._dataCallback(response, data) catch |err| {
log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len }); log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = response, .len = data.len });
return err; return err;
}; };
} }
fn _dataCallback(self: *Script, _: *HttpClient.Transfer, data: []const u8) !void {
fn _dataCallback(self: *Script, _: HttpClient.Response, data: []const u8) !void {
try self.source.remote.appendSlice(self.arena, data); try self.source.remote.appendSlice(self.arena, data);
} }

View File

@@ -5,7 +5,7 @@
<div id="host2"></div> <div id="host2"></div>
<div id="host3"></div> <div id="host3"></div>
<!-- <script id="attachShadow_open"> <script id="attachShadow_open">
{ {
const host = $('#host1'); const host = $('#host1');
const shadow = host.attachShadow({ mode: 'open' }); const shadow = host.attachShadow({ mode: 'open' });
@@ -140,7 +140,7 @@
shadow.replaceChildren('New content'); shadow.replaceChildren('New content');
testing.expectEqual('New content', shadow.innerHTML); testing.expectEqual('New content', shadow.innerHTML);
} }
</script> --> </script>
<script id="getElementById"> <script id="getElementById">
{ {
@@ -154,3 +154,16 @@
testing.expectEqual(null, shadow.getElementById('nonexistent')); testing.expectEqual(null, shadow.getElementById('nonexistent'));
} }
</script> </script>
<script id=adoptedStyleSheets>
{
const host = document.createElement('div');
const shadow = host.attachShadow({ mode: 'open' });
const acss = shadow.adoptedStyleSheets;
testing.expectEqual(0, acss.length);
acss.push(new CSSStyleSheet());
testing.expectEqual(1, acss.length);
}
</script>

View File

@@ -40,6 +40,7 @@ _mode: Mode,
_host: *Element, _host: *Element,
_elements_by_id: std.StringHashMapUnmanaged(*Element) = .{}, _elements_by_id: std.StringHashMapUnmanaged(*Element) = .{},
_removed_ids: std.StringHashMapUnmanaged(void) = .{}, _removed_ids: std.StringHashMapUnmanaged(void) = .{},
_adopted_style_sheets: ?js.Object.Global = null,
pub fn init(host: *Element, mode: Mode, page: *Page) !*ShadowRoot { pub fn init(host: *Element, mode: Mode, page: *Page) !*ShadowRoot {
return page._factory.documentFragment(ShadowRoot{ return page._factory.documentFragment(ShadowRoot{
@@ -99,6 +100,20 @@ pub fn getElementById(self: *ShadowRoot, id: []const u8, page: *Page) ?*Element
return null; return null;
} }
pub fn getAdoptedStyleSheets(self: *ShadowRoot, page: *Page) !js.Object.Global {
if (self._adopted_style_sheets) |ass| {
return ass;
}
const js_arr = page.js.local.?.newArray(0);
const js_obj = js_arr.toObject();
self._adopted_style_sheets = try js_obj.persist();
return self._adopted_style_sheets.?;
}
pub fn setAdoptedStyleSheets(self: *ShadowRoot, sheets: js.Object) !void {
self._adopted_style_sheets = try sheets.persist();
}
pub const JsApi = struct { pub const JsApi = struct {
pub const bridge = js.Bridge(ShadowRoot); pub const bridge = js.Bridge(ShadowRoot);
@@ -121,6 +136,7 @@ pub const JsApi = struct {
} }
return self.getElementById(try value.toZig([]const u8), page); return self.getElementById(try value.toZig([]const u8), page);
} }
pub const adoptedStyleSheets = bridge.accessor(ShadowRoot.getAdoptedStyleSheets, ShadowRoot.setAdoptedStyleSheets, .{});
}; };
const testing = @import("../../testing.zig"); const testing = @import("../../testing.zig");

View File

@@ -126,16 +126,16 @@ fn handleBlobUrl(url: []const u8, resolver: js.PromiseResolver, page: *Page) !js
return resolver.promise(); return resolver.promise();
} }
fn httpStartCallback(transfer: *HttpClient.Transfer) !void { fn httpStartCallback(response: HttpClient.Response) !void {
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx)); const self: *Fetch = @ptrCast(@alignCast(response.ctx));
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" }); log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" });
} }
self._response._transfer = transfer; self._response._http_response = response;
} }
fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { fn httpHeaderDoneCallback(response: HttpClient.Response) !bool {
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx)); const self: *Fetch = @ptrCast(@alignCast(response.ctx));
if (self._signal) |signal| { if (self._signal) |signal| {
if (signal._aborted) { if (signal._aborted) {
@@ -144,25 +144,24 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
} }
const arena = self._response._arena; const arena = self._response._arena;
if (transfer.getContentLength()) |cl| { if (response.contentLength()) |cl| {
try self._buf.ensureTotalCapacity(arena, cl); try self._buf.ensureTotalCapacity(arena, cl);
} }
const res = self._response; const res = self._response;
const header = transfer.response_header.?;
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
log.debug(.http, "request header", .{ log.debug(.http, "request header", .{
.source = "fetch", .source = "fetch",
.url = self._url, .url = self._url,
.status = header.status, .status = response.status(),
}); });
} }
res._status = header.status; res._status = response.status().?;
res._status_text = std.http.Status.phrase(@enumFromInt(header.status)) orelse ""; res._status_text = std.http.Status.phrase(@enumFromInt(response.status().?)) orelse "";
res._url = try arena.dupeZ(u8, std.mem.span(header.url)); res._url = try arena.dupeZ(u8, response.url());
res._is_redirected = header.redirect_count > 0; res._is_redirected = response.redirectCount().? > 0;
// Determine response type based on origin comparison // Determine response type based on origin comparison
const page_origin = URL.getOrigin(arena, self._page.url) catch null; const page_origin = URL.getOrigin(arena, self._page.url) catch null;
@@ -182,7 +181,7 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
res._type = .basic; res._type = .basic;
} }
var it = transfer.responseHeaderIterator(); var it = response.headerIterator();
while (it.next()) |hdr| { while (it.next()) |hdr| {
try res._headers.append(hdr.name, hdr.value, self._page); try res._headers.append(hdr.name, hdr.value, self._page);
} }
@@ -190,8 +189,8 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
return true; return true;
} }
fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx)); const self: *Fetch = @ptrCast(@alignCast(response.ctx));
// Check if aborted // Check if aborted
if (self._signal) |signal| { if (self._signal) |signal| {
@@ -206,7 +205,7 @@ fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
fn httpDoneCallback(ctx: *anyopaque) !void { fn httpDoneCallback(ctx: *anyopaque) !void {
const self: *Fetch = @ptrCast(@alignCast(ctx)); const self: *Fetch = @ptrCast(@alignCast(ctx));
var response = self._response; var response = self._response;
response._transfer = null; response._http_response = null;
response._body = self._buf.items; response._body = self._buf.items;
log.info(.http, "request complete", .{ log.info(.http, "request complete", .{
@@ -229,7 +228,7 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
const self: *Fetch = @ptrCast(@alignCast(ctx)); const self: *Fetch = @ptrCast(@alignCast(ctx));
var response = self._response; var response = self._response;
response._transfer = null; response._http_response = null;
// the response is only passed on v8 on success, if we're here, it's safe to // the response is only passed on v8 on success, if we're here, it's safe to
// clear this. (defer since `self is in the response's arena). // clear this. (defer since `self is in the response's arena).
@@ -255,7 +254,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {
if (self._owns_response) { if (self._owns_response) {
var response = self._response; var response = self._response;
response._transfer = null; response._http_response = null;
response.deinit(true, self._page._session); response.deinit(true, self._page._session);
// Do not access `self` after this point: the Fetch struct was // Do not access `self` after this point: the Fetch struct was
// allocated from response._arena which has been released. // allocated from response._arena which has been released.

View File

@@ -46,7 +46,7 @@ _type: Type,
_status_text: []const u8, _status_text: []const u8,
_url: [:0]const u8, _url: [:0]const u8,
_is_redirected: bool, _is_redirected: bool,
_transfer: ?*HttpClient.Transfer = null, _http_response: ?HttpClient.Response = null,
const InitOpts = struct { const InitOpts = struct {
status: u16 = 200, status: u16 = 200,
@@ -79,13 +79,13 @@ pub fn init(body_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*Response {
} }
pub fn deinit(self: *Response, shutdown: bool, session: *Session) void { pub fn deinit(self: *Response, shutdown: bool, session: *Session) void {
if (self._transfer) |transfer| { if (self._http_response) |resp| {
if (shutdown) { if (shutdown) {
transfer.terminate(); resp.terminate();
} else { } else {
transfer.abort(error.Abort); resp.abort(error.Abort);
} }
self._transfer = null; self._http_response = null;
} }
session.releaseArena(self._arena); session.releaseArena(self._arena);
} }
@@ -185,7 +185,7 @@ pub fn clone(self: *const Response, page: *Page) !*Response {
._type = self._type, ._type = self._type,
._is_redirected = self._is_redirected, ._is_redirected = self._is_redirected,
._headers = try Headers.init(.{ .obj = self._headers }, page), ._headers = try Headers.init(.{ .obj = self._headers }, page),
._transfer = null, ._http_response = null,
}; };
return cloned; return cloned;
} }

View File

@@ -42,7 +42,7 @@ const XMLHttpRequest = @This();
_page: *Page, _page: *Page,
_proto: *XMLHttpRequestEventTarget, _proto: *XMLHttpRequestEventTarget,
_arena: Allocator, _arena: Allocator,
_transfer: ?*HttpClient.Transfer = null, _http_response: ?HttpClient.Response = null,
_url: [:0]const u8 = "", _url: [:0]const u8 = "",
_method: net_http.Method = .GET, _method: net_http.Method = .GET,
@@ -97,13 +97,13 @@ pub fn init(page: *Page) !*XMLHttpRequest {
} }
pub fn deinit(self: *XMLHttpRequest, shutdown: bool, session: *Session) void { pub fn deinit(self: *XMLHttpRequest, shutdown: bool, session: *Session) void {
if (self._transfer) |transfer| { if (self._http_response) |transfer| {
if (shutdown) { if (shutdown) {
transfer.terminate(); transfer.terminate();
} else { } else {
transfer.abort(error.Abort); transfer.abort(error.Abort);
} }
self._transfer = null; self._http_response = null;
} }
if (self._on_ready_state_change) |func| { if (self._on_ready_state_change) |func| {
@@ -169,9 +169,9 @@ pub fn setWithCredentials(self: *XMLHttpRequest, value: bool) !void {
// TODO: url should be a union, as it can be multiple things // TODO: url should be a union, as it can be multiple things
pub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void { pub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void {
// Abort any in-progress request // Abort any in-progress request
if (self._transfer) |transfer| { if (self._http_response) |transfer| {
transfer.abort(error.Abort); transfer.abort(error.Abort);
self._transfer = null; self._http_response = null;
} }
// Reset internal state // Reset internal state
@@ -382,34 +382,32 @@ pub fn getResponseXML(self: *XMLHttpRequest, page: *Page) !?*Node.Document {
}; };
} }
fn httpStartCallback(transfer: *HttpClient.Transfer) !void { fn httpStartCallback(response: HttpClient.Response) !void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" }); log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });
} }
self._transfer = transfer; self._http_response = response;
} }
fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: net_http.Header) !void { fn httpHeaderCallback(response: HttpClient.Response, header: net_http.Header) !void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value }); const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value });
try self._response_headers.append(self._arena, joined); try self._response_headers.append(self._arena, joined);
} }
fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool { fn httpHeaderDoneCallback(response: HttpClient.Response) !bool {
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
const header = &transfer.response_header.?;
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
log.debug(.http, "request header", .{ log.debug(.http, "request header", .{
.source = "xhr", .source = "xhr",
.url = self._url, .url = self._url,
.status = header.status, .status = response.status(),
}); });
} }
if (header.contentType()) |ct| { if (response.contentType()) |ct| {
self._response_mime = Mime.parse(ct) catch |e| { self._response_mime = Mime.parse(ct) catch |e| {
log.info(.http, "invalid content type", .{ log.info(.http, "invalid content type", .{
.content_Type = ct, .content_Type = ct,
@@ -420,18 +418,18 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
}; };
} }
var it = transfer.responseHeaderIterator(); var it = response.headerIterator();
while (it.next()) |hdr| { while (it.next()) |hdr| {
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value }); const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value });
try self._response_headers.append(self._arena, joined); try self._response_headers.append(self._arena, joined);
} }
self._response_status = header.status; self._response_status = response.status().?;
if (transfer.getContentLength()) |cl| { if (response.contentLength()) |cl| {
self._response_len = cl; self._response_len = cl;
try self._response_data.ensureTotalCapacity(self._arena, cl); try self._response_data.ensureTotalCapacity(self._arena, cl);
} }
self._response_url = try self._arena.dupeZ(u8, std.mem.span(header.url)); self._response_url = try self._arena.dupeZ(u8, response.url());
const page = self._page; const page = self._page;
@@ -446,8 +444,8 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
return true; return true;
} }
fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx)); const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
try self._response_data.appendSlice(self._arena, data); try self._response_data.appendSlice(self._arena, data);
const page = self._page; const page = self._page;
@@ -470,7 +468,7 @@ fn httpDoneCallback(ctx: *anyopaque) !void {
// Not that the request is done, the http/client will free the transfer // Not that the request is done, the http/client will free the transfer
// object. It isn't safe to keep it around. // object. It isn't safe to keep it around.
self._transfer = null; self._http_response = null;
const page = self._page; const page = self._page;
@@ -492,21 +490,21 @@ fn httpDoneCallback(ctx: *anyopaque) !void {
fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void { fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx)); const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
// http client will close it after an error, it isn't safe to keep around // http client will close it after an error, it isn't safe to keep around
self._transfer = null; self._http_response = null;
self.handleError(err); self.handleError(err);
self._page.js.weakRef(self); self._page.js.weakRef(self);
} }
fn httpShutdownCallback(ctx: *anyopaque) void { fn httpShutdownCallback(ctx: *anyopaque) void {
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx)); const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
self._transfer = null; self._http_response = null;
} }
pub fn abort(self: *XMLHttpRequest) void { pub fn abort(self: *XMLHttpRequest) void {
self.handleError(error.Abort); self.handleError(error.Abort);
if (self._transfer) |transfer| { if (self._http_response) |transfer| {
transfer.abort(error.Abort); transfer.abort(error.Abort);
self._transfer = null; self._http_response = null;
} }
self._page.js.weakRef(self); self._page.js.weakRef(self);
} }

View File

@@ -29,7 +29,9 @@ const libcurl = @import("../sys/libcurl.zig");
const net_http = @import("http.zig"); const net_http = @import("http.zig");
const RobotStore = @import("Robots.zig").RobotStore; const RobotStore = @import("Robots.zig").RobotStore;
const WebBotAuth = @import("WebBotAuth.zig"); const WebBotAuth = @import("WebBotAuth.zig");
const Cache = @import("cache/Cache.zig");
const App = @import("../App.zig");
const Runtime = @This(); const Runtime = @This();
const Listener = struct { const Listener = struct {
@@ -45,10 +47,12 @@ const MAX_TICK_CALLBACKS = 16;
allocator: Allocator, allocator: Allocator,
app: *App,
config: *const Config, config: *const Config,
ca_blob: ?net_http.Blob, ca_blob: ?net_http.Blob,
robot_store: RobotStore, robot_store: RobotStore,
web_bot_auth: ?WebBotAuth, web_bot_auth: ?WebBotAuth,
cache: ?Cache,
connections: []net_http.Connection, connections: []net_http.Connection,
available: std.DoublyLinkedList = .{}, available: std.DoublyLinkedList = .{},
@@ -200,7 +204,7 @@ fn globalDeinit() void {
libcurl.curl_global_cleanup(); libcurl.curl_global_cleanup();
} }
pub fn init(allocator: Allocator, config: *const Config) !Runtime { pub fn init(allocator: Allocator, app: *App, config: *const Config) !Runtime {
globalInit(allocator); globalInit(allocator);
errdefer globalDeinit(); errdefer globalDeinit();
@@ -233,6 +237,11 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime {
else else
null; null;
const cache = if (config.cacheDir()) |cache_dir_path|
Cache{ .kind = .{ .fs = try .init(cache_dir_path) } }
else
null;
return .{ return .{
.allocator = allocator, .allocator = allocator,
.config = config, .config = config,
@@ -244,8 +253,10 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime {
.available = available, .available = available,
.connections = connections, .connections = connections,
.app = app,
.robot_store = RobotStore.init(allocator), .robot_store = RobotStore.init(allocator),
.web_bot_auth = web_bot_auth, .web_bot_auth = web_bot_auth,
.cache = cache,
}; };
} }

173
src/network/cache/Cache.zig vendored Normal file
View File

@@ -0,0 +1,173 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const Http = @import("../http.zig");
const FsCache = @import("FsCache.zig");
/// A browser-wide cache for resources across the network.
/// This mostly conforms to RFC9111 with regards to caching behavior.
pub const Cache = @This();
kind: union(enum) {
fs: FsCache,
},
pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse {
return switch (self.kind) {
inline else => |*c| c.get(arena, req),
};
}
pub fn put(self: *Cache, metadata: CachedMetadata, body: []const u8) !void {
return switch (self.kind) {
inline else => |*c| c.put(metadata, body),
};
}
pub const CacheControl = struct {
max_age: u64,
must_revalidate: bool = false,
immutable: bool = false,
pub fn parse(value: []const u8) ?CacheControl {
var cc: CacheControl = .{ .max_age = undefined };
var max_age_set = false;
var max_s_age_set = false;
var is_public = false;
var iter = std.mem.splitScalar(u8, value, ',');
while (iter.next()) |part| {
const directive = std.mem.trim(u8, part, &std.ascii.whitespace);
if (std.ascii.eqlIgnoreCase(directive, "no-store")) {
return null;
} else if (std.ascii.eqlIgnoreCase(directive, "no-cache")) {
return null;
} else if (std.ascii.eqlIgnoreCase(directive, "must-revalidate")) {
cc.must_revalidate = true;
} else if (std.ascii.eqlIgnoreCase(directive, "immutable")) {
cc.immutable = true;
} else if (std.ascii.eqlIgnoreCase(directive, "public")) {
is_public = true;
} else if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) {
if (!max_s_age_set) {
if (std.fmt.parseInt(u64, directive[8..], 10) catch null) |max_age| {
cc.max_age = max_age;
max_age_set = true;
}
}
} else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) {
if (std.fmt.parseInt(u64, directive[9..], 10) catch null) |max_age| {
cc.max_age = max_age;
max_age_set = true;
max_s_age_set = true;
}
}
}
if (!max_age_set) return null;
if (!is_public) return null;
if (cc.max_age == 0) return null;
return cc;
}
};
pub const Vary = union(enum) {
wildcard: void,
value: []const u8,
pub fn parse(value: []const u8) Vary {
if (std.mem.eql(u8, value, "*")) return .wildcard;
return .{ .value = value };
}
pub fn toString(self: Vary) []const u8 {
return switch (self) {
.wildcard => "*",
.value => |v| v,
};
}
};
pub const CachedMetadata = struct {
url: [:0]const u8,
content_type: []const u8,
status: u16,
stored_at: i64,
age_at_store: u64,
// for If-None-Match
etag: ?[]const u8,
// for If-Modified-Since
last_modified: ?[]const u8,
cache_control: CacheControl,
vary: ?Vary,
headers: []const Http.Header,
};
pub const CacheRequest = struct {
url: []const u8,
timestamp: i64,
};
pub const CachedData = union(enum) {
buffer: []const u8,
file: std.fs.File,
};
pub const CachedResponse = struct {
metadata: CachedMetadata,
data: CachedData,
};
pub fn tryCache(
arena: std.mem.Allocator,
timestamp: i64,
url: [:0]const u8,
status: u16,
content_type: ?[]const u8,
cache_control: ?[]const u8,
vary: ?[]const u8,
etag: ?[]const u8,
last_modified: ?[]const u8,
age: ?[]const u8,
has_set_cookie: bool,
has_authorization: bool,
) !?CachedMetadata {
if (status != 200) return null;
if (has_set_cookie) return null;
if (has_authorization) return null;
const cc = CacheControl.parse(cache_control orelse return null) orelse return null;
return .{
.url = url,
.content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",
.status = status,
.stored_at = timestamp,
.age_at_store = if (age) |a| std.fmt.parseInt(u64, a, 10) catch 0 else 0,
.cache_control = cc,
.vary = if (vary) |v| Vary.parse(v) else null,
.etag = if (etag) |e| try arena.dupe(u8, e) else null,
.last_modified = if (last_modified) |lm| try arena.dupe(u8, lm) else null,
.headers = &.{},
};
}

293
src/network/cache/FsCache.zig vendored Normal file
View File

@@ -0,0 +1,293 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const Cache = @import("Cache.zig");
const Http = @import("../http.zig");
const CacheRequest = Cache.CacheRequest;
const CachedMetadata = Cache.CachedMetadata;
const CachedResponse = Cache.CachedResponse;
const CACHE_VERSION: usize = 1;
const LOCK_STRIPES = 16;
pub const FsCache = @This();
dir: std.fs.Dir,
locks: [LOCK_STRIPES]std.Thread.Mutex = .{std.Thread.Mutex{}} ** LOCK_STRIPES,
const CacheMetadataFile = struct {
version: usize,
metadata: CachedMetadata,
};
fn getLockPtr(self: *FsCache, key: *const [HASHED_KEY_LEN]u8) *std.Thread.Mutex {
const lock_idx: usize = @truncate(std.hash.Wyhash.hash(0, key) % LOCK_STRIPES);
return &self.locks[lock_idx];
}
const HASHED_KEY_LEN = 64;
const HASHED_PATH_LEN = HASHED_KEY_LEN + 5;
const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4;
fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 {
var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined;
std.crypto.hash.sha2.Sha256.hash(key, &digest, .{});
var hex: [HASHED_KEY_LEN]u8 = undefined;
_ = std.fmt.bufPrint(&hex, "{s}", .{std.fmt.bytesToHex(&digest, .lower)}) catch unreachable;
return hex;
}
fn metaPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 {
var path: [HASHED_PATH_LEN]u8 = undefined;
_ = std.fmt.bufPrint(&path, "{s}.meta", .{hashed_key}) catch unreachable;
return path;
}
fn bodyPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 {
var path: [HASHED_PATH_LEN]u8 = undefined;
_ = std.fmt.bufPrint(&path, "{s}.body", .{hashed_key}) catch unreachable;
return path;
}
fn metaTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 {
var path: [HASHED_TMP_PATH_LEN]u8 = undefined;
_ = std.fmt.bufPrint(&path, "{s}.meta.tmp", .{hashed_key}) catch unreachable;
return path;
}
fn bodyTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 {
var path: [HASHED_TMP_PATH_LEN]u8 = undefined;
_ = std.fmt.bufPrint(&path, "{s}.body.tmp", .{hashed_key}) catch unreachable;
return path;
}
pub fn init(path: []const u8) !FsCache {
const cwd = std.fs.cwd();
cwd.makeDir(path) catch |err| switch (err) {
error.PathAlreadyExists => {},
else => return err,
};
const dir = try cwd.openDir(path, .{ .iterate = true });
return .{ .dir = dir };
}
pub fn deinit(self: *FsCache) void {
self.dir.close();
}
pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.CachedResponse {
const hashed_key = hashKey(req.url);
const meta_p = metaPath(&hashed_key);
const body_p = bodyPath(&hashed_key);
const lock = self.getLockPtr(&hashed_key);
lock.lock();
defer lock.unlock();
const meta_file = self.dir.openFile(&meta_p, .{ .mode = .read_only }) catch return null;
defer meta_file.close();
const contents = meta_file.readToEndAlloc(arena, 1 * 1024 * 1024) catch return null;
defer arena.free(contents);
const cache_file: CacheMetadataFile = std.json.parseFromSliceLeaky(
CacheMetadataFile,
arena,
contents,
.{ .allocate = .alloc_always },
) catch {
self.dir.deleteFile(&meta_p) catch {};
self.dir.deleteFile(&body_p) catch {};
return null;
};
const metadata = cache_file.metadata;
if (cache_file.version != CACHE_VERSION) {
self.dir.deleteFile(&meta_p) catch {};
self.dir.deleteFile(&body_p) catch {};
return null;
}
const now = req.timestamp;
const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store));
if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) {
self.dir.deleteFile(&meta_p) catch {};
self.dir.deleteFile(&body_p) catch {};
return null;
}
const body_file = self.dir.openFile(
&body_p,
.{ .mode = .read_only },
) catch return null;
return .{
.metadata = metadata,
.data = .{ .file = body_file },
};
}
pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void {
const hashed_key = hashKey(meta.url);
const meta_p = metaPath(&hashed_key);
const meta_tmp_p = metaTmpPath(&hashed_key);
const body_p = bodyPath(&hashed_key);
const body_tmp_p = bodyTmpPath(&hashed_key);
var writer_buf: [512]u8 = undefined;
const lock = self.getLockPtr(&hashed_key);
lock.lock();
defer lock.unlock();
{
const meta_file = try self.dir.createFile(&meta_tmp_p, .{});
errdefer {
meta_file.close();
self.dir.deleteFile(&meta_tmp_p) catch {};
}
var meta_file_writer = meta_file.writer(&writer_buf);
const meta_file_writer_iface = &meta_file_writer.interface;
try std.json.Stringify.value(
CacheMetadataFile{ .version = CACHE_VERSION, .metadata = meta },
.{ .whitespace = .minified },
meta_file_writer_iface,
);
try meta_file_writer_iface.flush();
meta_file.close();
}
errdefer self.dir.deleteFile(&meta_tmp_p) catch {};
try self.dir.rename(&meta_tmp_p, &meta_p);
{
const body_file = try self.dir.createFile(&body_tmp_p, .{});
errdefer {
body_file.close();
self.dir.deleteFile(&body_tmp_p) catch {};
}
var body_file_writer = body_file.writer(&writer_buf);
const body_file_writer_iface = &body_file_writer.interface;
try body_file_writer_iface.writeAll(body);
try body_file_writer_iface.flush();
body_file.close();
}
errdefer self.dir.deleteFile(&body_tmp_p) catch {};
errdefer self.dir.deleteFile(&meta_p) catch {};
try self.dir.rename(&body_tmp_p, &body_p);
}
const testing = std.testing;
test "FsCache: basic put and get" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
const path = try tmp.dir.realpathAlloc(testing.allocator, ".");
defer testing.allocator.free(path);
var fs_cache = try FsCache.init(path);
defer fs_cache.deinit();
var cache = Cache{ .kind = .{ .fs = fs_cache } };
var arena = std.heap.ArenaAllocator.init(testing.allocator);
defer arena.deinit();
const now = std.time.timestamp();
const meta = CachedMetadata{
.url = "https://example.com",
.content_type = "text/html",
.status = 200,
.stored_at = now,
.age_at_store = 0,
.etag = null,
.last_modified = null,
.cache_control = .{ .max_age = 600 },
.vary = null,
.headers = &.{},
};
const body = "hello world";
try cache.put(meta, body);
const result = cache.get(arena.allocator(), .{ .url = "https://example.com", .timestamp = now }) orelse return error.CacheMiss;
defer result.data.file.close();
var buf: [64]u8 = undefined;
var file_reader = result.data.file.reader(&buf);
const read_buf = try file_reader.interface.allocRemaining(testing.allocator, .unlimited);
defer testing.allocator.free(read_buf);
try testing.expectEqualStrings(body, read_buf);
}
test "FsCache: get expiration" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
const path = try tmp.dir.realpathAlloc(testing.allocator, ".");
defer testing.allocator.free(path);
var fs_cache = try FsCache.init(path);
defer fs_cache.deinit();
var cache = Cache{ .kind = .{ .fs = fs_cache } };
var arena = std.heap.ArenaAllocator.init(testing.allocator);
defer arena.deinit();
const now = 5000;
const max_age = 1000;
const meta = CachedMetadata{
.url = "https://example.com",
.content_type = "text/html",
.status = 200,
.stored_at = now,
.age_at_store = 900,
.etag = null,
.last_modified = null,
.cache_control = .{ .max_age = max_age },
.vary = null,
.headers = &.{},
};
const body = "hello world";
try cache.put(meta, body);
const result = cache.get(
arena.allocator(),
.{ .url = "https://example.com", .timestamp = now + 50 },
) orelse return error.CacheMiss;
result.data.file.close();
try testing.expectEqual(null, cache.get(
arena.allocator(),
.{ .url = "https://example.com", .timestamp = now + 200 },
));
try testing.expectEqual(null, cache.get(
arena.allocator(),
.{ .url = "https://example.com", .timestamp = now },
));
}