mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-03-28 15:40:04 +00:00
Compare commits
29 Commits
finalizers
...
http-cache
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c9b8632b50 | ||
|
|
41f154b042 | ||
|
|
666b1d1016 | ||
|
|
1559346a67 | ||
|
|
0c97ff2cab | ||
|
|
2a085dc0c2 | ||
|
|
26ba3b52ca | ||
|
|
076c1942a4 | ||
|
|
def6e564d2 | ||
|
|
368ac00573 | ||
|
|
756373e0ba | ||
|
|
e596e0c310 | ||
|
|
4ccf150ed0 | ||
|
|
dcb08b97ad | ||
|
|
c2afb2fb17 | ||
|
|
8a9007b939 | ||
|
|
cd31e68212 | ||
|
|
46d25652e0 | ||
|
|
6972a3e130 | ||
|
|
8f13ace8a2 | ||
|
|
31921e4890 | ||
|
|
3abf026f77 | ||
|
|
3f0b17c755 | ||
|
|
a97ce89362 | ||
|
|
9da1b96a14 | ||
|
|
6bd5f1d013 | ||
|
|
3f2a4ada53 | ||
|
|
edd2c661f4 | ||
|
|
9e902d176e |
@@ -55,7 +55,7 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
|
|||||||
.arena_pool = undefined,
|
.arena_pool = undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
app.network = try Network.init(allocator, config);
|
app.network = try Network.init(allocator, app, config);
|
||||||
errdefer app.network.deinit();
|
errdefer app.network.deinit();
|
||||||
|
|
||||||
app.platform = try Platform.init();
|
app.platform = try Platform.init();
|
||||||
|
|||||||
@@ -156,6 +156,13 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn cacheDir(self: *const Config) ?[]const u8 {
|
||||||
|
return switch (self.mode) {
|
||||||
|
inline .serve, .fetch, .mcp => |opts| opts.common.cache_dir,
|
||||||
|
else => null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
pub fn cdpTimeout(self: *const Config) usize {
|
pub fn cdpTimeout(self: *const Config) usize {
|
||||||
return switch (self.mode) {
|
return switch (self.mode) {
|
||||||
.serve => |opts| if (opts.timeout > 604_800) 604_800_000 else @as(usize, opts.timeout) * 1000,
|
.serve => |opts| if (opts.timeout > 604_800) 604_800_000 else @as(usize, opts.timeout) * 1000,
|
||||||
@@ -240,6 +247,7 @@ pub const Common = struct {
|
|||||||
log_format: ?log.Format = null,
|
log_format: ?log.Format = null,
|
||||||
log_filter_scopes: ?[]log.Scope = null,
|
log_filter_scopes: ?[]log.Scope = null,
|
||||||
user_agent_suffix: ?[]const u8 = null,
|
user_agent_suffix: ?[]const u8 = null,
|
||||||
|
cache_dir: ?[]const u8 = null,
|
||||||
|
|
||||||
web_bot_auth_key_file: ?[]const u8 = null,
|
web_bot_auth_key_file: ?[]const u8 = null,
|
||||||
web_bot_auth_keyid: ?[]const u8 = null,
|
web_bot_auth_keyid: ?[]const u8 = null,
|
||||||
@@ -907,5 +915,14 @@ fn parseCommonArg(
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (std.mem.eql(u8, "--cache_dir", opt)) {
|
||||||
|
const str = args.next() orelse {
|
||||||
|
log.fatal(.app, "missing argument value", .{ .arg = "--cache_dir" });
|
||||||
|
return error.InvalidArgument;
|
||||||
|
};
|
||||||
|
common.cache_dir = try allocator.dupe(u8, str);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,6 +32,10 @@ const Robots = @import("../network/Robots.zig");
|
|||||||
const RobotStore = Robots.RobotStore;
|
const RobotStore = Robots.RobotStore;
|
||||||
const WebBotAuth = @import("../network/WebBotAuth.zig");
|
const WebBotAuth = @import("../network/WebBotAuth.zig");
|
||||||
|
|
||||||
|
const Cache = @import("../network/cache/Cache.zig");
|
||||||
|
const CacheMetadata = Cache.CachedMetadata;
|
||||||
|
const CachedResponse = Cache.CachedResponse;
|
||||||
|
|
||||||
const Allocator = std.mem.Allocator;
|
const Allocator = std.mem.Allocator;
|
||||||
const ArenaAllocator = std.heap.ArenaAllocator;
|
const ArenaAllocator = std.heap.ArenaAllocator;
|
||||||
|
|
||||||
@@ -289,7 +293,69 @@ pub fn request(self: *Client, req: Request) !void {
|
|||||||
return self.fetchRobotsThenProcessRequest(robots_url, req);
|
return self.fetchRobotsThenProcessRequest(robots_url, req);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn serveFromCache(req: Request, cached: *const CachedResponse) !void {
|
||||||
|
const response = Response.fromCached(req.ctx, cached);
|
||||||
|
|
||||||
|
if (req.start_callback) |cb| {
|
||||||
|
try cb(response);
|
||||||
|
}
|
||||||
|
|
||||||
|
const proceed = try req.header_callback(response);
|
||||||
|
if (!proceed) {
|
||||||
|
switch (cached.data) {
|
||||||
|
.buffer => |_| {},
|
||||||
|
.file => |file| file.close(),
|
||||||
|
}
|
||||||
|
req.error_callback(req.ctx, error.Abort);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
switch (cached.data) {
|
||||||
|
.buffer => |data| {
|
||||||
|
if (data.len > 0) {
|
||||||
|
try req.data_callback(response, data);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
.file => |file| {
|
||||||
|
defer file.close();
|
||||||
|
var buf: [1024]u8 = undefined;
|
||||||
|
var file_reader = file.reader(&buf);
|
||||||
|
|
||||||
|
const reader = &file_reader.interface;
|
||||||
|
var read_buf: [1024]u8 = undefined;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const curr = try reader.readSliceShort(&read_buf);
|
||||||
|
if (curr == 0) break;
|
||||||
|
try req.data_callback(response, read_buf[0..curr]);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
try req.done_callback(req.ctx);
|
||||||
|
}
|
||||||
|
|
||||||
fn processRequest(self: *Client, req: Request) !void {
|
fn processRequest(self: *Client, req: Request) !void {
|
||||||
|
if (self.network.cache) |*cache| {
|
||||||
|
if (req.method == .GET) {
|
||||||
|
const arena = try self.network.app.arena_pool.acquire();
|
||||||
|
defer self.network.app.arena_pool.release(arena);
|
||||||
|
|
||||||
|
if (cache.get(arena, .{ .url = req.url, .timestamp = std.time.timestamp() })) |cached| {
|
||||||
|
log.debug(.browser, "http.cache.get", .{
|
||||||
|
.url = req.url,
|
||||||
|
.found = true,
|
||||||
|
.metadata = cached.metadata,
|
||||||
|
});
|
||||||
|
|
||||||
|
defer req.headers.deinit();
|
||||||
|
return serveFromCache(req, &cached);
|
||||||
|
} else {
|
||||||
|
log.debug(.browser, "http.cache.get", .{ .url = req.url, .found = false });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const transfer = try self.makeTransfer(req);
|
const transfer = try self.makeTransfer(req);
|
||||||
|
|
||||||
transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
|
transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
|
||||||
@@ -376,8 +442,10 @@ fn fetchRobotsThenProcessRequest(self: *Client, robots_url: [:0]const u8, req: R
|
|||||||
try entry.value_ptr.append(self.allocator, req);
|
try entry.value_ptr.append(self.allocator, req);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn robotsHeaderCallback(transfer: *Transfer) !bool {
|
fn robotsHeaderCallback(response: Response) !bool {
|
||||||
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx));
|
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx));
|
||||||
|
// Robots callbacks only happen on real live requests.
|
||||||
|
const transfer = response.inner.live;
|
||||||
|
|
||||||
if (transfer.response_header) |hdr| {
|
if (transfer.response_header) |hdr| {
|
||||||
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = ctx.robots_url });
|
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = ctx.robots_url });
|
||||||
@@ -391,8 +459,8 @@ fn robotsHeaderCallback(transfer: *Transfer) !bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn robotsDataCallback(transfer: *Transfer, data: []const u8) !void {
|
fn robotsDataCallback(response: Response, data: []const u8) !void {
|
||||||
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx));
|
const ctx: *RobotsRequestContext = @ptrCast(@alignCast(response.ctx));
|
||||||
try ctx.buffer.appendSlice(ctx.client.allocator, data);
|
try ctx.buffer.appendSlice(ctx.client.allocator, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -611,7 +679,6 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer {
|
|||||||
.id = id,
|
.id = id,
|
||||||
.url = req.url,
|
.url = req.url,
|
||||||
.req = req,
|
.req = req,
|
||||||
.ctx = req.ctx,
|
|
||||||
.client = self,
|
.client = self,
|
||||||
.max_response_size = self.network.config.httpMaxResponseSize(),
|
.max_response_size = self.network.config.httpMaxResponseSize(),
|
||||||
};
|
};
|
||||||
@@ -634,9 +701,9 @@ fn requestFailed(transfer: *Transfer, err: anyerror, comptime execute_callback:
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (execute_callback) {
|
if (execute_callback) {
|
||||||
transfer.req.error_callback(transfer.ctx, err);
|
transfer.req.error_callback(transfer.req.ctx, err);
|
||||||
} else if (transfer.req.shutdown_callback) |cb| {
|
} else if (transfer.req.shutdown_callback) |cb| {
|
||||||
cb(transfer.ctx);
|
cb(transfer.req.ctx);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -743,7 +810,7 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (req.start_callback) |cb| {
|
if (req.start_callback) |cb| {
|
||||||
cb(transfer) catch |err| {
|
cb(Response.fromLive(transfer)) catch |err| {
|
||||||
transfer.deinit();
|
transfer.deinit();
|
||||||
return err;
|
return err;
|
||||||
};
|
};
|
||||||
@@ -875,18 +942,31 @@ fn processMessages(self: *Client) !bool {
|
|||||||
break :blk;
|
break :blk;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
transfer.req.done_callback(transfer.ctx) catch |err| {
|
transfer.req.done_callback(transfer.req.ctx) catch |err| {
|
||||||
// transfer isn't valid at this point, don't use it.
|
// transfer isn't valid at this point, don't use it.
|
||||||
log.err(.http, "done_callback", .{ .err = err });
|
log.err(.http, "done_callback", .{ .err = err });
|
||||||
requestFailed(transfer, err, true);
|
requestFailed(transfer, err, true);
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
|
||||||
transfer.req.notification.dispatch(.http_request_done, &.{
|
if (transfer.pending_cache_metadata) |metadata| {
|
||||||
.transfer = transfer,
|
const cache = &self.network.cache.?;
|
||||||
});
|
|
||||||
processed = true;
|
// TODO: Support Vary Keying
|
||||||
|
const cache_key = transfer.req.url;
|
||||||
|
|
||||||
|
log.debug(.browser, "http cache", .{ .key = cache_key, .metadata = metadata });
|
||||||
|
cache.put(metadata, transfer.pending_cache_body.items) catch |err| {
|
||||||
|
log.warn(.http, "cache put failed", .{ .err = err });
|
||||||
|
};
|
||||||
|
log.debug(.browser, "http.cache.put", .{ .url = transfer.req.url });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
transfer.req.notification.dispatch(.http_request_done, &.{
|
||||||
|
.transfer = transfer,
|
||||||
|
});
|
||||||
|
processed = true;
|
||||||
}
|
}
|
||||||
return processed;
|
return processed;
|
||||||
}
|
}
|
||||||
@@ -962,9 +1042,9 @@ pub const Request = struct {
|
|||||||
// arbitrary data that can be associated with this request
|
// arbitrary data that can be associated with this request
|
||||||
ctx: *anyopaque = undefined,
|
ctx: *anyopaque = undefined,
|
||||||
|
|
||||||
start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null,
|
start_callback: ?*const fn (response: Response) anyerror!void = null,
|
||||||
header_callback: *const fn (transfer: *Transfer) anyerror!bool,
|
header_callback: *const fn (response: Response) anyerror!bool,
|
||||||
data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void,
|
data_callback: *const fn (response: Response, data: []const u8) anyerror!void,
|
||||||
done_callback: *const fn (ctx: *anyopaque) anyerror!void,
|
done_callback: *const fn (ctx: *anyopaque) anyerror!void,
|
||||||
error_callback: *const fn (ctx: *anyopaque, err: anyerror) void,
|
error_callback: *const fn (ctx: *anyopaque, err: anyerror) void,
|
||||||
shutdown_callback: ?*const fn (ctx: *anyopaque) void = null,
|
shutdown_callback: ?*const fn (ctx: *anyopaque) void = null,
|
||||||
@@ -992,16 +1072,92 @@ pub const Request = struct {
|
|||||||
|
|
||||||
const AuthChallenge = Net.AuthChallenge;
|
const AuthChallenge = Net.AuthChallenge;
|
||||||
|
|
||||||
|
pub const Response = struct {
|
||||||
|
ctx: *anyopaque,
|
||||||
|
inner: union(enum) {
|
||||||
|
live: *Transfer,
|
||||||
|
cached: *const CachedResponse,
|
||||||
|
},
|
||||||
|
|
||||||
|
pub fn fromLive(transfer: *Transfer) Response {
|
||||||
|
return .{ .ctx = transfer.req.ctx, .inner = .{ .live = transfer } };
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn fromCached(ctx: *anyopaque, resp: *const CachedResponse) Response {
|
||||||
|
return .{ .ctx = ctx, .inner = .{ .cached = resp } };
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn status(self: Response) ?u16 {
|
||||||
|
return switch (self.inner) {
|
||||||
|
.live => |live| if (live.response_header) |rh| rh.status else null,
|
||||||
|
.cached => |c| c.metadata.status,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn contentType(self: Response) ?[]const u8 {
|
||||||
|
return switch (self.inner) {
|
||||||
|
.live => |live| if (live.response_header) |*rh| rh.contentType() else null,
|
||||||
|
.cached => |c| c.metadata.content_type,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn contentLength(self: Response) ?u32 {
|
||||||
|
return switch (self.inner) {
|
||||||
|
.live => |live| live.getContentLength(),
|
||||||
|
.cached => |c| switch (c.data) {
|
||||||
|
.buffer => |buf| @intCast(buf.len),
|
||||||
|
.file => |f| @intCast(f.getEndPos() catch 0),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn redirectCount(self: Response) ?u32 {
|
||||||
|
return switch (self.inner) {
|
||||||
|
.live => |live| if (live.response_header) |rh| rh.redirect_count else null,
|
||||||
|
.cached => 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn url(self: Response) [:0]const u8 {
|
||||||
|
return switch (self.inner) {
|
||||||
|
.live => |live| live.url,
|
||||||
|
.cached => |c| c.metadata.url,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn headerIterator(self: Response) HeaderIterator {
|
||||||
|
return switch (self.inner) {
|
||||||
|
.live => |live| live.responseHeaderIterator(),
|
||||||
|
.cached => |c| HeaderIterator{ .list = .{ .list = c.metadata.headers } },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn abort(self: Response, err: anyerror) void {
|
||||||
|
switch (self.inner) {
|
||||||
|
.live => |live| live.abort(err),
|
||||||
|
.cached => {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn terminate(self: Response) void {
|
||||||
|
switch (self.inner) {
|
||||||
|
.live => |live| live.terminate(),
|
||||||
|
.cached => {},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
pub const Transfer = struct {
|
pub const Transfer = struct {
|
||||||
arena: ArenaAllocator,
|
arena: ArenaAllocator,
|
||||||
id: u32 = 0,
|
id: u32 = 0,
|
||||||
req: Request,
|
req: Request,
|
||||||
url: [:0]const u8,
|
url: [:0]const u8,
|
||||||
ctx: *anyopaque, // copied from req.ctx to make it easier for callback handlers
|
|
||||||
client: *Client,
|
client: *Client,
|
||||||
// total bytes received in the response, including the response status line,
|
// total bytes received in the response, including the response status line,
|
||||||
// the headers, and the [encoded] body.
|
// the headers, and the [encoded] body.
|
||||||
bytes_received: usize = 0,
|
bytes_received: usize = 0,
|
||||||
|
pending_cache_body: std.ArrayList(u8) = .empty,
|
||||||
|
pending_cache_metadata: ?CacheMetadata = null,
|
||||||
|
|
||||||
aborted: bool = false,
|
aborted: bool = false,
|
||||||
|
|
||||||
@@ -1053,6 +1209,8 @@ pub const Transfer = struct {
|
|||||||
self._notified_fail = false;
|
self._notified_fail = false;
|
||||||
self.response_header = null;
|
self.response_header = null;
|
||||||
self.bytes_received = 0;
|
self.bytes_received = 0;
|
||||||
|
self.pending_cache_metadata = null;
|
||||||
|
self.pending_cache_body = .empty;
|
||||||
|
|
||||||
self._tries += 1;
|
self._tries += 1;
|
||||||
}
|
}
|
||||||
@@ -1161,7 +1319,7 @@ pub const Transfer = struct {
|
|||||||
self.client.endTransfer(self);
|
self.client.endTransfer(self);
|
||||||
}
|
}
|
||||||
if (self.req.shutdown_callback) |cb| {
|
if (self.req.shutdown_callback) |cb| {
|
||||||
cb(self.ctx);
|
cb(self.req.ctx);
|
||||||
}
|
}
|
||||||
self.deinit();
|
self.deinit();
|
||||||
}
|
}
|
||||||
@@ -1263,11 +1421,48 @@ pub const Transfer = struct {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const proceed = transfer.req.header_callback(transfer) catch |err| {
|
const proceed = transfer.req.header_callback(Response.fromLive(transfer)) catch |err| {
|
||||||
log.err(.http, "header_callback", .{ .err = err, .req = transfer });
|
log.err(.http, "header_callback", .{ .err = err, .req = transfer });
|
||||||
return err;
|
return err;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (transfer.client.network.cache != null and transfer.req.method == .GET) {
|
||||||
|
const rh = &transfer.response_header.?;
|
||||||
|
const allocator = transfer.arena.allocator();
|
||||||
|
|
||||||
|
const maybe_cm = try Cache.tryCache(
|
||||||
|
allocator,
|
||||||
|
std.time.timestamp(),
|
||||||
|
transfer.url,
|
||||||
|
rh.status,
|
||||||
|
rh.contentType(),
|
||||||
|
if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,
|
||||||
|
if (conn.getResponseHeader("vary", 0)) |h| h.value else null,
|
||||||
|
if (conn.getResponseHeader("etag", 0)) |h| h.value else null,
|
||||||
|
if (conn.getResponseHeader("last-modified", 0)) |h| h.value else null,
|
||||||
|
if (conn.getResponseHeader("age", 0)) |h| h.value else null,
|
||||||
|
conn.getResponseHeader("set-cookie", 0) != null,
|
||||||
|
conn.getResponseHeader("authorization", 0) != null,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (maybe_cm) |cm| {
|
||||||
|
var header_list: std.ArrayList(Net.Header) = .empty;
|
||||||
|
var it = transfer.responseHeaderIterator();
|
||||||
|
while (it.next()) |hdr| {
|
||||||
|
try header_list.append(allocator, .{
|
||||||
|
.name = try allocator.dupe(u8, hdr.name),
|
||||||
|
.value = try allocator.dupe(u8, hdr.value),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
transfer.pending_cache_metadata = cm;
|
||||||
|
transfer.pending_cache_metadata.?.headers = header_list.items;
|
||||||
|
if (transfer.getContentLength()) |cl| {
|
||||||
|
try transfer.pending_cache_body.ensureTotalCapacity(allocator, cl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
transfer.req.notification.dispatch(.http_response_header_done, &.{
|
transfer.req.notification.dispatch(.http_response_header_done, &.{
|
||||||
.transfer = transfer,
|
.transfer = transfer,
|
||||||
});
|
});
|
||||||
@@ -1428,7 +1623,14 @@ pub const Transfer = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const chunk = buffer[0..chunk_len];
|
const chunk = buffer[0..chunk_len];
|
||||||
transfer.req.data_callback(transfer, chunk) catch |err| {
|
if (transfer.pending_cache_metadata != null) {
|
||||||
|
transfer.pending_cache_body.appendSlice(transfer.arena.allocator(), chunk) catch |err| {
|
||||||
|
log.err(.http, "cache body append", .{ .err = err, .req = transfer });
|
||||||
|
return Net.writefunc_error;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
transfer.req.data_callback(Response.fromLive(transfer), chunk) catch |err| {
|
||||||
log.err(.http, "data_callback", .{ .err = err, .req = transfer });
|
log.err(.http, "data_callback", .{ .err = err, .req = transfer });
|
||||||
return Net.writefunc_error;
|
return Net.writefunc_error;
|
||||||
};
|
};
|
||||||
@@ -1480,7 +1682,7 @@ pub const Transfer = struct {
|
|||||||
fn _fulfill(transfer: *Transfer, status: u16, headers: []const Net.Header, body: ?[]const u8) !void {
|
fn _fulfill(transfer: *Transfer, status: u16, headers: []const Net.Header, body: ?[]const u8) !void {
|
||||||
const req = &transfer.req;
|
const req = &transfer.req;
|
||||||
if (req.start_callback) |cb| {
|
if (req.start_callback) |cb| {
|
||||||
try cb(transfer);
|
try cb(Response.fromLive(transfer));
|
||||||
}
|
}
|
||||||
|
|
||||||
transfer.response_header = .{
|
transfer.response_header = .{
|
||||||
@@ -1499,13 +1701,13 @@ pub const Transfer = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
lp.assert(transfer._header_done_called == false, "Transfer.fulfill header_done_called", .{});
|
lp.assert(transfer._header_done_called == false, "Transfer.fulfill header_done_called", .{});
|
||||||
if (try req.header_callback(transfer) == false) {
|
if (try req.header_callback(Response.fromLive(transfer)) == false) {
|
||||||
transfer.abort(error.Abort);
|
transfer.abort(error.Abort);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (body) |b| {
|
if (body) |b| {
|
||||||
try req.data_callback(transfer, b);
|
try req.data_callback(Response.fromLive(transfer), b);
|
||||||
}
|
}
|
||||||
|
|
||||||
try req.done_callback(req.ctx);
|
try req.done_callback(req.ctx);
|
||||||
|
|||||||
@@ -27,6 +27,9 @@ charset: [41]u8 = default_charset,
|
|||||||
charset_len: usize = default_charset_len,
|
charset_len: usize = default_charset_len,
|
||||||
is_default_charset: bool = true,
|
is_default_charset: bool = true,
|
||||||
|
|
||||||
|
type_buf: [127]u8 = @splat(0),
|
||||||
|
sub_type_buf: [127]u8 = @splat(0),
|
||||||
|
|
||||||
/// String "UTF-8" continued by null characters.
|
/// String "UTF-8" continued by null characters.
|
||||||
const default_charset = .{ 'U', 'T', 'F', '-', '8' } ++ .{0} ** 36;
|
const default_charset = .{ 'U', 'T', 'F', '-', '8' } ++ .{0} ** 36;
|
||||||
const default_charset_len = 5;
|
const default_charset_len = 5;
|
||||||
@@ -61,7 +64,10 @@ pub const ContentType = union(ContentTypeEnum) {
|
|||||||
image_webp: void,
|
image_webp: void,
|
||||||
application_json: void,
|
application_json: void,
|
||||||
unknown: void,
|
unknown: void,
|
||||||
other: struct { type: []const u8, sub_type: []const u8 },
|
other: struct {
|
||||||
|
type: []const u8,
|
||||||
|
sub_type: []const u8,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
pub fn contentTypeString(mime: *const Mime) []const u8 {
|
pub fn contentTypeString(mime: *const Mime) []const u8 {
|
||||||
@@ -112,17 +118,18 @@ fn parseCharset(value: []const u8) error{ CharsetTooBig, Invalid }![]const u8 {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parse(input: []u8) !Mime {
|
pub fn parse(input: []const u8) !Mime {
|
||||||
if (input.len > 255) {
|
if (input.len > 255) {
|
||||||
return error.TooBig;
|
return error.TooBig;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Zig's trim API is broken. The return type is always `[]const u8`,
|
var buf: [255]u8 = undefined;
|
||||||
// even if the input type is `[]u8`. @constCast is safe here.
|
const normalized = std.ascii.lowerString(&buf, std.mem.trim(u8, input, &std.ascii.whitespace));
|
||||||
var normalized = @constCast(std.mem.trim(u8, input, &std.ascii.whitespace));
|
|
||||||
_ = std.ascii.lowerString(normalized, normalized);
|
_ = std.ascii.lowerString(normalized, normalized);
|
||||||
|
|
||||||
const content_type, const type_len = try parseContentType(normalized);
|
var mime = Mime{ .content_type = undefined };
|
||||||
|
|
||||||
|
const content_type, const type_len = try parseContentType(normalized, &mime.type_buf, &mime.sub_type_buf);
|
||||||
if (type_len >= normalized.len) {
|
if (type_len >= normalized.len) {
|
||||||
return .{ .content_type = content_type };
|
return .{ .content_type = content_type };
|
||||||
}
|
}
|
||||||
@@ -163,13 +170,12 @@ pub fn parse(input: []u8) !Mime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return .{
|
mime.params = params;
|
||||||
.params = params,
|
mime.charset = charset;
|
||||||
.charset = charset,
|
mime.charset_len = charset_len;
|
||||||
.charset_len = charset_len,
|
mime.content_type = content_type;
|
||||||
.content_type = content_type,
|
mime.is_default_charset = !has_explicit_charset;
|
||||||
.is_default_charset = !has_explicit_charset,
|
return mime;
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Prescan the first 1024 bytes of an HTML document for a charset declaration.
|
/// Prescan the first 1024 bytes of an HTML document for a charset declaration.
|
||||||
@@ -387,7 +393,7 @@ pub fn isHTML(self: *const Mime) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// we expect value to be lowercase
|
// we expect value to be lowercase
|
||||||
fn parseContentType(value: []const u8) !struct { ContentType, usize } {
|
fn parseContentType(value: []const u8, type_buf: []u8, sub_type_buf: []u8) !struct { ContentType, usize } {
|
||||||
const end = std.mem.indexOfScalarPos(u8, value, 0, ';') orelse value.len;
|
const end = std.mem.indexOfScalarPos(u8, value, 0, ';') orelse value.len;
|
||||||
const type_name = trimRight(value[0..end]);
|
const type_name = trimRight(value[0..end]);
|
||||||
const attribute_start = end + 1;
|
const attribute_start = end + 1;
|
||||||
@@ -436,10 +442,18 @@ fn parseContentType(value: []const u8) !struct { ContentType, usize } {
|
|||||||
return error.Invalid;
|
return error.Invalid;
|
||||||
}
|
}
|
||||||
|
|
||||||
return .{ .{ .other = .{
|
@memcpy(type_buf[0..main_type.len], main_type);
|
||||||
.type = main_type,
|
@memcpy(sub_type_buf[0..sub_type.len], sub_type);
|
||||||
.sub_type = sub_type,
|
|
||||||
} }, attribute_start };
|
return .{
|
||||||
|
.{
|
||||||
|
.other = .{
|
||||||
|
.type = type_buf[0..main_type.len],
|
||||||
|
.sub_type = sub_type_buf[0..sub_type.len],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
attribute_start,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const VALID_CODEPOINTS = blk: {
|
const VALID_CODEPOINTS = blk: {
|
||||||
@@ -453,6 +467,13 @@ const VALID_CODEPOINTS = blk: {
|
|||||||
break :blk v;
|
break :blk v;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub fn typeString(self: *const Mime) []const u8 {
|
||||||
|
return switch (self.content_type) {
|
||||||
|
.other => |o| o.type[0..o.type_len],
|
||||||
|
else => "",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
fn validType(value: []const u8) bool {
|
fn validType(value: []const u8) bool {
|
||||||
for (value) |b| {
|
for (value) |b| {
|
||||||
if (VALID_CODEPOINTS[b] == false) {
|
if (VALID_CODEPOINTS[b] == false) {
|
||||||
|
|||||||
@@ -821,12 +821,10 @@ fn notifyParentLoadComplete(self: *Page) void {
|
|||||||
parent.iframeCompletedLoading(self.iframe.?);
|
parent.iframeCompletedLoading(self.iframe.?);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
fn pageHeaderDoneCallback(response: HttpClient.Response) !bool {
|
||||||
var self: *Page = @ptrCast(@alignCast(transfer.ctx));
|
var self: *Page = @ptrCast(@alignCast(response.ctx));
|
||||||
|
|
||||||
const header = &transfer.response_header.?;
|
const response_url = response.url();
|
||||||
|
|
||||||
const response_url = std.mem.span(header.url);
|
|
||||||
if (std.mem.eql(u8, response_url, self.url) == false) {
|
if (std.mem.eql(u8, response_url, self.url) == false) {
|
||||||
// would be different than self.url in the case of a redirect
|
// would be different than self.url in the case of a redirect
|
||||||
self.url = try self.arena.dupeZ(u8, response_url);
|
self.url = try self.arena.dupeZ(u8, response_url);
|
||||||
@@ -840,8 +838,8 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
|||||||
if (comptime IS_DEBUG) {
|
if (comptime IS_DEBUG) {
|
||||||
log.debug(.page, "navigate header", .{
|
log.debug(.page, "navigate header", .{
|
||||||
.url = self.url,
|
.url = self.url,
|
||||||
.status = header.status,
|
.status = response.status(),
|
||||||
.content_type = header.contentType(),
|
.content_type = response.contentType(),
|
||||||
.type = self._type,
|
.type = self._type,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -849,14 +847,14 @@ fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
fn pageDataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||||
var self: *Page = @ptrCast(@alignCast(transfer.ctx));
|
var self: *Page = @ptrCast(@alignCast(response.ctx));
|
||||||
|
|
||||||
if (self._parse_state == .pre) {
|
if (self._parse_state == .pre) {
|
||||||
// we lazily do this, because we might need the first chunk of data
|
// we lazily do this, because we might need the first chunk of data
|
||||||
// to sniff the content type
|
// to sniff the content type
|
||||||
var mime: Mime = blk: {
|
var mime: Mime = blk: {
|
||||||
if (transfer.response_header.?.contentType()) |ct| {
|
if (response.contentType()) |ct| {
|
||||||
break :blk try Mime.parse(ct);
|
break :blk try Mime.parse(ct);
|
||||||
}
|
}
|
||||||
break :blk Mime.sniff(data);
|
break :blk Mime.sniff(data);
|
||||||
|
|||||||
@@ -694,85 +694,86 @@ pub const Script = struct {
|
|||||||
self.manager.page.releaseArena(self.arena);
|
self.manager.page.releaseArena(self.arena);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn startCallback(transfer: *HttpClient.Transfer) !void {
|
fn startCallback(response: HttpClient.Response) !void {
|
||||||
log.debug(.http, "script fetch start", .{ .req = transfer });
|
log.debug(.http, "script fetch start", .{ .req = response });
|
||||||
}
|
}
|
||||||
|
|
||||||
fn headerCallback(transfer: *HttpClient.Transfer) !bool {
|
fn headerCallback(response: HttpClient.Response) !bool {
|
||||||
const self: *Script = @ptrCast(@alignCast(transfer.ctx));
|
const self: *Script = @ptrCast(@alignCast(response.ctx));
|
||||||
const header = &transfer.response_header.?;
|
|
||||||
self.status = header.status;
|
self.status = response.status().?;
|
||||||
if (header.status != 200) {
|
if (response.status() != 200) {
|
||||||
log.info(.http, "script header", .{
|
log.info(.http, "script header", .{
|
||||||
.req = transfer,
|
.req = response,
|
||||||
.status = header.status,
|
.status = response.status(),
|
||||||
.content_type = header.contentType(),
|
.content_type = response.contentType(),
|
||||||
});
|
});
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (comptime IS_DEBUG) {
|
if (comptime IS_DEBUG) {
|
||||||
log.debug(.http, "script header", .{
|
log.debug(.http, "script header", .{
|
||||||
.req = transfer,
|
.req = response,
|
||||||
.status = header.status,
|
.status = response.status(),
|
||||||
.content_type = header.contentType(),
|
.content_type = response.contentType(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
// {
|
||||||
// temp debug, trying to figure out why the next assert sometimes
|
// // temp debug, trying to figure out why the next assert sometimes
|
||||||
// fails. Is the buffer just corrupt or is headerCallback really
|
// // fails. Is the buffer just corrupt or is headerCallback really
|
||||||
// being called twice?
|
// // being called twice?
|
||||||
lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{
|
// lp.assert(self.header_callback_called == false, "ScriptManager.Header recall", .{
|
||||||
.m = @tagName(std.meta.activeTag(self.mode)),
|
// .m = @tagName(std.meta.activeTag(self.mode)),
|
||||||
.a1 = self.debug_transfer_id,
|
// .a1 = self.debug_transfer_id,
|
||||||
.a2 = self.debug_transfer_tries,
|
// .a2 = self.debug_transfer_tries,
|
||||||
.a3 = self.debug_transfer_aborted,
|
// .a3 = self.debug_transfer_aborted,
|
||||||
.a4 = self.debug_transfer_bytes_received,
|
// .a4 = self.debug_transfer_bytes_received,
|
||||||
.a5 = self.debug_transfer_notified_fail,
|
// .a5 = self.debug_transfer_notified_fail,
|
||||||
.a6 = self.debug_transfer_redirecting,
|
// .a6 = self.debug_transfer_redirecting,
|
||||||
.a7 = self.debug_transfer_intercept_state,
|
// .a7 = self.debug_transfer_intercept_state,
|
||||||
.a8 = self.debug_transfer_auth_challenge,
|
// .a8 = self.debug_transfer_auth_challenge,
|
||||||
.a9 = self.debug_transfer_easy_id,
|
// .a9 = self.debug_transfer_easy_id,
|
||||||
.b1 = transfer.id,
|
// .b1 = transfer.id,
|
||||||
.b2 = transfer._tries,
|
// .b2 = transfer._tries,
|
||||||
.b3 = transfer.aborted,
|
// .b3 = transfer.aborted,
|
||||||
.b4 = transfer.bytes_received,
|
// .b4 = transfer.bytes_received,
|
||||||
.b5 = transfer._notified_fail,
|
// .b5 = transfer._notified_fail,
|
||||||
.b6 = transfer._redirecting,
|
// .b6 = transfer._redirecting,
|
||||||
.b7 = @intFromEnum(transfer._intercept_state),
|
// .b7 = @intFromEnum(transfer._intercept_state),
|
||||||
.b8 = transfer._auth_challenge != null,
|
// .b8 = transfer._auth_challenge != null,
|
||||||
.b9 = if (transfer._conn) |c| @intFromPtr(c.easy) else 0,
|
// .b9 = if (transfer._conn) |c| @intFromPtr(c.easy) else 0,
|
||||||
});
|
// });
|
||||||
self.header_callback_called = true;
|
// self.header_callback_called = true;
|
||||||
self.debug_transfer_id = transfer.id;
|
// self.debug_transfer_id = transfer.id;
|
||||||
self.debug_transfer_tries = transfer._tries;
|
// self.debug_transfer_tries = transfer._tries;
|
||||||
self.debug_transfer_aborted = transfer.aborted;
|
// self.debug_transfer_aborted = transfer.aborted;
|
||||||
self.debug_transfer_bytes_received = transfer.bytes_received;
|
// self.debug_transfer_bytes_received = transfer.bytes_received;
|
||||||
self.debug_transfer_notified_fail = transfer._notified_fail;
|
// self.debug_transfer_notified_fail = transfer._notified_fail;
|
||||||
self.debug_transfer_redirecting = transfer._redirecting;
|
// self.debug_transfer_redirecting = transfer._redirecting;
|
||||||
self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
|
// self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
|
||||||
self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
|
// self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
|
||||||
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c.easy) else 0;
|
// self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c.easy) else 0;
|
||||||
}
|
// }
|
||||||
|
|
||||||
lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity });
|
lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity });
|
||||||
var buffer: std.ArrayList(u8) = .empty;
|
var buffer: std.ArrayList(u8) = .empty;
|
||||||
if (transfer.getContentLength()) |cl| {
|
if (response.contentLength()) |cl| {
|
||||||
try buffer.ensureTotalCapacity(self.arena, cl);
|
try buffer.ensureTotalCapacity(self.arena, cl);
|
||||||
}
|
}
|
||||||
self.source = .{ .remote = buffer };
|
self.source = .{ .remote = buffer };
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
fn dataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||||
const self: *Script = @ptrCast(@alignCast(transfer.ctx));
|
const self: *Script = @ptrCast(@alignCast(response.ctx));
|
||||||
self._dataCallback(transfer, data) catch |err| {
|
self._dataCallback(response, data) catch |err| {
|
||||||
log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len });
|
log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = response, .len = data.len });
|
||||||
return err;
|
return err;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
fn _dataCallback(self: *Script, _: *HttpClient.Transfer, data: []const u8) !void {
|
|
||||||
|
fn _dataCallback(self: *Script, _: HttpClient.Response, data: []const u8) !void {
|
||||||
try self.source.remote.appendSlice(self.arena, data);
|
try self.source.remote.appendSlice(self.arena, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
<div id="host2"></div>
|
<div id="host2"></div>
|
||||||
<div id="host3"></div>
|
<div id="host3"></div>
|
||||||
|
|
||||||
<!-- <script id="attachShadow_open">
|
<script id="attachShadow_open">
|
||||||
{
|
{
|
||||||
const host = $('#host1');
|
const host = $('#host1');
|
||||||
const shadow = host.attachShadow({ mode: 'open' });
|
const shadow = host.attachShadow({ mode: 'open' });
|
||||||
@@ -140,7 +140,7 @@
|
|||||||
shadow.replaceChildren('New content');
|
shadow.replaceChildren('New content');
|
||||||
testing.expectEqual('New content', shadow.innerHTML);
|
testing.expectEqual('New content', shadow.innerHTML);
|
||||||
}
|
}
|
||||||
</script> -->
|
</script>
|
||||||
|
|
||||||
<script id="getElementById">
|
<script id="getElementById">
|
||||||
{
|
{
|
||||||
@@ -154,3 +154,16 @@
|
|||||||
testing.expectEqual(null, shadow.getElementById('nonexistent'));
|
testing.expectEqual(null, shadow.getElementById('nonexistent'));
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|
||||||
|
<script id=adoptedStyleSheets>
|
||||||
|
{
|
||||||
|
const host = document.createElement('div');
|
||||||
|
const shadow = host.attachShadow({ mode: 'open' });
|
||||||
|
|
||||||
|
const acss = shadow.adoptedStyleSheets;
|
||||||
|
testing.expectEqual(0, acss.length);
|
||||||
|
acss.push(new CSSStyleSheet());
|
||||||
|
testing.expectEqual(1, acss.length);
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ _mode: Mode,
|
|||||||
_host: *Element,
|
_host: *Element,
|
||||||
_elements_by_id: std.StringHashMapUnmanaged(*Element) = .{},
|
_elements_by_id: std.StringHashMapUnmanaged(*Element) = .{},
|
||||||
_removed_ids: std.StringHashMapUnmanaged(void) = .{},
|
_removed_ids: std.StringHashMapUnmanaged(void) = .{},
|
||||||
|
_adopted_style_sheets: ?js.Object.Global = null,
|
||||||
|
|
||||||
pub fn init(host: *Element, mode: Mode, page: *Page) !*ShadowRoot {
|
pub fn init(host: *Element, mode: Mode, page: *Page) !*ShadowRoot {
|
||||||
return page._factory.documentFragment(ShadowRoot{
|
return page._factory.documentFragment(ShadowRoot{
|
||||||
@@ -99,6 +100,20 @@ pub fn getElementById(self: *ShadowRoot, id: []const u8, page: *Page) ?*Element
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn getAdoptedStyleSheets(self: *ShadowRoot, page: *Page) !js.Object.Global {
|
||||||
|
if (self._adopted_style_sheets) |ass| {
|
||||||
|
return ass;
|
||||||
|
}
|
||||||
|
const js_arr = page.js.local.?.newArray(0);
|
||||||
|
const js_obj = js_arr.toObject();
|
||||||
|
self._adopted_style_sheets = try js_obj.persist();
|
||||||
|
return self._adopted_style_sheets.?;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn setAdoptedStyleSheets(self: *ShadowRoot, sheets: js.Object) !void {
|
||||||
|
self._adopted_style_sheets = try sheets.persist();
|
||||||
|
}
|
||||||
|
|
||||||
pub const JsApi = struct {
|
pub const JsApi = struct {
|
||||||
pub const bridge = js.Bridge(ShadowRoot);
|
pub const bridge = js.Bridge(ShadowRoot);
|
||||||
|
|
||||||
@@ -121,6 +136,7 @@ pub const JsApi = struct {
|
|||||||
}
|
}
|
||||||
return self.getElementById(try value.toZig([]const u8), page);
|
return self.getElementById(try value.toZig([]const u8), page);
|
||||||
}
|
}
|
||||||
|
pub const adoptedStyleSheets = bridge.accessor(ShadowRoot.getAdoptedStyleSheets, ShadowRoot.setAdoptedStyleSheets, .{});
|
||||||
};
|
};
|
||||||
|
|
||||||
const testing = @import("../../testing.zig");
|
const testing = @import("../../testing.zig");
|
||||||
|
|||||||
@@ -126,16 +126,16 @@ fn handleBlobUrl(url: []const u8, resolver: js.PromiseResolver, page: *Page) !js
|
|||||||
return resolver.promise();
|
return resolver.promise();
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
|
fn httpStartCallback(response: HttpClient.Response) !void {
|
||||||
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx));
|
const self: *Fetch = @ptrCast(@alignCast(response.ctx));
|
||||||
if (comptime IS_DEBUG) {
|
if (comptime IS_DEBUG) {
|
||||||
log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" });
|
log.debug(.http, "request start", .{ .url = self._url, .source = "fetch" });
|
||||||
}
|
}
|
||||||
self._response._transfer = transfer;
|
self._response._http_response = response;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
fn httpHeaderDoneCallback(response: HttpClient.Response) !bool {
|
||||||
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx));
|
const self: *Fetch = @ptrCast(@alignCast(response.ctx));
|
||||||
|
|
||||||
if (self._signal) |signal| {
|
if (self._signal) |signal| {
|
||||||
if (signal._aborted) {
|
if (signal._aborted) {
|
||||||
@@ -144,25 +144,24 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const arena = self._response._arena;
|
const arena = self._response._arena;
|
||||||
if (transfer.getContentLength()) |cl| {
|
if (response.contentLength()) |cl| {
|
||||||
try self._buf.ensureTotalCapacity(arena, cl);
|
try self._buf.ensureTotalCapacity(arena, cl);
|
||||||
}
|
}
|
||||||
|
|
||||||
const res = self._response;
|
const res = self._response;
|
||||||
const header = transfer.response_header.?;
|
|
||||||
|
|
||||||
if (comptime IS_DEBUG) {
|
if (comptime IS_DEBUG) {
|
||||||
log.debug(.http, "request header", .{
|
log.debug(.http, "request header", .{
|
||||||
.source = "fetch",
|
.source = "fetch",
|
||||||
.url = self._url,
|
.url = self._url,
|
||||||
.status = header.status,
|
.status = response.status(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
res._status = header.status;
|
res._status = response.status().?;
|
||||||
res._status_text = std.http.Status.phrase(@enumFromInt(header.status)) orelse "";
|
res._status_text = std.http.Status.phrase(@enumFromInt(response.status().?)) orelse "";
|
||||||
res._url = try arena.dupeZ(u8, std.mem.span(header.url));
|
res._url = try arena.dupeZ(u8, response.url());
|
||||||
res._is_redirected = header.redirect_count > 0;
|
res._is_redirected = response.redirectCount().? > 0;
|
||||||
|
|
||||||
// Determine response type based on origin comparison
|
// Determine response type based on origin comparison
|
||||||
const page_origin = URL.getOrigin(arena, self._page.url) catch null;
|
const page_origin = URL.getOrigin(arena, self._page.url) catch null;
|
||||||
@@ -182,7 +181,7 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
|||||||
res._type = .basic;
|
res._type = .basic;
|
||||||
}
|
}
|
||||||
|
|
||||||
var it = transfer.responseHeaderIterator();
|
var it = response.headerIterator();
|
||||||
while (it.next()) |hdr| {
|
while (it.next()) |hdr| {
|
||||||
try res._headers.append(hdr.name, hdr.value, self._page);
|
try res._headers.append(hdr.name, hdr.value, self._page);
|
||||||
}
|
}
|
||||||
@@ -190,8 +189,8 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||||
const self: *Fetch = @ptrCast(@alignCast(transfer.ctx));
|
const self: *Fetch = @ptrCast(@alignCast(response.ctx));
|
||||||
|
|
||||||
// Check if aborted
|
// Check if aborted
|
||||||
if (self._signal) |signal| {
|
if (self._signal) |signal| {
|
||||||
@@ -206,7 +205,7 @@ fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
|||||||
fn httpDoneCallback(ctx: *anyopaque) !void {
|
fn httpDoneCallback(ctx: *anyopaque) !void {
|
||||||
const self: *Fetch = @ptrCast(@alignCast(ctx));
|
const self: *Fetch = @ptrCast(@alignCast(ctx));
|
||||||
var response = self._response;
|
var response = self._response;
|
||||||
response._transfer = null;
|
response._http_response = null;
|
||||||
response._body = self._buf.items;
|
response._body = self._buf.items;
|
||||||
|
|
||||||
log.info(.http, "request complete", .{
|
log.info(.http, "request complete", .{
|
||||||
@@ -229,7 +228,7 @@ fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
|
|||||||
const self: *Fetch = @ptrCast(@alignCast(ctx));
|
const self: *Fetch = @ptrCast(@alignCast(ctx));
|
||||||
|
|
||||||
var response = self._response;
|
var response = self._response;
|
||||||
response._transfer = null;
|
response._http_response = null;
|
||||||
// the response is only passed on v8 on success, if we're here, it's safe to
|
// the response is only passed on v8 on success, if we're here, it's safe to
|
||||||
// clear this. (defer since `self is in the response's arena).
|
// clear this. (defer since `self is in the response's arena).
|
||||||
|
|
||||||
@@ -255,7 +254,7 @@ fn httpShutdownCallback(ctx: *anyopaque) void {
|
|||||||
|
|
||||||
if (self._owns_response) {
|
if (self._owns_response) {
|
||||||
var response = self._response;
|
var response = self._response;
|
||||||
response._transfer = null;
|
response._http_response = null;
|
||||||
response.deinit(true, self._page._session);
|
response.deinit(true, self._page._session);
|
||||||
// Do not access `self` after this point: the Fetch struct was
|
// Do not access `self` after this point: the Fetch struct was
|
||||||
// allocated from response._arena which has been released.
|
// allocated from response._arena which has been released.
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ _type: Type,
|
|||||||
_status_text: []const u8,
|
_status_text: []const u8,
|
||||||
_url: [:0]const u8,
|
_url: [:0]const u8,
|
||||||
_is_redirected: bool,
|
_is_redirected: bool,
|
||||||
_transfer: ?*HttpClient.Transfer = null,
|
_http_response: ?HttpClient.Response = null,
|
||||||
|
|
||||||
const InitOpts = struct {
|
const InitOpts = struct {
|
||||||
status: u16 = 200,
|
status: u16 = 200,
|
||||||
@@ -79,13 +79,13 @@ pub fn init(body_: ?[]const u8, opts_: ?InitOpts, page: *Page) !*Response {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn deinit(self: *Response, shutdown: bool, session: *Session) void {
|
pub fn deinit(self: *Response, shutdown: bool, session: *Session) void {
|
||||||
if (self._transfer) |transfer| {
|
if (self._http_response) |resp| {
|
||||||
if (shutdown) {
|
if (shutdown) {
|
||||||
transfer.terminate();
|
resp.terminate();
|
||||||
} else {
|
} else {
|
||||||
transfer.abort(error.Abort);
|
resp.abort(error.Abort);
|
||||||
}
|
}
|
||||||
self._transfer = null;
|
self._http_response = null;
|
||||||
}
|
}
|
||||||
session.releaseArena(self._arena);
|
session.releaseArena(self._arena);
|
||||||
}
|
}
|
||||||
@@ -185,7 +185,7 @@ pub fn clone(self: *const Response, page: *Page) !*Response {
|
|||||||
._type = self._type,
|
._type = self._type,
|
||||||
._is_redirected = self._is_redirected,
|
._is_redirected = self._is_redirected,
|
||||||
._headers = try Headers.init(.{ .obj = self._headers }, page),
|
._headers = try Headers.init(.{ .obj = self._headers }, page),
|
||||||
._transfer = null,
|
._http_response = null,
|
||||||
};
|
};
|
||||||
return cloned;
|
return cloned;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ const XMLHttpRequest = @This();
|
|||||||
_page: *Page,
|
_page: *Page,
|
||||||
_proto: *XMLHttpRequestEventTarget,
|
_proto: *XMLHttpRequestEventTarget,
|
||||||
_arena: Allocator,
|
_arena: Allocator,
|
||||||
_transfer: ?*HttpClient.Transfer = null,
|
_http_response: ?HttpClient.Response = null,
|
||||||
|
|
||||||
_url: [:0]const u8 = "",
|
_url: [:0]const u8 = "",
|
||||||
_method: net_http.Method = .GET,
|
_method: net_http.Method = .GET,
|
||||||
@@ -97,13 +97,13 @@ pub fn init(page: *Page) !*XMLHttpRequest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn deinit(self: *XMLHttpRequest, shutdown: bool, session: *Session) void {
|
pub fn deinit(self: *XMLHttpRequest, shutdown: bool, session: *Session) void {
|
||||||
if (self._transfer) |transfer| {
|
if (self._http_response) |transfer| {
|
||||||
if (shutdown) {
|
if (shutdown) {
|
||||||
transfer.terminate();
|
transfer.terminate();
|
||||||
} else {
|
} else {
|
||||||
transfer.abort(error.Abort);
|
transfer.abort(error.Abort);
|
||||||
}
|
}
|
||||||
self._transfer = null;
|
self._http_response = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (self._on_ready_state_change) |func| {
|
if (self._on_ready_state_change) |func| {
|
||||||
@@ -169,9 +169,9 @@ pub fn setWithCredentials(self: *XMLHttpRequest, value: bool) !void {
|
|||||||
// TODO: url should be a union, as it can be multiple things
|
// TODO: url should be a union, as it can be multiple things
|
||||||
pub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void {
|
pub fn open(self: *XMLHttpRequest, method_: []const u8, url: [:0]const u8) !void {
|
||||||
// Abort any in-progress request
|
// Abort any in-progress request
|
||||||
if (self._transfer) |transfer| {
|
if (self._http_response) |transfer| {
|
||||||
transfer.abort(error.Abort);
|
transfer.abort(error.Abort);
|
||||||
self._transfer = null;
|
self._http_response = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset internal state
|
// Reset internal state
|
||||||
@@ -382,34 +382,32 @@ pub fn getResponseXML(self: *XMLHttpRequest, page: *Page) !?*Node.Document {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
|
fn httpStartCallback(response: HttpClient.Response) !void {
|
||||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||||
if (comptime IS_DEBUG) {
|
if (comptime IS_DEBUG) {
|
||||||
log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });
|
log.debug(.http, "request start", .{ .method = self._method, .url = self._url, .source = "xhr" });
|
||||||
}
|
}
|
||||||
self._transfer = transfer;
|
self._http_response = response;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: net_http.Header) !void {
|
fn httpHeaderCallback(response: HttpClient.Response, header: net_http.Header) !void {
|
||||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||||
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value });
|
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value });
|
||||||
try self._response_headers.append(self._arena, joined);
|
try self._response_headers.append(self._arena, joined);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
fn httpHeaderDoneCallback(response: HttpClient.Response) !bool {
|
||||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||||
|
|
||||||
const header = &transfer.response_header.?;
|
|
||||||
|
|
||||||
if (comptime IS_DEBUG) {
|
if (comptime IS_DEBUG) {
|
||||||
log.debug(.http, "request header", .{
|
log.debug(.http, "request header", .{
|
||||||
.source = "xhr",
|
.source = "xhr",
|
||||||
.url = self._url,
|
.url = self._url,
|
||||||
.status = header.status,
|
.status = response.status(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (header.contentType()) |ct| {
|
if (response.contentType()) |ct| {
|
||||||
self._response_mime = Mime.parse(ct) catch |e| {
|
self._response_mime = Mime.parse(ct) catch |e| {
|
||||||
log.info(.http, "invalid content type", .{
|
log.info(.http, "invalid content type", .{
|
||||||
.content_Type = ct,
|
.content_Type = ct,
|
||||||
@@ -420,18 +418,18 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
var it = transfer.responseHeaderIterator();
|
var it = response.headerIterator();
|
||||||
while (it.next()) |hdr| {
|
while (it.next()) |hdr| {
|
||||||
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value });
|
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ hdr.name, hdr.value });
|
||||||
try self._response_headers.append(self._arena, joined);
|
try self._response_headers.append(self._arena, joined);
|
||||||
}
|
}
|
||||||
|
|
||||||
self._response_status = header.status;
|
self._response_status = response.status().?;
|
||||||
if (transfer.getContentLength()) |cl| {
|
if (response.contentLength()) |cl| {
|
||||||
self._response_len = cl;
|
self._response_len = cl;
|
||||||
try self._response_data.ensureTotalCapacity(self._arena, cl);
|
try self._response_data.ensureTotalCapacity(self._arena, cl);
|
||||||
}
|
}
|
||||||
self._response_url = try self._arena.dupeZ(u8, std.mem.span(header.url));
|
self._response_url = try self._arena.dupeZ(u8, response.url());
|
||||||
|
|
||||||
const page = self._page;
|
const page = self._page;
|
||||||
|
|
||||||
@@ -446,8 +444,8 @@ fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
|
fn httpDataCallback(response: HttpClient.Response, data: []const u8) !void {
|
||||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
const self: *XMLHttpRequest = @ptrCast(@alignCast(response.ctx));
|
||||||
try self._response_data.appendSlice(self._arena, data);
|
try self._response_data.appendSlice(self._arena, data);
|
||||||
|
|
||||||
const page = self._page;
|
const page = self._page;
|
||||||
@@ -470,7 +468,7 @@ fn httpDoneCallback(ctx: *anyopaque) !void {
|
|||||||
|
|
||||||
// Not that the request is done, the http/client will free the transfer
|
// Not that the request is done, the http/client will free the transfer
|
||||||
// object. It isn't safe to keep it around.
|
// object. It isn't safe to keep it around.
|
||||||
self._transfer = null;
|
self._http_response = null;
|
||||||
|
|
||||||
const page = self._page;
|
const page = self._page;
|
||||||
|
|
||||||
@@ -492,21 +490,21 @@ fn httpDoneCallback(ctx: *anyopaque) !void {
|
|||||||
fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
|
fn httpErrorCallback(ctx: *anyopaque, err: anyerror) void {
|
||||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
|
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
|
||||||
// http client will close it after an error, it isn't safe to keep around
|
// http client will close it after an error, it isn't safe to keep around
|
||||||
self._transfer = null;
|
self._http_response = null;
|
||||||
self.handleError(err);
|
self.handleError(err);
|
||||||
self._page.js.weakRef(self);
|
self._page.js.weakRef(self);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpShutdownCallback(ctx: *anyopaque) void {
|
fn httpShutdownCallback(ctx: *anyopaque) void {
|
||||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
|
const self: *XMLHttpRequest = @ptrCast(@alignCast(ctx));
|
||||||
self._transfer = null;
|
self._http_response = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn abort(self: *XMLHttpRequest) void {
|
pub fn abort(self: *XMLHttpRequest) void {
|
||||||
self.handleError(error.Abort);
|
self.handleError(error.Abort);
|
||||||
if (self._transfer) |transfer| {
|
if (self._http_response) |transfer| {
|
||||||
transfer.abort(error.Abort);
|
transfer.abort(error.Abort);
|
||||||
self._transfer = null;
|
self._http_response = null;
|
||||||
}
|
}
|
||||||
self._page.js.weakRef(self);
|
self._page.js.weakRef(self);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,9 @@ const libcurl = @import("../sys/libcurl.zig");
|
|||||||
const net_http = @import("http.zig");
|
const net_http = @import("http.zig");
|
||||||
const RobotStore = @import("Robots.zig").RobotStore;
|
const RobotStore = @import("Robots.zig").RobotStore;
|
||||||
const WebBotAuth = @import("WebBotAuth.zig");
|
const WebBotAuth = @import("WebBotAuth.zig");
|
||||||
|
const Cache = @import("cache/Cache.zig");
|
||||||
|
|
||||||
|
const App = @import("../App.zig");
|
||||||
const Runtime = @This();
|
const Runtime = @This();
|
||||||
|
|
||||||
const Listener = struct {
|
const Listener = struct {
|
||||||
@@ -45,10 +47,12 @@ const MAX_TICK_CALLBACKS = 16;
|
|||||||
|
|
||||||
allocator: Allocator,
|
allocator: Allocator,
|
||||||
|
|
||||||
|
app: *App,
|
||||||
config: *const Config,
|
config: *const Config,
|
||||||
ca_blob: ?net_http.Blob,
|
ca_blob: ?net_http.Blob,
|
||||||
robot_store: RobotStore,
|
robot_store: RobotStore,
|
||||||
web_bot_auth: ?WebBotAuth,
|
web_bot_auth: ?WebBotAuth,
|
||||||
|
cache: ?Cache,
|
||||||
|
|
||||||
connections: []net_http.Connection,
|
connections: []net_http.Connection,
|
||||||
available: std.DoublyLinkedList = .{},
|
available: std.DoublyLinkedList = .{},
|
||||||
@@ -200,7 +204,7 @@ fn globalDeinit() void {
|
|||||||
libcurl.curl_global_cleanup();
|
libcurl.curl_global_cleanup();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn init(allocator: Allocator, config: *const Config) !Runtime {
|
pub fn init(allocator: Allocator, app: *App, config: *const Config) !Runtime {
|
||||||
globalInit(allocator);
|
globalInit(allocator);
|
||||||
errdefer globalDeinit();
|
errdefer globalDeinit();
|
||||||
|
|
||||||
@@ -233,6 +237,11 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime {
|
|||||||
else
|
else
|
||||||
null;
|
null;
|
||||||
|
|
||||||
|
const cache = if (config.cacheDir()) |cache_dir_path|
|
||||||
|
Cache{ .kind = .{ .fs = try .init(cache_dir_path) } }
|
||||||
|
else
|
||||||
|
null;
|
||||||
|
|
||||||
return .{
|
return .{
|
||||||
.allocator = allocator,
|
.allocator = allocator,
|
||||||
.config = config,
|
.config = config,
|
||||||
@@ -244,8 +253,10 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime {
|
|||||||
.available = available,
|
.available = available,
|
||||||
.connections = connections,
|
.connections = connections,
|
||||||
|
|
||||||
|
.app = app,
|
||||||
.robot_store = RobotStore.init(allocator),
|
.robot_store = RobotStore.init(allocator),
|
||||||
.web_bot_auth = web_bot_auth,
|
.web_bot_auth = web_bot_auth,
|
||||||
|
.cache = cache,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
173
src/network/cache/Cache.zig
vendored
Normal file
173
src/network/cache/Cache.zig
vendored
Normal file
@@ -0,0 +1,173 @@
|
|||||||
|
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||||
|
//
|
||||||
|
// Francis Bouvier <francis@lightpanda.io>
|
||||||
|
// Pierre Tachoire <pierre@lightpanda.io>
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as
|
||||||
|
// published by the Free Software Foundation, either version 3 of the
|
||||||
|
// License, or (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
const Http = @import("../http.zig");
|
||||||
|
const FsCache = @import("FsCache.zig");
|
||||||
|
|
||||||
|
/// A browser-wide cache for resources across the network.
|
||||||
|
/// This mostly conforms to RFC9111 with regards to caching behavior.
|
||||||
|
pub const Cache = @This();
|
||||||
|
|
||||||
|
kind: union(enum) {
|
||||||
|
fs: FsCache,
|
||||||
|
},
|
||||||
|
|
||||||
|
pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse {
|
||||||
|
return switch (self.kind) {
|
||||||
|
inline else => |*c| c.get(arena, req),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn put(self: *Cache, metadata: CachedMetadata, body: []const u8) !void {
|
||||||
|
return switch (self.kind) {
|
||||||
|
inline else => |*c| c.put(metadata, body),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub const CacheControl = struct {
|
||||||
|
max_age: u64,
|
||||||
|
must_revalidate: bool = false,
|
||||||
|
immutable: bool = false,
|
||||||
|
|
||||||
|
pub fn parse(value: []const u8) ?CacheControl {
|
||||||
|
var cc: CacheControl = .{ .max_age = undefined };
|
||||||
|
|
||||||
|
var max_age_set = false;
|
||||||
|
var max_s_age_set = false;
|
||||||
|
var is_public = false;
|
||||||
|
|
||||||
|
var iter = std.mem.splitScalar(u8, value, ',');
|
||||||
|
while (iter.next()) |part| {
|
||||||
|
const directive = std.mem.trim(u8, part, &std.ascii.whitespace);
|
||||||
|
if (std.ascii.eqlIgnoreCase(directive, "no-store")) {
|
||||||
|
return null;
|
||||||
|
} else if (std.ascii.eqlIgnoreCase(directive, "no-cache")) {
|
||||||
|
return null;
|
||||||
|
} else if (std.ascii.eqlIgnoreCase(directive, "must-revalidate")) {
|
||||||
|
cc.must_revalidate = true;
|
||||||
|
} else if (std.ascii.eqlIgnoreCase(directive, "immutable")) {
|
||||||
|
cc.immutable = true;
|
||||||
|
} else if (std.ascii.eqlIgnoreCase(directive, "public")) {
|
||||||
|
is_public = true;
|
||||||
|
} else if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) {
|
||||||
|
if (!max_s_age_set) {
|
||||||
|
if (std.fmt.parseInt(u64, directive[8..], 10) catch null) |max_age| {
|
||||||
|
cc.max_age = max_age;
|
||||||
|
max_age_set = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) {
|
||||||
|
if (std.fmt.parseInt(u64, directive[9..], 10) catch null) |max_age| {
|
||||||
|
cc.max_age = max_age;
|
||||||
|
max_age_set = true;
|
||||||
|
max_s_age_set = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!max_age_set) return null;
|
||||||
|
if (!is_public) return null;
|
||||||
|
if (cc.max_age == 0) return null;
|
||||||
|
|
||||||
|
return cc;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const Vary = union(enum) {
|
||||||
|
wildcard: void,
|
||||||
|
value: []const u8,
|
||||||
|
|
||||||
|
pub fn parse(value: []const u8) Vary {
|
||||||
|
if (std.mem.eql(u8, value, "*")) return .wildcard;
|
||||||
|
return .{ .value = value };
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn toString(self: Vary) []const u8 {
|
||||||
|
return switch (self) {
|
||||||
|
.wildcard => "*",
|
||||||
|
.value => |v| v,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const CachedMetadata = struct {
|
||||||
|
url: [:0]const u8,
|
||||||
|
content_type: []const u8,
|
||||||
|
|
||||||
|
status: u16,
|
||||||
|
stored_at: i64,
|
||||||
|
age_at_store: u64,
|
||||||
|
|
||||||
|
// for If-None-Match
|
||||||
|
etag: ?[]const u8,
|
||||||
|
// for If-Modified-Since
|
||||||
|
last_modified: ?[]const u8,
|
||||||
|
|
||||||
|
cache_control: CacheControl,
|
||||||
|
vary: ?Vary,
|
||||||
|
headers: []const Http.Header,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const CacheRequest = struct {
|
||||||
|
url: []const u8,
|
||||||
|
timestamp: i64,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const CachedData = union(enum) {
|
||||||
|
buffer: []const u8,
|
||||||
|
file: std.fs.File,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const CachedResponse = struct {
|
||||||
|
metadata: CachedMetadata,
|
||||||
|
data: CachedData,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub fn tryCache(
|
||||||
|
arena: std.mem.Allocator,
|
||||||
|
timestamp: i64,
|
||||||
|
url: [:0]const u8,
|
||||||
|
status: u16,
|
||||||
|
content_type: ?[]const u8,
|
||||||
|
cache_control: ?[]const u8,
|
||||||
|
vary: ?[]const u8,
|
||||||
|
etag: ?[]const u8,
|
||||||
|
last_modified: ?[]const u8,
|
||||||
|
age: ?[]const u8,
|
||||||
|
has_set_cookie: bool,
|
||||||
|
has_authorization: bool,
|
||||||
|
) !?CachedMetadata {
|
||||||
|
if (status != 200) return null;
|
||||||
|
if (has_set_cookie) return null;
|
||||||
|
if (has_authorization) return null;
|
||||||
|
const cc = CacheControl.parse(cache_control orelse return null) orelse return null;
|
||||||
|
|
||||||
|
return .{
|
||||||
|
.url = url,
|
||||||
|
.content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream",
|
||||||
|
.status = status,
|
||||||
|
.stored_at = timestamp,
|
||||||
|
.age_at_store = if (age) |a| std.fmt.parseInt(u64, a, 10) catch 0 else 0,
|
||||||
|
.cache_control = cc,
|
||||||
|
.vary = if (vary) |v| Vary.parse(v) else null,
|
||||||
|
.etag = if (etag) |e| try arena.dupe(u8, e) else null,
|
||||||
|
.last_modified = if (last_modified) |lm| try arena.dupe(u8, lm) else null,
|
||||||
|
.headers = &.{},
|
||||||
|
};
|
||||||
|
}
|
||||||
293
src/network/cache/FsCache.zig
vendored
Normal file
293
src/network/cache/FsCache.zig
vendored
Normal file
@@ -0,0 +1,293 @@
|
|||||||
|
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||||
|
//
|
||||||
|
// Francis Bouvier <francis@lightpanda.io>
|
||||||
|
// Pierre Tachoire <pierre@lightpanda.io>
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as
|
||||||
|
// published by the Free Software Foundation, either version 3 of the
|
||||||
|
// License, or (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
const Cache = @import("Cache.zig");
|
||||||
|
const Http = @import("../http.zig");
|
||||||
|
const CacheRequest = Cache.CacheRequest;
|
||||||
|
const CachedMetadata = Cache.CachedMetadata;
|
||||||
|
const CachedResponse = Cache.CachedResponse;
|
||||||
|
|
||||||
|
const CACHE_VERSION: usize = 1;
|
||||||
|
const LOCK_STRIPES = 16;
|
||||||
|
|
||||||
|
pub const FsCache = @This();
|
||||||
|
|
||||||
|
dir: std.fs.Dir,
|
||||||
|
locks: [LOCK_STRIPES]std.Thread.Mutex = .{std.Thread.Mutex{}} ** LOCK_STRIPES,
|
||||||
|
|
||||||
|
const CacheMetadataFile = struct {
|
||||||
|
version: usize,
|
||||||
|
metadata: CachedMetadata,
|
||||||
|
};
|
||||||
|
|
||||||
|
fn getLockPtr(self: *FsCache, key: *const [HASHED_KEY_LEN]u8) *std.Thread.Mutex {
|
||||||
|
const lock_idx: usize = @truncate(std.hash.Wyhash.hash(0, key) % LOCK_STRIPES);
|
||||||
|
return &self.locks[lock_idx];
|
||||||
|
}
|
||||||
|
|
||||||
|
const HASHED_KEY_LEN = 64;
|
||||||
|
const HASHED_PATH_LEN = HASHED_KEY_LEN + 5;
|
||||||
|
const HASHED_TMP_PATH_LEN = HASHED_PATH_LEN + 4;
|
||||||
|
|
||||||
|
fn hashKey(key: []const u8) [HASHED_KEY_LEN]u8 {
|
||||||
|
var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined;
|
||||||
|
std.crypto.hash.sha2.Sha256.hash(key, &digest, .{});
|
||||||
|
var hex: [HASHED_KEY_LEN]u8 = undefined;
|
||||||
|
_ = std.fmt.bufPrint(&hex, "{s}", .{std.fmt.bytesToHex(&digest, .lower)}) catch unreachable;
|
||||||
|
return hex;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn metaPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 {
|
||||||
|
var path: [HASHED_PATH_LEN]u8 = undefined;
|
||||||
|
_ = std.fmt.bufPrint(&path, "{s}.meta", .{hashed_key}) catch unreachable;
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bodyPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_PATH_LEN]u8 {
|
||||||
|
var path: [HASHED_PATH_LEN]u8 = undefined;
|
||||||
|
_ = std.fmt.bufPrint(&path, "{s}.body", .{hashed_key}) catch unreachable;
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn metaTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 {
|
||||||
|
var path: [HASHED_TMP_PATH_LEN]u8 = undefined;
|
||||||
|
_ = std.fmt.bufPrint(&path, "{s}.meta.tmp", .{hashed_key}) catch unreachable;
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bodyTmpPath(hashed_key: *const [HASHED_KEY_LEN]u8) [HASHED_TMP_PATH_LEN]u8 {
|
||||||
|
var path: [HASHED_TMP_PATH_LEN]u8 = undefined;
|
||||||
|
_ = std.fmt.bufPrint(&path, "{s}.body.tmp", .{hashed_key}) catch unreachable;
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn init(path: []const u8) !FsCache {
|
||||||
|
const cwd = std.fs.cwd();
|
||||||
|
|
||||||
|
cwd.makeDir(path) catch |err| switch (err) {
|
||||||
|
error.PathAlreadyExists => {},
|
||||||
|
else => return err,
|
||||||
|
};
|
||||||
|
|
||||||
|
const dir = try cwd.openDir(path, .{ .iterate = true });
|
||||||
|
return .{ .dir = dir };
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deinit(self: *FsCache) void {
|
||||||
|
self.dir.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get(self: *FsCache, arena: std.mem.Allocator, req: CacheRequest) ?Cache.CachedResponse {
|
||||||
|
const hashed_key = hashKey(req.url);
|
||||||
|
const meta_p = metaPath(&hashed_key);
|
||||||
|
const body_p = bodyPath(&hashed_key);
|
||||||
|
|
||||||
|
const lock = self.getLockPtr(&hashed_key);
|
||||||
|
lock.lock();
|
||||||
|
defer lock.unlock();
|
||||||
|
|
||||||
|
const meta_file = self.dir.openFile(&meta_p, .{ .mode = .read_only }) catch return null;
|
||||||
|
defer meta_file.close();
|
||||||
|
|
||||||
|
const contents = meta_file.readToEndAlloc(arena, 1 * 1024 * 1024) catch return null;
|
||||||
|
defer arena.free(contents);
|
||||||
|
|
||||||
|
const cache_file: CacheMetadataFile = std.json.parseFromSliceLeaky(
|
||||||
|
CacheMetadataFile,
|
||||||
|
arena,
|
||||||
|
contents,
|
||||||
|
.{ .allocate = .alloc_always },
|
||||||
|
) catch {
|
||||||
|
self.dir.deleteFile(&meta_p) catch {};
|
||||||
|
self.dir.deleteFile(&body_p) catch {};
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const metadata = cache_file.metadata;
|
||||||
|
|
||||||
|
if (cache_file.version != CACHE_VERSION) {
|
||||||
|
self.dir.deleteFile(&meta_p) catch {};
|
||||||
|
self.dir.deleteFile(&body_p) catch {};
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = req.timestamp;
|
||||||
|
const age = (now - metadata.stored_at) + @as(i64, @intCast(metadata.age_at_store));
|
||||||
|
if (age < 0 or @as(u64, @intCast(age)) >= metadata.cache_control.max_age) {
|
||||||
|
self.dir.deleteFile(&meta_p) catch {};
|
||||||
|
self.dir.deleteFile(&body_p) catch {};
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const body_file = self.dir.openFile(
|
||||||
|
&body_p,
|
||||||
|
.{ .mode = .read_only },
|
||||||
|
) catch return null;
|
||||||
|
|
||||||
|
return .{
|
||||||
|
.metadata = metadata,
|
||||||
|
.data = .{ .file = body_file },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn put(self: *FsCache, meta: CachedMetadata, body: []const u8) !void {
|
||||||
|
const hashed_key = hashKey(meta.url);
|
||||||
|
const meta_p = metaPath(&hashed_key);
|
||||||
|
const meta_tmp_p = metaTmpPath(&hashed_key);
|
||||||
|
const body_p = bodyPath(&hashed_key);
|
||||||
|
const body_tmp_p = bodyTmpPath(&hashed_key);
|
||||||
|
var writer_buf: [512]u8 = undefined;
|
||||||
|
|
||||||
|
const lock = self.getLockPtr(&hashed_key);
|
||||||
|
lock.lock();
|
||||||
|
defer lock.unlock();
|
||||||
|
|
||||||
|
{
|
||||||
|
const meta_file = try self.dir.createFile(&meta_tmp_p, .{});
|
||||||
|
errdefer {
|
||||||
|
meta_file.close();
|
||||||
|
self.dir.deleteFile(&meta_tmp_p) catch {};
|
||||||
|
}
|
||||||
|
|
||||||
|
var meta_file_writer = meta_file.writer(&writer_buf);
|
||||||
|
const meta_file_writer_iface = &meta_file_writer.interface;
|
||||||
|
try std.json.Stringify.value(
|
||||||
|
CacheMetadataFile{ .version = CACHE_VERSION, .metadata = meta },
|
||||||
|
.{ .whitespace = .minified },
|
||||||
|
meta_file_writer_iface,
|
||||||
|
);
|
||||||
|
try meta_file_writer_iface.flush();
|
||||||
|
meta_file.close();
|
||||||
|
}
|
||||||
|
errdefer self.dir.deleteFile(&meta_tmp_p) catch {};
|
||||||
|
try self.dir.rename(&meta_tmp_p, &meta_p);
|
||||||
|
|
||||||
|
{
|
||||||
|
const body_file = try self.dir.createFile(&body_tmp_p, .{});
|
||||||
|
errdefer {
|
||||||
|
body_file.close();
|
||||||
|
self.dir.deleteFile(&body_tmp_p) catch {};
|
||||||
|
}
|
||||||
|
|
||||||
|
var body_file_writer = body_file.writer(&writer_buf);
|
||||||
|
const body_file_writer_iface = &body_file_writer.interface;
|
||||||
|
try body_file_writer_iface.writeAll(body);
|
||||||
|
try body_file_writer_iface.flush();
|
||||||
|
body_file.close();
|
||||||
|
}
|
||||||
|
errdefer self.dir.deleteFile(&body_tmp_p) catch {};
|
||||||
|
|
||||||
|
errdefer self.dir.deleteFile(&meta_p) catch {};
|
||||||
|
try self.dir.rename(&body_tmp_p, &body_p);
|
||||||
|
}
|
||||||
|
|
||||||
|
const testing = std.testing;
|
||||||
|
|
||||||
|
test "FsCache: basic put and get" {
|
||||||
|
var tmp = testing.tmpDir(.{});
|
||||||
|
defer tmp.cleanup();
|
||||||
|
|
||||||
|
const path = try tmp.dir.realpathAlloc(testing.allocator, ".");
|
||||||
|
defer testing.allocator.free(path);
|
||||||
|
|
||||||
|
var fs_cache = try FsCache.init(path);
|
||||||
|
defer fs_cache.deinit();
|
||||||
|
var cache = Cache{ .kind = .{ .fs = fs_cache } };
|
||||||
|
|
||||||
|
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||||
|
defer arena.deinit();
|
||||||
|
|
||||||
|
const now = std.time.timestamp();
|
||||||
|
const meta = CachedMetadata{
|
||||||
|
.url = "https://example.com",
|
||||||
|
.content_type = "text/html",
|
||||||
|
.status = 200,
|
||||||
|
.stored_at = now,
|
||||||
|
.age_at_store = 0,
|
||||||
|
.etag = null,
|
||||||
|
.last_modified = null,
|
||||||
|
.cache_control = .{ .max_age = 600 },
|
||||||
|
.vary = null,
|
||||||
|
.headers = &.{},
|
||||||
|
};
|
||||||
|
|
||||||
|
const body = "hello world";
|
||||||
|
try cache.put(meta, body);
|
||||||
|
|
||||||
|
const result = cache.get(arena.allocator(), .{ .url = "https://example.com", .timestamp = now }) orelse return error.CacheMiss;
|
||||||
|
defer result.data.file.close();
|
||||||
|
|
||||||
|
var buf: [64]u8 = undefined;
|
||||||
|
var file_reader = result.data.file.reader(&buf);
|
||||||
|
|
||||||
|
const read_buf = try file_reader.interface.allocRemaining(testing.allocator, .unlimited);
|
||||||
|
defer testing.allocator.free(read_buf);
|
||||||
|
|
||||||
|
try testing.expectEqualStrings(body, read_buf);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "FsCache: get expiration" {
|
||||||
|
var tmp = testing.tmpDir(.{});
|
||||||
|
defer tmp.cleanup();
|
||||||
|
|
||||||
|
const path = try tmp.dir.realpathAlloc(testing.allocator, ".");
|
||||||
|
defer testing.allocator.free(path);
|
||||||
|
|
||||||
|
var fs_cache = try FsCache.init(path);
|
||||||
|
defer fs_cache.deinit();
|
||||||
|
var cache = Cache{ .kind = .{ .fs = fs_cache } };
|
||||||
|
|
||||||
|
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||||
|
defer arena.deinit();
|
||||||
|
|
||||||
|
const now = 5000;
|
||||||
|
const max_age = 1000;
|
||||||
|
|
||||||
|
const meta = CachedMetadata{
|
||||||
|
.url = "https://example.com",
|
||||||
|
.content_type = "text/html",
|
||||||
|
.status = 200,
|
||||||
|
.stored_at = now,
|
||||||
|
.age_at_store = 900,
|
||||||
|
.etag = null,
|
||||||
|
.last_modified = null,
|
||||||
|
.cache_control = .{ .max_age = max_age },
|
||||||
|
.vary = null,
|
||||||
|
.headers = &.{},
|
||||||
|
};
|
||||||
|
|
||||||
|
const body = "hello world";
|
||||||
|
try cache.put(meta, body);
|
||||||
|
|
||||||
|
const result = cache.get(
|
||||||
|
arena.allocator(),
|
||||||
|
.{ .url = "https://example.com", .timestamp = now + 50 },
|
||||||
|
) orelse return error.CacheMiss;
|
||||||
|
result.data.file.close();
|
||||||
|
|
||||||
|
try testing.expectEqual(null, cache.get(
|
||||||
|
arena.allocator(),
|
||||||
|
.{ .url = "https://example.com", .timestamp = now + 200 },
|
||||||
|
));
|
||||||
|
|
||||||
|
try testing.expectEqual(null, cache.get(
|
||||||
|
arena.allocator(),
|
||||||
|
.{ .url = "https://example.com", .timestamp = now },
|
||||||
|
));
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user