fix WebBotAuthLayer crash

src/Server.zig

@@ -260,14 +260,14 @@ pub const Client = struct {
 
     fn start(self: *Client) void {
         const http = self.http;
-        http.cdp_client = .{
+        http.setCdpClient(.{
             .socket = self.ws.socket,
             .ctx = self,
             .blocking_read_start = Client.blockingReadStart,
             .blocking_read = Client.blockingRead,
             .blocking_read_end = Client.blockingReadStop,
-        };
-        defer http.cdp_client = null;
+        });
+        defer http.setCdpClient(null);
 
         self.httpLoop(http) catch |err| {
             log.err(.app, "CDP client loop", .{ .err = err });

src/browser/Runner.zig

@@ -135,7 +135,7 @@ fn _tick(self: *Runner, comptime is_cdp: bool, opts: TickOpts) !CDPTickResult {
         .pre, .raw, .text, .image => {
             // The main page hasn't started/finished navigating.
             // There's no JS to run, and no reason to run the scheduler.
-            if (http_client.active == 0 and (comptime is_cdp) == false) {
+            if (http_client.active() == 0 and (comptime is_cdp) == false) {
                 // haven't started navigating, I guess.
                 return .done;
             }
@@ -169,8 +169,8 @@ fn _tick(self: *Runner, comptime is_cdp: bool, opts: TickOpts) !CDPTickResult {
             // Each call to this runs scheduled load events.
             try page.dispatchLoad();
 
-            const http_active = http_client.active;
-            const total_network_activity = http_active + http_client.intercepted;
+            const http_active = http_client.active();
+            const total_network_activity = http_active + http_client.intercepted();
             if (page._notified_network_almost_idle.check(total_network_activity <= 2)) {
                 page.notifyNetworkAlmostIdle();
             }
@@ -183,7 +183,7 @@ fn _tick(self: *Runner, comptime is_cdp: bool, opts: TickOpts) !CDPTickResult {
                 // because is_cdp is true, and that can only be
                 // the case when interception isn't possible.
                 if (comptime IS_DEBUG) {
-                    std.debug.assert(http_client.intercepted == 0);
+                    std.debug.assert(http_client.intercepted() == 0);
                 }
 
                 if (browser.hasBackgroundTasks()) {

src/cdp/domains/page.zig

@@ -139,8 +139,8 @@ fn setLifecycleEventsEnabled(cmd: *CDP.Command) !void {
         try sendPageLifecycle(bc, "load", now, frame_id, loader_id);
 
         const http_client = page._session.browser.http_client;
-        const http_active = http_client.active;
-        const total_network_activity = http_active + http_client.intercepted;
+        const http_active = http_client.active();
+        const total_network_activity = http_active + http_client.intercepted();
         if (page._notified_network_almost_idle.check(total_network_activity <= 2)) {
             try sendPageLifecycle(bc, "networkAlmostIdle", now, frame_id, loader_id);
         }

src/network/layer/CacheLayer.zig

@@ -0,0 +1,240 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const log = @import("../../log.zig");
+
+const http = @import("../http.zig");
+const Transfer = @import("../../browser/HttpClient.zig").Transfer;
+const Context = @import("../../browser/HttpClient.zig").Context;
+const Request = @import("../../browser/HttpClient.zig").Request;
+const Response = @import("../../browser/HttpClient.zig").Response;
+const Layer = @import("../../browser/HttpClient.zig").Layer;
+
+const Cache = @import("../cache/Cache.zig");
+const CachedMetadata = @import("../cache/Cache.zig").CachedMetadata;
+const CachedResponse = @import("../cache/Cache.zig").CachedResponse;
+const Forward = @import("Forward.zig");
+
+const CacheLayer = @This();
+
+next: Layer = undefined,
+
+pub fn layer(self: *CacheLayer) Layer {
+    return .{
+        .ptr = self,
+        .vtable = &.{
+            .request = request,
+        },
+    };
+}
+
+fn request(ptr: *anyopaque, ctx: Context, req: Request) anyerror!void {
+    const self: *CacheLayer = @ptrCast(@alignCast(ptr));
+    const network = ctx.network;
+
+    if (network.cache == null or req.method != .GET) {
+        return self.next.request(ctx, req);
+    }
+
+    const arena = try network.app.arena_pool.acquire(.{ .debug = "CacheLayer" });
+    errdefer network.app.arena_pool.release(arena);
+
+    var iter = req.headers.iterator();
+    const req_header_list = try iter.collect(arena);
+
+    if (network.cache.?.get(arena, .{
+        .url = req.url,
+        .timestamp = std.time.timestamp(),
+        .request_headers = req_header_list.items,
+    })) |cached| {
+        defer req.headers.deinit();
+        defer network.app.arena_pool.release(arena);
+        return serveFromCache(req, &cached);
+    }
+
+    const cache_ctx = try arena.create(CacheContext);
+    cache_ctx.* = .{
+        .arena = arena,
+        .context = ctx,
+        .forward = Forward.fromRequest(req),
+        .req_url = req.url,
+        .req_headers = req.headers,
+    };
+
+    const wrapped = cache_ctx.forward.wrapRequest(
+        req,
+        cache_ctx,
+        "forward",
+        .{
+            .start = CacheContext.startCallback,
+            .header = CacheContext.headerCallback,
+            .done = CacheContext.doneCallback,
+            .shutdown = CacheContext.shutdownCallback,
+            .err = CacheContext.errorCallback,
+        },
+    );
+
+    return self.next.request(ctx, wrapped);
+}
+
+fn serveFromCache(req: Request, cached: *const CachedResponse) !void {
+    const response = Response.fromCached(req.ctx, cached);
+    defer switch (cached.data) {
+        .buffer => |_| {},
+        .file => |f| f.file.close(),
+    };
+
+    if (req.start_callback) |cb| {
+        try cb(response);
+    }
+
+    const proceed = try req.header_callback(response);
+    if (!proceed) {
+        req.error_callback(req.ctx, error.Abort);
+        return;
+    }
+
+    switch (cached.data) {
+        .buffer => |data| {
+            if (data.len > 0) {
+                try req.data_callback(response, data);
+            }
+        },
+        .file => |f| {
+            const file = f.file;
+            var buf: [1024]u8 = undefined;
+            var file_reader = file.reader(&buf);
+            try file_reader.seekTo(f.offset);
+            const reader = &file_reader.interface;
+            var read_buf: [1024]u8 = undefined;
+            var remaining = f.len;
+            while (remaining > 0) {
+                const read_len = @min(read_buf.len, remaining);
+                const n = try reader.readSliceShort(read_buf[0..read_len]);
+                if (n == 0) break;
+                remaining -= n;
+                try req.data_callback(response, read_buf[0..n]);
+            }
+        },
+    }
+
+    try req.done_callback(req.ctx);
+}
+
+const CacheContext = struct {
+    arena: std.mem.Allocator,
+    context: Context,
+    transfer: ?*Transfer = null,
+    forward: Forward,
+    req_url: [:0]const u8,
+    req_headers: http.Headers,
+    pending_metadata: ?*CachedMetadata = null,
+
+    fn startCallback(response: Response) anyerror!void {
+        const self: *CacheContext = @ptrCast(@alignCast(response.ctx));
+        self.transfer = response.inner.transfer;
+        return self.forward.forwardStart(response);
+    }
+
+    fn headerCallback(response: Response) anyerror!bool {
+        const self: *CacheContext = @ptrCast(@alignCast(response.ctx));
+        const allocator = self.arena;
+
+        const transfer = response.inner.transfer;
+        var rh = &transfer.response_header.?;
+
+        const conn = transfer._conn.?;
+
+        const vary = if (conn.getResponseHeader("vary", 0)) |h| h.value else null;
+
+        const maybe_cm = try Cache.tryCache(
+            allocator,
+            std.time.timestamp(),
+            transfer.url,
+            rh.status,
+            rh.contentType(),
+            if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,
+            vary,
+            if (conn.getResponseHeader("age", 0)) |h| h.value else null,
+            conn.getResponseHeader("set-cookie", 0) != null,
+            conn.getResponseHeader("authorization", 0) != null,
+        );
+
+        if (maybe_cm) |cm| {
+            var iter = transfer.responseHeaderIterator();
+            var header_list = try iter.collect(allocator);
+            const end_of_response = header_list.items.len;
+
+            if (vary) |vary_str| {
+                var req_it = self.req_headers.iterator();
+                while (req_it.next()) |hdr| {
+                    var vary_iter = std.mem.splitScalar(u8, vary_str, ',');
+                    while (vary_iter.next()) |part| {
+                        const name = std.mem.trim(u8, part, &std.ascii.whitespace);
+                        if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
+                            try header_list.append(allocator, .{
+                                .name = try allocator.dupe(u8, hdr.name),
+                                .value = try allocator.dupe(u8, hdr.value),
+                            });
+                        }
+                    }
+                }
+
+                const metadata = try allocator.create(CachedMetadata);
+                metadata.* = cm;
+                metadata.headers = header_list.items[0..end_of_response];
+                metadata.vary_headers = header_list.items[end_of_response..];
+                self.pending_metadata = metadata;
+            }
+        }
+
+        return self.forward.forwardHeader(response);
+    }
+
+    fn doneCallback(ctx: *anyopaque) anyerror!void {
+        const self: *CacheContext = @ptrCast(@alignCast(ctx));
+        defer self.context.network.app.arena_pool.release(self.arena);
+
+        const transfer = self.transfer orelse @panic("Start Callback didn't set CacheLayer.transfer");
+
+        if (self.pending_metadata) |metadata| {
+            const cache = &self.context.network.cache.?;
+
+            log.debug(.browser, "http cache", .{ .key = self.req_url, .metadata = metadata });
+            cache.put(metadata.*, transfer._stream_buffer.items) catch |err| {
+                log.warn(.http, "cache put failed", .{ .err = err });
+            };
+            log.debug(.browser, "http.cache.put", .{ .url = self.req_url });
+        }
+
+        return self.forward.forwardDone();
+    }
+
+    fn shutdownCallback(ctx: *anyopaque) void {
+        const self: *CacheContext = @ptrCast(@alignCast(ctx));
+        defer self.context.network.app.arena_pool.release(self.arena);
+        self.forward.forwardShutdown();
+    }
+
+    fn errorCallback(ctx: *anyopaque, e: anyerror) void {
+        const self: *CacheContext = @ptrCast(@alignCast(ctx));
+        defer self.context.network.app.arena_pool.release(self.arena);
+        self.forward.forwardErr(e);
+    }
+};

src/network/layer/Forward.zig

@@ -0,0 +1,135 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const Request = @import("../../browser/HttpClient.zig").Request;
+const Response = @import("../../browser/HttpClient.zig").Response;
+
+const Forward = @This();
+
+ctx: *anyopaque,
+start: ?Request.StartCallback,
+header: Request.HeaderCallback,
+data: Request.DataCallback,
+done: Request.DoneCallback,
+err: Request.ErrorCallback,
+shutdown: ?Request.ShutdownCallback,
+
+pub fn fromRequest(req: Request) Forward {
+    return .{
+        .ctx = req.ctx,
+        .start = req.start_callback,
+        .header = req.header_callback,
+        .data = req.data_callback,
+        .done = req.done_callback,
+        .err = req.error_callback,
+        .shutdown = req.shutdown_callback,
+    };
+}
+
+pub const Overrides = struct {
+    start: ?Request.StartCallback = null,
+    header: ?Request.HeaderCallback = null,
+    data: ?Request.DataCallback = null,
+    done: ?Request.DoneCallback = null,
+    err: ?Request.ErrorCallback = null,
+    shutdown: ?Request.ShutdownCallback = null,
+};
+
+pub fn wrapRequest(
+    self: *Forward,
+    req: Request,
+    new_ctx: anytype,
+    comptime field: []const u8,
+    overrides: Overrides,
+) Request {
+    const T = @TypeOf(new_ctx.*);
+    const PassthroughT = makePassthrough(T, field);
+    var wrapped = req;
+    wrapped.ctx = new_ctx;
+    wrapped.start_callback = overrides.start orelse if (self.start != null) PassthroughT.start else null;
+    wrapped.header_callback = overrides.header orelse PassthroughT.header;
+    wrapped.data_callback = overrides.data orelse PassthroughT.data;
+    wrapped.done_callback = overrides.done orelse PassthroughT.done;
+    wrapped.error_callback = overrides.err orelse PassthroughT.err;
+    wrapped.shutdown_callback = overrides.shutdown orelse if (self.shutdown != null) PassthroughT.shutdown else null;
+    return wrapped;
+}
+
+fn makePassthrough(comptime T: type, comptime field: []const u8) type {
+    return struct {
+        pub fn start(response: Response) anyerror!void {
+            const self: *T = @ptrCast(@alignCast(response.ctx));
+            return @field(self, field).forwardStart(response);
+        }
+
+        pub fn header(response: Response) anyerror!bool {
+            const self: *T = @ptrCast(@alignCast(response.ctx));
+            return @field(self, field).forwardHeader(response);
+        }
+
+        pub fn data(response: Response, chunk: []const u8) anyerror!void {
+            const self: *T = @ptrCast(@alignCast(response.ctx));
+            return @field(self, field).forwardData(response, chunk);
+        }
+
+        pub fn done(ctx_ptr: *anyopaque) anyerror!void {
+            const self: *T = @ptrCast(@alignCast(ctx_ptr));
+            return @field(self, field).forwardDone();
+        }
+
+        pub fn err(ctx_ptr: *anyopaque, e: anyerror) void {
+            const self: *T = @ptrCast(@alignCast(ctx_ptr));
+            @field(self, field).forwardErr(e);
+        }
+
+        pub fn shutdown(ctx_ptr: *anyopaque) void {
+            const self: *T = @ptrCast(@alignCast(ctx_ptr));
+            @field(self, field).forwardShutdown();
+        }
+    };
+}
+
+pub fn forwardStart(self: Forward, response: Response) anyerror!void {
+    var fwd = response;
+    fwd.ctx = self.ctx;
+    if (self.start) |cb| try cb(fwd);
+}
+
+pub fn forwardHeader(self: Forward, response: Response) anyerror!bool {
+    var fwd = response;
+    fwd.ctx = self.ctx;
+    return self.header(fwd);
+}
+
+pub fn forwardData(self: Forward, response: Response, chunk: []const u8) anyerror!void {
+    var fwd = response;
+    fwd.ctx = self.ctx;
+    return self.data(fwd, chunk);
+}
+
+pub fn forwardDone(self: Forward) anyerror!void {
+    return self.done(self.ctx);
+}
+
+pub fn forwardErr(self: Forward, e: anyerror) void {
+    self.err(self.ctx, e);
+}
+
+pub fn forwardShutdown(self: Forward) void {
+    if (self.shutdown) |cb| cb(self.ctx);
+}

src/network/layer/RobotsLayer.zig

@@ -0,0 +1,255 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const log = @import("../../log.zig");
+
+const URL = @import("../../browser/URL.zig");
+const Robots = @import("../Robots.zig");
+const Context = @import("../../browser/HttpClient.zig").Context;
+const Request = @import("../../browser/HttpClient.zig").Request;
+const Response = @import("../../browser/HttpClient.zig").Response;
+const Layer = @import("../../browser/HttpClient.zig").Layer;
+const Forward = @import("Forward.zig");
+
+const RobotsLayer = @This();
+
+next: Layer = undefined,
+obey_robots: bool,
+allocator: std.mem.Allocator,
+pending: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(Request)) = .empty,
+
+pub fn layer(self: *RobotsLayer) Layer {
+    return .{
+        .ptr = self,
+        .vtable = &.{
+            .request = request,
+        },
+    };
+}
+
+pub fn deinit(self: *RobotsLayer, allocator: std.mem.Allocator) void {
+    var it = self.pending.iterator();
+    while (it.next()) |entry| {
+        entry.value_ptr.deinit(allocator);
+    }
+    self.pending.deinit(allocator);
+}
+
+fn request(ptr: *anyopaque, ctx: Context, req: Request) anyerror!void {
+    const self: *RobotsLayer = @ptrCast(@alignCast(ptr));
+
+    if (!self.obey_robots) {
+        return self.next.request(ctx, req);
+    }
+
+    const robots_url = try URL.getRobotsUrl(self.allocator, req.url);
+    errdefer self.allocator.free(robots_url);
+
+    if (ctx.network.robot_store.get(robots_url)) |robot_entry| {
+        defer self.allocator.free(robots_url);
+        switch (robot_entry) {
+            .present => |robots| {
+                const path = URL.getPathname(req.url);
+                if (!robots.isAllowed(path)) {
+                    log.warn(.http, "blocked by robots", .{ .url = req.url });
+                    req.error_callback(req.ctx, error.RobotsBlocked);
+                    return;
+                }
+            },
+            .absent => {},
+        }
+        return self.next.request(ctx, req);
+    }
+
+    return self.fetchRobotsThenRequest(ctx, robots_url, req);
+}
+
+fn fetchRobotsThenRequest(self: *RobotsLayer, ctx: Context, robots_url: [:0]const u8, req: Request) !void {
+    const entry = try self.pending.getOrPut(self.allocator, robots_url);
+
+    if (!entry.found_existing) {
+        errdefer self.allocator.free(robots_url);
+        entry.value_ptr.* = .empty;
+
+        const robots_ctx = try self.allocator.create(RobotsContext);
+        errdefer self.allocator.destroy(robots_ctx);
+        robots_ctx.* = .{
+            .layer = self,
+            .ctx = ctx,
+            .robots_url = robots_url,
+            .buffer = .empty,
+        };
+
+        const headers = try ctx.newHeaders();
+        log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });
+
+        try self.next.request(ctx, .{
+            .ctx = robots_ctx,
+            .url = robots_url,
+            .method = .GET,
+            .headers = headers,
+            .blocking = false,
+            .frame_id = req.frame_id,
+            .cookie_jar = req.cookie_jar,
+            .cookie_origin = req.cookie_origin,
+            .notification = req.notification,
+            .resource_type = .fetch,
+            .header_callback = RobotsContext.headerCallback,
+            .data_callback = RobotsContext.dataCallback,
+            .done_callback = RobotsContext.doneCallback,
+            .error_callback = RobotsContext.errorCallback,
+            .shutdown_callback = RobotsContext.shutdownCallback,
+        });
+    } else {
+        self.allocator.free(robots_url);
+    }
+
+    try entry.value_ptr.append(self.allocator, req);
+}
+
+fn flushPending(self: *RobotsLayer, ctx: Context, robots_url: [:0]const u8, allowed: bool) void {
+    var queued = self.pending.fetchRemove(robots_url) orelse
+        @panic("RobotsLayer.flushPending: missing queue");
+    defer queued.value.deinit(self.allocator);
+
+    for (queued.value.items) |queued_req| {
+        if (!allowed) {
+            log.warn(.http, "blocked by robots", .{ .url = queued_req.url });
+            defer queued_req.headers.deinit();
+            queued_req.error_callback(queued_req.ctx, error.RobotsBlocked);
+        } else {
+            self.next.request(ctx, queued_req) catch |e| {
+                defer queued_req.headers.deinit();
+                queued_req.error_callback(queued_req.ctx, e);
+            };
+        }
+    }
+}
+
+fn flushPendingShutdown(self: *RobotsLayer, robots_url: [:0]const u8) void {
+    var queued = self.pending.fetchRemove(robots_url) orelse
+        @panic("RobotsLayer.flushPendingShutdown: missing queue");
+    defer queued.value.deinit(self.allocator);
+
+    for (queued.value.items) |queued_req| {
+        defer queued_req.headers.deinit();
+        if (queued_req.shutdown_callback) |cb| cb(queued_req.ctx);
+    }
+}
+
+const RobotsContext = struct {
+    layer: *RobotsLayer,
+    ctx: Context,
+    robots_url: [:0]const u8,
+    buffer: std.ArrayListUnmanaged(u8),
+    status: u16 = 0,
+
+    fn deinit(self: *RobotsContext) void {
+        self.buffer.deinit(self.layer.allocator);
+        self.layer.allocator.destroy(self);
+    }
+
+    fn headerCallback(response: Response) anyerror!bool {
+        const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));
+        switch (response.inner) {
+            .transfer => |t| {
+                if (t.response_header) |hdr| {
+                    log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });
+                    self.status = hdr.status;
+                }
+                if (t.getContentLength()) |cl| {
+                    try self.buffer.ensureTotalCapacity(self.layer.allocator, cl);
+                }
+            },
+            .cached => {},
+        }
+        return true;
+    }
+
+    fn dataCallback(response: Response, data: []const u8) anyerror!void {
+        const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));
+        try self.buffer.appendSlice(self.layer.allocator, data);
+    }
+
+    fn doneCallback(ctx_ptr: *anyopaque) anyerror!void {
+        const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
+        const l = self.layer;
+        const ctx = self.ctx;
+        const robots_url = self.robots_url;
+        defer l.allocator.free(robots_url);
+        defer self.deinit();
+
+        var allowed = true;
+        const network = ctx.network;
+
+        switch (self.status) {
+            200 => {
+                if (self.buffer.items.len > 0) {
+                    const robots: ?Robots = network.robot_store.robotsFromBytes(
+                        network.config.http_headers.user_agent,
+                        self.buffer.items,
+                    ) catch blk: {
+                        log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });
+                        try network.robot_store.putAbsent(robots_url);
+                        break :blk null;
+                    };
+                    if (robots) |r| {
+                        try network.robot_store.put(robots_url, r);
+                        const path = URL.getPathname(self.layer.pending.get(robots_url).?.items[0].url);
+                        allowed = r.isAllowed(path);
+                    }
+                }
+            },
+            404 => {
+                log.debug(.http, "robots not found", .{ .url = robots_url });
+                try network.robot_store.putAbsent(robots_url);
+            },
+            else => {
+                log.debug(.http, "unexpected status on robots", .{
+                    .url = robots_url,
+                    .status = self.status,
+                });
+                try network.robot_store.putAbsent(robots_url);
+            },
+        }
+
+        l.flushPending(ctx, robots_url, allowed);
+    }
+
+    fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void {
+        const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
+        const l = self.layer;
+        const ctx = self.ctx;
+        const robots_url = self.robots_url;
+        defer l.allocator.free(robots_url);
+        defer self.deinit();
+        log.warn(.http, "robots fetch failed", .{ .err = err });
+        l.flushPending(ctx, robots_url, true);
+    }
+
+    fn shutdownCallback(ctx_ptr: *anyopaque) void {
+        const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
+        const l = self.layer;
+        const robots_url = self.robots_url;
+        defer l.allocator.free(robots_url);
+        defer self.deinit();
+        log.debug(.http, "robots fetch shutdown", .{});
+        l.flushPendingShutdown(robots_url);
+    }
+};

src/network/layer/WebBotAuthLayer.zig

@@ -0,0 +1,54 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const log = @import("../../log.zig");
+
+const URL = @import("../../browser/URL.zig");
+const WebBotAuth = @import("../WebBotAuth.zig");
+const Context = @import("../../browser/HttpClient.zig").Context;
+const Request = @import("../../browser/HttpClient.zig").Request;
+const Layer = @import("../../browser/HttpClient.zig").Layer;
+
+const WebBotAuthLayer = @This();
+
+next: Layer = undefined,
+
+pub fn layer(self: *WebBotAuthLayer) Layer {
+    return .{
+        .ptr = self,
+        .vtable = &.{ .request = request },
+    };
+}
+
+pub fn deinit(_: *WebBotAuthLayer, _: std.mem.Allocator) void {}
+
+fn request(ptr: *anyopaque, ctx: Context, req: Request) anyerror!void {
+    const self: *WebBotAuthLayer = @ptrCast(@alignCast(ptr));
+    var our_req = req;
+
+    if (ctx.network.web_bot_auth) |*wba| {
+        const arena = try ctx.network.app.arena_pool.acquire(.{ .debug = "WebBotAuthLayer" });
+        defer ctx.network.app.arena_pool.release(arena);
+
+        const authority = URL.getHost(req.url);
+        try wba.signRequest(arena, &our_req.headers, authority);
+    }
+
+    return self.next.request(ctx, our_req);
+}