ci: add a web bot auth signature test

.github/workflows/e2e-test.yml

@@ -117,6 +117,98 @@ jobs:
           BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 
+  # e2e tests w/ web-bot-auth configuration on.
+  wba-demo-scripts:
+    name: wba-demo-scripts
+    needs: zig-build-release
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: 'lightpanda-io/demo'
+          fetch-depth: 0
+
+      - run: npm install
+
+      - name: download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lightpanda-build-release
+
+      - run: chmod a+x ./lightpanda
+
+      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+      - name: run end to end tests
+        run: |
+          ./lightpanda serve \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            & echo $! > LPD.pid
+          go run runner/main.go
+          kill `cat LPD.pid`
+
+      - name: build proxy
+        run: |
+          cd proxy
+          go build
+
+      - name: run end to end tests through proxy
+        run: |
+          ./proxy/proxy & echo $! > PROXY.id
+          ./lightpanda serve \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            --http_proxy 'http://127.0.0.1:3000' \
+            & echo $! > LPD.pid
+          go run runner/main.go
+          kill `cat LPD.pid` `cat PROXY.id`
+
+      - name: run request interception through proxy
+        run: |
+          export PROXY_USERNAME=username PROXY_PASSWORD=password
+          ./proxy/proxy & echo $! > PROXY.id
+          ./lightpanda serve & echo $! > LPD.pid
+          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
+          BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
+          kill `cat LPD.pid` `cat PROXY.id`
+
+  wba-test:
+    name: wba-test
+    needs: zig-build-release
+
+    env:
+      LIGHTPANDA_DISABLE_TELEMETRY: true
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lightpanda-build-release
+
+      - run: chmod a+x ./lightpanda
+
+      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+      - run: |
+          ./lightpanda fetch https://crawltest.com/cdn-cgi/web-bot-auth \
+            --log_level error \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            --dump markdown \
+            | tee output.log
+
+      - run: cat output.log | grep -q "unknown public key or unknown verified bot ID for keyid"
+
   cdp-and-hyperfine-bench:
     name: cdp-and-hyperfine-bench
     needs: zig-build-release

src/App.zig

@@ -26,6 +26,7 @@ const Snapshot = @import("browser/js/Snapshot.zig");
 const Platform = @import("browser/js/Platform.zig");
 const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
 const RobotStore = @import("browser/Robots.zig").RobotStore;
+const WebBotAuth = @import("browser/WebBotAuth.zig");
 
 pub const Http = @import("http/Http.zig");
 pub const ArenaPool = @import("ArenaPool.zig");
@@ -40,6 +41,7 @@ telemetry: Telemetry,
 allocator: Allocator,
 arena_pool: ArenaPool,
 robots: RobotStore,
+web_bot_auth: ?WebBotAuth,
 app_dir_path: ?[]const u8,
 shutdown: bool = false,
 
@@ -52,7 +54,14 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
 
     app.robots = RobotStore.init(allocator);
 
-    app.http = try Http.init(allocator, &app.robots, config);
+    if (config.webBotAuth()) |wba_cfg| {
+        app.web_bot_auth = try WebBotAuth.fromConfig(allocator, &wba_cfg);
+    } else {
+        app.web_bot_auth = null;
+    }
+    errdefer if (app.web_bot_auth) |wba| wba.deinit(allocator);
+
+    app.http = try Http.init(allocator, &app.robots, &app.web_bot_auth, config);
     errdefer app.http.deinit();
 
     app.platform = try Platform.init();
@@ -84,6 +93,9 @@ pub fn deinit(self: *App) void {
     }
     self.telemetry.deinit();
     self.robots.deinit();
+    if (self.web_bot_auth) |wba| {
+        wba.deinit(allocator);
+    }
     self.http.deinit();
     self.snapshot.deinit();
     self.platform.deinit();

src/Config.zig

@@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator;
 const log = @import("log.zig");
 const dump = @import("browser/dump.zig");
 
+const WebBotAuthConfig = @import("browser/WebBotAuth.zig").Config;
+
 pub const RunMode = enum {
     help,
     fetch,
@@ -153,6 +155,17 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
     };
 }
 
+pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
+    return switch (self.mode) {
+        inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
+            .key_file = opts.common.web_bot_auth_key_file orelse return null,
+            .keyid = opts.common.web_bot_auth_keyid orelse return null,
+            .domain = opts.common.web_bot_auth_domain orelse return null,
+        },
+        .help, .version => null,
+    };
+}
+
 pub fn maxConnections(self: *const Config) u16 {
     return switch (self.mode) {
         .serve => |opts| opts.cdp_max_connections,
@@ -217,6 +230,10 @@ pub const Common = struct {
     log_format: ?log.Format = null,
     log_filter_scopes: ?[]log.Scope = null,
     user_agent_suffix: ?[]const u8 = null,
+
+    web_bot_auth_key_file: ?[]const u8 = null,
+    web_bot_auth_keyid: ?[]const u8 = null,
+    web_bot_auth_domain: ?[]const u8 = null,
 };
 
 /// Pre-formatted HTTP headers for reuse across Http and Client.
@@ -324,6 +341,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
         \\--user_agent_suffix
         \\                Suffix to append to the Lightpanda/X.Y User-Agent
         \\
+        \\--web_bot_auth_key_file
+        \\                Path to the Ed25519 private key PEM file.
+        \\
+        \\--web_bot_auth_keyid
+        \\                The JWK thumbprint of your public key.
+        \\
+        \\--web_bot_auth_domain
+        \\                Your domain e.g. yourdomain.com
     ;
 
     //                                                                     MAX_HELP_LEN|
@@ -845,5 +870,32 @@ fn parseCommonArg(
         return true;
     }
 
+    if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_key_file = try allocator.dupe(u8, str);
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_keyid = try allocator.dupe(u8, str);
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_domain = try allocator.dupe(u8, str);
+        return true;
+    }
+
     return false;
 }

src/browser/WebBotAuth.zig

@@ -0,0 +1,284 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const crypto = @import("../crypto.zig");
+
+const Http = @import("../http/Http.zig");
+
+const WebBotAuth = @This();
+
+pkey: *crypto.EVP_PKEY,
+keyid: []const u8,
+directory_url: [:0]const u8,
+
+pub const Config = struct {
+    key_file: []const u8,
+    keyid: []const u8,
+    domain: []const u8,
+};
+
+fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY {
+    const begin = "-----BEGIN PRIVATE KEY-----";
+    const end = "-----END PRIVATE KEY-----";
+    const start_idx = std.mem.indexOf(u8, pem, begin) orelse return error.InvalidPem;
+    const end_idx = std.mem.indexOf(u8, pem, end) orelse return error.InvalidPem;
+
+    const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace);
+
+    // decode base64 into 48-byte DER buffer
+    var der: [48]u8 = undefined;
+    try std.base64.standard.Decoder.decode(der[0..48], b64);
+
+    // Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16.
+    const key_bytes = der[16..48];
+
+    const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32);
+    return pkey orelse error.InvalidKey;
+}
+
+fn signEd25519(pkey: *crypto.EVP_PKEY, message: []const u8, out: *[64]u8) !void {
+    const ctx = crypto.EVP_MD_CTX_new() orelse return error.OutOfMemory;
+    defer crypto.EVP_MD_CTX_free(ctx);
+
+    if (crypto.EVP_DigestSignInit(ctx, null, null, null, pkey) != 1)
+        return error.SignInit;
+
+    var sig_len: usize = 64;
+    if (crypto.EVP_DigestSign(ctx, out.ptr, &sig_len, message.ptr, message.len) != 1)
+        return error.SignFailed;
+}
+
+pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAuth {
+    const pem = try std.fs.cwd().readFileAlloc(allocator, config.key_file, 1024 * 4);
+    defer allocator.free(pem);
+
+    const pkey = try parsePemPrivateKey(pem);
+    errdefer crypto.EVP_PKEY_free(pkey);
+
+    const directory_url = try std.fmt.allocPrintSentinel(
+        allocator,
+        "https://{s}/.well-known/http-message-signatures-directory",
+        .{config.domain},
+        0,
+    );
+    errdefer allocator.free(directory_url);
+
+    return .{
+        .pkey = pkey,
+        // Owned by the Config so it's okay.
+        .keyid = config.keyid,
+        .directory_url = directory_url,
+    };
+}
+
+pub fn signRequest(
+    self: *const WebBotAuth,
+    allocator: std.mem.Allocator,
+    headers: *Http.Headers,
+    authority: []const u8,
+) !void {
+    const now = std.time.timestamp();
+    const expires = now + 60;
+
+    // build the signature-input value (without the sig1= label)
+    const sig_input_value = try std.fmt.allocPrint(
+        allocator,
+        "(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"",
+        .{ now, expires, self.keyid },
+    );
+    defer allocator.free(sig_input_value);
+
+    // build the canonical string to sign
+    const canonical = try std.fmt.allocPrint(
+        allocator,
+        "\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}",
+        .{ authority, self.directory_url, sig_input_value },
+    );
+    defer allocator.free(canonical);
+
+    // sign it
+    var sig: [64]u8 = undefined;
+    try signEd25519(self.pkey, canonical, &sig);
+
+    // base64 encode
+    const encoded_len = std.base64.standard.Encoder.calcSize(sig.len);
+    const encoded = try allocator.alloc(u8, encoded_len);
+    defer allocator.free(encoded);
+    _ = std.base64.standard.Encoder.encode(encoded, &sig);
+
+    // build the 3 headers and add them
+    const sig_agent = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature-Agent: \"{s}\"",
+        .{self.directory_url},
+        0,
+    );
+    defer allocator.free(sig_agent);
+
+    const sig_input = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature-Input: sig1={s}",
+        .{sig_input_value},
+        0,
+    );
+    defer allocator.free(sig_input);
+
+    const signature = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature: sig1=:{s}:",
+        .{encoded},
+        0,
+    );
+    defer allocator.free(signature);
+
+    try headers.add(sig_agent);
+    try headers.add(sig_input);
+    try headers.add(signature);
+}
+
+pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {
+    crypto.EVP_PKEY_free(self.pkey);
+    allocator.free(self.directory_url);
+}
+
+test "parsePemPrivateKey: valid Ed25519 PKCS#8 PEM" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+}
+
+test "parsePemPrivateKey: missing BEGIN marker returns error" {
+    const bad_pem = "-----END PRIVATE KEY-----\n";
+    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
+}
+
+test "parsePemPrivateKey: missing END marker returns error" {
+    const bad_pem = "-----BEGIN PRIVATE KEY-----\nMC4CAQA=\n";
+    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
+}
+
+test "signEd25519: signature length is always 64 bytes" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig: [64]u8 = @splat(0);
+    try signEd25519(pkey, "hello world", &sig);
+
+    var all_zero = true;
+    for (sig) |b| if (b != 0) {
+        all_zero = false;
+        break;
+    };
+    try std.testing.expect(!all_zero);
+}
+
+test "signEd25519: same key + message produces same signature (deterministic)" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig1: [64]u8 = undefined;
+    var sig2: [64]u8 = undefined;
+    try signEd25519(pkey, "deterministic test", &sig1);
+    try signEd25519(pkey, "deterministic test", &sig2);
+
+    try std.testing.expectEqualSlices(u8, &sig1, &sig2);
+}
+
+test "signEd25519: same key + diff message produces different signature (deterministic)" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig1: [64]u8 = undefined;
+    var sig2: [64]u8 = undefined;
+    try signEd25519(pkey, "msg 1", &sig1);
+    try signEd25519(pkey, "msg 2", &sig2);
+
+    try std.testing.expect(!std.mem.eql(u8, &sig1, &sig2));
+}
+
+test "signRequest: adds headers with correct names" {
+    const allocator = std.testing.allocator;
+
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+
+    const directory_url = try allocator.dupeZ(
+        u8,
+        "https://example.com/.well-known/http-message-signatures-directory",
+    );
+
+    var auth = WebBotAuth{
+        .pkey = pkey,
+        .keyid = "test-key-id",
+        .directory_url = directory_url,
+    };
+    defer auth.deinit(allocator);
+
+    var headers = try Http.Headers.init("User-Agent: Test-Agent");
+    defer headers.deinit();
+
+    try auth.signRequest(allocator, &headers, "example.com");
+
+    var it = headers.iterator();
+    var found_sig_agent = false;
+    var found_sig_input = false;
+    var found_signature = false;
+    var count: usize = 0;
+
+    while (it.next()) |h| {
+        count += 1;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true;
+    }
+
+    try std.testing.expect(count >= 3);
+    try std.testing.expect(found_sig_agent);
+    try std.testing.expect(found_sig_input);
+    try std.testing.expect(found_signature);
+}

src/crypto.zig

@@ -228,6 +228,8 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8
 
 pub const NID_X25519 = @as(c_int, 948);
 pub const EVP_PKEY_X25519 = NID_X25519;
+pub const NID_ED25519 = 949;
+pub const EVP_PKEY_ED25519 = NID_ED25519;
 
 pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
 pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
@@ -236,3 +238,11 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void;
 pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int;
 pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int;
 pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int;
+pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void;
+
+pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int;
+pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int;
+pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX;
+pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void;
+pub const struct_evp_md_ctx_st = opaque {};
+pub const EVP_MD_CTX = struct_evp_md_ctx_st;

src/http/Client.zig

@@ -29,6 +29,8 @@ const Notification = @import("../Notification.zig");
 const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
 const Robots = @import("../browser/Robots.zig");
 const RobotStore = Robots.RobotStore;
+const WebBotAuth = @import("../browser/WebBotAuth.zig");
+
 const posix = std.posix;
 
 const Allocator = std.mem.Allocator;
@@ -83,6 +85,9 @@ robot_store: *RobotStore,
 // Allows us to fetch the robots.txt just once.
 pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empty,
 
+// Reference to the App-owned WebBotAuth.
+web_bot_auth: *const ?WebBotAuth,
+
 // Once we have a handle/easy to process a request with, we create a Transfer
 // which contains the Request as well as any state we need to process the
 // request. These wil come and go with each request.
@@ -121,7 +126,13 @@ pub const CDPClient = struct {
 
 const TransferQueue = std.DoublyLinkedList;
 
-pub fn init(allocator: Allocator, ca_blob: ?Net.Blob, robot_store: *RobotStore, config: *const Config) !*Client {
+pub fn init(
+    allocator: Allocator,
+    ca_blob: ?Net.Blob,
+    robot_store: *RobotStore,
+    web_bot_auth: *const ?WebBotAuth,
+    config: *const Config,
+) !*Client {
     var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
     errdefer transfer_pool.deinit();
 
@@ -145,6 +156,7 @@ pub fn init(allocator: Allocator, ca_blob: ?Net.Blob, robot_store: *RobotStore,
         .handles = handles,
         .allocator = allocator,
         .robot_store = robot_store,
+        .web_bot_auth = web_bot_auth,
         .http_proxy = http_proxy,
         .use_proxy = http_proxy != null,
         .config = config,
@@ -709,6 +721,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
         try conn.secretHeaders(&header_list, &self.config.http_headers); // Add headers that must be hidden from intercepts
         try conn.setHeaders(&header_list);
 
+        // If we have WebBotAuth, sign our request.
+        if (self.web_bot_auth.*) |wba| {
+            const authority = URL.getHost(req.url);
+            try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
+        }
+
         // Add cookies.
         if (header_list.cookies) |cookies| {
             try conn.setCookies(cookies);
@@ -1302,7 +1320,7 @@ pub const Transfer = struct {
             }
             transfer._redirecting = false;
 
-            if (status == 401 or status == 407) {
+            if ((status == 401 or status == 407) and transfer.client.use_proxy) {
                 // The auth challenge must be parsed from a following
                 // WWW-Authenticate or Proxy-Authenticate header.
                 transfer._auth_challenge = .{

src/http/Http.zig

@@ -29,6 +29,7 @@ pub const Headers = Net.Headers;
 
 const Config = @import("../Config.zig");
 const RobotStore = @import("../browser/Robots.zig").RobotStore;
+const WebBotAuth = @import("../browser/WebBotAuth.zig");
 
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
@@ -45,8 +46,14 @@ allocator: Allocator,
 config: *const Config,
 ca_blob: ?Net.Blob,
 robot_store: *RobotStore,
+web_bot_auth: *const ?WebBotAuth,
 
-pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Config) !Http {
+pub fn init(
+    allocator: Allocator,
+    robot_store: *RobotStore,
+    web_bot_auth: *const ?WebBotAuth,
+    config: *const Config,
+) !Http {
     try Net.globalInit();
     errdefer Net.globalDeinit();
 
@@ -68,6 +75,7 @@ pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Confi
         .config = config,
         .ca_blob = ca_blob,
         .robot_store = robot_store,
+        .web_bot_auth = web_bot_auth,
     };
 }
 
@@ -81,7 +89,7 @@ pub fn deinit(self: *Http) void {
 }
 
 pub fn createClient(self: *Http, allocator: Allocator) !*Client {
-    return Client.init(allocator, self.ca_blob, self.robot_store, self.config);
+    return Client.init(allocator, self.ca_blob, self.robot_store, self.web_bot_auth, self.config);
 }
 
 pub fn newConnection(self: *Http) !Net.Connection {