ci: add a web bot auth signature test

Run the message loop more!

.github/workflows/e2e-test.yml

@@ -117,6 +117,98 @@ jobs:
           BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 
+  # e2e tests w/ web-bot-auth configuration on.
+  wba-demo-scripts:
+    name: wba-demo-scripts
+    needs: zig-build-release
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: 'lightpanda-io/demo'
+          fetch-depth: 0
+
+      - run: npm install
+
+      - name: download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lightpanda-build-release
+
+      - run: chmod a+x ./lightpanda
+
+      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+      - name: run end to end tests
+        run: |
+          ./lightpanda serve \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            & echo $! > LPD.pid
+          go run runner/main.go
+          kill `cat LPD.pid`
+
+      - name: build proxy
+        run: |
+          cd proxy
+          go build
+
+      - name: run end to end tests through proxy
+        run: |
+          ./proxy/proxy & echo $! > PROXY.id
+          ./lightpanda serve \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            --http_proxy 'http://127.0.0.1:3000' \
+            & echo $! > LPD.pid
+          go run runner/main.go
+          kill `cat LPD.pid` `cat PROXY.id`
+
+      - name: run request interception through proxy
+        run: |
+          export PROXY_USERNAME=username PROXY_PASSWORD=password
+          ./proxy/proxy & echo $! > PROXY.id
+          ./lightpanda serve & echo $! > LPD.pid
+          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
+          BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
+          kill `cat LPD.pid` `cat PROXY.id`
+
+  wba-test:
+    name: wba-test
+    needs: zig-build-release
+
+    env:
+      LIGHTPANDA_DISABLE_TELEMETRY: true
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lightpanda-build-release
+
+      - run: chmod a+x ./lightpanda
+
+      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+      - run: |
+          ./lightpanda fetch https://crawltest.com/cdn-cgi/web-bot-auth \
+            --log_level error \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            --dump markdown \
+            | tee output.log
+
+      - run: cat output.log | grep -q "unknown public key or unknown verified bot ID for keyid"
+
   cdp-and-hyperfine-bench:
     name: cdp-and-hyperfine-bench
     needs: zig-build-release

Dockerfile

@@ -36,10 +36,6 @@ RUN ZIG=$(grep '\.minimum_zig_version = "' "build.zig.zon" | cut -d'"' -f2) && \
     mv zig-${ARCH}-linux-${ZIG} /usr/local/lib && \
     ln -s /usr/local/lib/zig-${ARCH}-linux-${ZIG}/zig /usr/local/bin/zig
 
-# install deps
-RUN git submodule init && \
-    git submodule update --recursive
-
 # download and install v8
 RUN case $TARGETPLATFORM in \
     "linux/arm64") ARCH="aarch64" ;; \

src/App.zig

@@ -26,6 +26,7 @@ const Snapshot = @import("browser/js/Snapshot.zig");
 const Platform = @import("browser/js/Platform.zig");
 const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
 const RobotStore = @import("browser/Robots.zig").RobotStore;
+const WebBotAuth = @import("browser/WebBotAuth.zig");
 
 pub const Http = @import("http/Http.zig");
 pub const ArenaPool = @import("ArenaPool.zig");
@@ -40,6 +41,7 @@ telemetry: Telemetry,
 allocator: Allocator,
 arena_pool: ArenaPool,
 robots: RobotStore,
+web_bot_auth: ?WebBotAuth,
 app_dir_path: ?[]const u8,
 shutdown: bool = false,
 
@@ -52,7 +54,14 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
 
     app.robots = RobotStore.init(allocator);
 
-    app.http = try Http.init(allocator, &app.robots, config);
+    if (config.webBotAuth()) |wba_cfg| {
+        app.web_bot_auth = try WebBotAuth.fromConfig(allocator, &wba_cfg);
+    } else {
+        app.web_bot_auth = null;
+    }
+    errdefer if (app.web_bot_auth) |wba| wba.deinit(allocator);
+
+    app.http = try Http.init(allocator, &app.robots, &app.web_bot_auth, config);
     errdefer app.http.deinit();
 
     app.platform = try Platform.init();
@@ -84,6 +93,9 @@ pub fn deinit(self: *App) void {
     }
     self.telemetry.deinit();
     self.robots.deinit();
+    if (self.web_bot_auth) |wba| {
+        wba.deinit(allocator);
+    }
     self.http.deinit();
     self.snapshot.deinit();
     self.platform.deinit();

src/Config.zig

@@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator;
 const log = @import("log.zig");
 const dump = @import("browser/dump.zig");
 
+const WebBotAuthConfig = @import("browser/WebBotAuth.zig").Config;
+
 pub const RunMode = enum {
     help,
     fetch,
@@ -153,6 +155,17 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
     };
 }
 
+pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
+    return switch (self.mode) {
+        inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
+            .key_file = opts.common.web_bot_auth_key_file orelse return null,
+            .keyid = opts.common.web_bot_auth_keyid orelse return null,
+            .domain = opts.common.web_bot_auth_domain orelse return null,
+        },
+        .help, .version => null,
+    };
+}
+
 pub fn maxConnections(self: *const Config) u16 {
     return switch (self.mode) {
         .serve => |opts| opts.cdp_max_connections,
@@ -217,6 +230,10 @@ pub const Common = struct {
     log_format: ?log.Format = null,
     log_filter_scopes: ?[]log.Scope = null,
     user_agent_suffix: ?[]const u8 = null,
+
+    web_bot_auth_key_file: ?[]const u8 = null,
+    web_bot_auth_keyid: ?[]const u8 = null,
+    web_bot_auth_domain: ?[]const u8 = null,
 };
 
 /// Pre-formatted HTTP headers for reuse across Http and Client.
@@ -324,6 +341,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
         \\--user_agent_suffix
         \\                Suffix to append to the Lightpanda/X.Y User-Agent
         \\
+        \\--web_bot_auth_key_file
+        \\                Path to the Ed25519 private key PEM file.
+        \\
+        \\--web_bot_auth_keyid
+        \\                The JWK thumbprint of your public key.
+        \\
+        \\--web_bot_auth_domain
+        \\                Your domain e.g. yourdomain.com
     ;
 
     //                                                                     MAX_HELP_LEN|
@@ -845,5 +870,32 @@ fn parseCommonArg(
         return true;
     }
 
+    if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_key_file = try allocator.dupe(u8, str);
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_keyid = try allocator.dupe(u8, str);
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_domain = try allocator.dupe(u8, str);
+        return true;
+    }
+
     return false;
 }

src/browser/Session.zig

@@ -261,7 +261,7 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
                         std.debug.assert(http_client.intercepted == 0);
                     }
 
-                    const ms: u64 = ms_to_next_task orelse blk: {
+                    var ms: u64 = ms_to_next_task orelse blk: {
                         if (wait_ms - ms_remaining < 100) {
                             if (comptime builtin.is_test) {
                                 return .done;
@@ -288,7 +288,13 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
                         // Same as above, except we have a scheduled task,
                         // it just happens to be too far into the future
                         // compared to how long we were told to wait.
-                        return .done;
+                        if (!browser.hasBackgroundTasks()) {
+                            return .done;
+                        }
+                        // _we_ have nothing to run, but v8 is working on
+                        // background tasks. We'll wait for them.
+                        browser.waitForBackgroundTasks();
+                        ms = 20;
                     }
 
                     // We have a task to run in the not-so-distant future.

src/browser/WebBotAuth.zig

@@ -0,0 +1,284 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const crypto = @import("../crypto.zig");
+
+const Http = @import("../http/Http.zig");
+
+const WebBotAuth = @This();
+
+pkey: *crypto.EVP_PKEY,
+keyid: []const u8,
+directory_url: [:0]const u8,
+
+pub const Config = struct {
+    key_file: []const u8,
+    keyid: []const u8,
+    domain: []const u8,
+};
+
+fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY {
+    const begin = "-----BEGIN PRIVATE KEY-----";
+    const end = "-----END PRIVATE KEY-----";
+    const start_idx = std.mem.indexOf(u8, pem, begin) orelse return error.InvalidPem;
+    const end_idx = std.mem.indexOf(u8, pem, end) orelse return error.InvalidPem;
+
+    const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace);
+
+    // decode base64 into 48-byte DER buffer
+    var der: [48]u8 = undefined;
+    try std.base64.standard.Decoder.decode(der[0..48], b64);
+
+    // Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16.
+    const key_bytes = der[16..48];
+
+    const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32);
+    return pkey orelse error.InvalidKey;
+}
+
+fn signEd25519(pkey: *crypto.EVP_PKEY, message: []const u8, out: *[64]u8) !void {
+    const ctx = crypto.EVP_MD_CTX_new() orelse return error.OutOfMemory;
+    defer crypto.EVP_MD_CTX_free(ctx);
+
+    if (crypto.EVP_DigestSignInit(ctx, null, null, null, pkey) != 1)
+        return error.SignInit;
+
+    var sig_len: usize = 64;
+    if (crypto.EVP_DigestSign(ctx, out.ptr, &sig_len, message.ptr, message.len) != 1)
+        return error.SignFailed;
+}
+
+pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAuth {
+    const pem = try std.fs.cwd().readFileAlloc(allocator, config.key_file, 1024 * 4);
+    defer allocator.free(pem);
+
+    const pkey = try parsePemPrivateKey(pem);
+    errdefer crypto.EVP_PKEY_free(pkey);
+
+    const directory_url = try std.fmt.allocPrintSentinel(
+        allocator,
+        "https://{s}/.well-known/http-message-signatures-directory",
+        .{config.domain},
+        0,
+    );
+    errdefer allocator.free(directory_url);
+
+    return .{
+        .pkey = pkey,
+        // Owned by the Config so it's okay.
+        .keyid = config.keyid,
+        .directory_url = directory_url,
+    };
+}
+
+pub fn signRequest(
+    self: *const WebBotAuth,
+    allocator: std.mem.Allocator,
+    headers: *Http.Headers,
+    authority: []const u8,
+) !void {
+    const now = std.time.timestamp();
+    const expires = now + 60;
+
+    // build the signature-input value (without the sig1= label)
+    const sig_input_value = try std.fmt.allocPrint(
+        allocator,
+        "(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"",
+        .{ now, expires, self.keyid },
+    );
+    defer allocator.free(sig_input_value);
+
+    // build the canonical string to sign
+    const canonical = try std.fmt.allocPrint(
+        allocator,
+        "\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}",
+        .{ authority, self.directory_url, sig_input_value },
+    );
+    defer allocator.free(canonical);
+
+    // sign it
+    var sig: [64]u8 = undefined;
+    try signEd25519(self.pkey, canonical, &sig);
+
+    // base64 encode
+    const encoded_len = std.base64.standard.Encoder.calcSize(sig.len);
+    const encoded = try allocator.alloc(u8, encoded_len);
+    defer allocator.free(encoded);
+    _ = std.base64.standard.Encoder.encode(encoded, &sig);
+
+    // build the 3 headers and add them
+    const sig_agent = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature-Agent: \"{s}\"",
+        .{self.directory_url},
+        0,
+    );
+    defer allocator.free(sig_agent);
+
+    const sig_input = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature-Input: sig1={s}",
+        .{sig_input_value},
+        0,
+    );
+    defer allocator.free(sig_input);
+
+    const signature = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature: sig1=:{s}:",
+        .{encoded},
+        0,
+    );
+    defer allocator.free(signature);
+
+    try headers.add(sig_agent);
+    try headers.add(sig_input);
+    try headers.add(signature);
+}
+
+pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {
+    crypto.EVP_PKEY_free(self.pkey);
+    allocator.free(self.directory_url);
+}
+
+test "parsePemPrivateKey: valid Ed25519 PKCS#8 PEM" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+}
+
+test "parsePemPrivateKey: missing BEGIN marker returns error" {
+    const bad_pem = "-----END PRIVATE KEY-----\n";
+    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
+}
+
+test "parsePemPrivateKey: missing END marker returns error" {
+    const bad_pem = "-----BEGIN PRIVATE KEY-----\nMC4CAQA=\n";
+    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
+}
+
+test "signEd25519: signature length is always 64 bytes" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig: [64]u8 = @splat(0);
+    try signEd25519(pkey, "hello world", &sig);
+
+    var all_zero = true;
+    for (sig) |b| if (b != 0) {
+        all_zero = false;
+        break;
+    };
+    try std.testing.expect(!all_zero);
+}
+
+test "signEd25519: same key + message produces same signature (deterministic)" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig1: [64]u8 = undefined;
+    var sig2: [64]u8 = undefined;
+    try signEd25519(pkey, "deterministic test", &sig1);
+    try signEd25519(pkey, "deterministic test", &sig2);
+
+    try std.testing.expectEqualSlices(u8, &sig1, &sig2);
+}
+
+test "signEd25519: same key + diff message produces different signature (deterministic)" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig1: [64]u8 = undefined;
+    var sig2: [64]u8 = undefined;
+    try signEd25519(pkey, "msg 1", &sig1);
+    try signEd25519(pkey, "msg 2", &sig2);
+
+    try std.testing.expect(!std.mem.eql(u8, &sig1, &sig2));
+}
+
+test "signRequest: adds headers with correct names" {
+    const allocator = std.testing.allocator;
+
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+
+    const directory_url = try allocator.dupeZ(
+        u8,
+        "https://example.com/.well-known/http-message-signatures-directory",
+    );
+
+    var auth = WebBotAuth{
+        .pkey = pkey,
+        .keyid = "test-key-id",
+        .directory_url = directory_url,
+    };
+    defer auth.deinit(allocator);
+
+    var headers = try Http.Headers.init("User-Agent: Test-Agent");
+    defer headers.deinit();
+
+    try auth.signRequest(allocator, &headers, "example.com");
+
+    var it = headers.iterator();
+    var found_sig_agent = false;
+    var found_sig_input = false;
+    var found_signature = false;
+    var count: usize = 0;
+
+    while (it.next()) |h| {
+        count += 1;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true;
+    }
+
+    try std.testing.expect(count >= 3);
+    try std.testing.expect(found_sig_agent);
+    try std.testing.expect(found_sig_input);
+    try std.testing.expect(found_signature);
+}

src/browser/markdown.zig

@@ -19,6 +19,8 @@
 const std = @import("std");
 
 const Page = @import("Page.zig");
+const URL = @import("URL.zig");
+const TreeWalker = @import("webapi/TreeWalker.zig");
 const CData = @import("webapi/CData.zig");
 const Element = @import("webapi/Element.zig");
 const Node = @import("webapi/Node.zig");
@@ -103,20 +105,37 @@ fn isVisibleElement(el: *Element) bool {
     };
 }
 
+fn getAnchorLabel(el: *Element) ?[]const u8 {
+    return el.getAttributeSafe(comptime .wrap("aria-label")) orelse el.getAttributeSafe(comptime .wrap("title"));
+}
+
 fn isAllWhitespace(text: []const u8) bool {
     return for (text) |c| {
         if (!std.ascii.isWhitespace(c)) break false;
     } else true;
 }
 
-fn hasBlockDescendant(node: *Node) bool {
-    var it = node.childrenIterator();
-    return while (it.next()) |child| {
-        if (child.is(Element)) |el| {
-            if (isBlock(el.getTag())) break true;
-            if (hasBlockDescendant(child)) break true;
+fn hasBlockDescendant(root: *Node) bool {
+    var tw = TreeWalker.FullExcludeSelf.Elements.init(root, .{});
+    while (tw.next()) |el| {
+        if (isBlock(el.getTag())) return true;
+    }
+    return false;
+}
+
+fn hasVisibleContent(root: *Node) bool {
+    var tw = TreeWalker.FullExcludeSelf.init(root, .{});
+    while (tw.next()) |node| {
+        if (isSignificantText(node)) return true;
+        if (node.is(Element)) |el| {
+            if (!isVisibleElement(el)) {
+                tw.skipChildren();
+            } else if (el.getTag() == .img) {
+                return true;
+            }
         }
-    } else false;
+    }
+    return false;
 }
 
 fn ensureNewline(state: *State, writer: *std.Io.Writer) !void {
@@ -278,20 +297,29 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
             }
             try writer.writeAll("](");
             if (el.getAttributeSafe(comptime .wrap("src"))) |src| {
-                try writer.writeAll(src);
+                const absolute_src = URL.resolve(page.call_arena, page.base(), src, .{ .encode = true }) catch src;
+                try writer.writeAll(absolute_src);
             }
             try writer.writeAll(")");
             state.last_char_was_newline = false;
             return;
         },
         .anchor => {
+            const has_content = hasVisibleContent(el.asNode());
+            const label = getAnchorLabel(el);
+            const href_raw = el.getAttributeSafe(comptime .wrap("href"));
+
+            if (!has_content and label == null and href_raw == null) return;
+
             const has_block = hasBlockDescendant(el.asNode());
+            const href = if (href_raw) |h| URL.resolve(page.call_arena, page.base(), h, .{ .encode = true }) catch h else null;
+
             if (has_block) {
                 try renderChildren(el.asNode(), state, writer, page);
-                if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
+                if (href) |h| {
                     if (!state.last_char_was_newline) try writer.writeByte('\n');
-                    try writer.writeAll("([Link](");
-                    try writer.writeAll(href);
+                    try writer.writeAll("([](");
+                    try writer.writeAll(h);
                     try writer.writeAll("))\n");
                     state.last_char_was_newline = true;
                 }
@@ -301,10 +329,14 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
             if (isStandaloneAnchor(el)) {
                 if (!state.last_char_was_newline) try writer.writeByte('\n');
                 try writer.writeByte('[');
-                try renderChildren(el.asNode(), state, writer, page);
+                if (has_content) {
+                    try renderChildren(el.asNode(), state, writer, page);
+                } else {
+                    try writer.writeAll(label orelse "");
+                }
                 try writer.writeAll("](");
-                if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
-                    try writer.writeAll(href);
+                if (href) |h| {
+                    try writer.writeAll(h);
                 }
                 try writer.writeAll(")\n");
                 state.last_char_was_newline = true;
@@ -312,10 +344,14 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
             }
 
             try writer.writeByte('[');
-            try renderChildren(el.asNode(), state, writer, page);
+            if (has_content) {
+                try renderChildren(el.asNode(), state, writer, page);
+            } else {
+                try writer.writeAll(label orelse "");
+            }
             try writer.writeAll("](");
-            if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
-                try writer.writeAll(href);
+            if (href) |h| {
+                try writer.writeAll(h);
             }
             try writer.writeByte(')');
             state.last_char_was_newline = false;
@@ -452,6 +488,8 @@ fn testMarkdownHTML(html: []const u8, expected: []const u8) !void {
     const testing = @import("../testing.zig");
     const page = try testing.test_session.createPage();
     defer testing.test_session.removePage();
+    page.url = "http://localhost/";
+
     const doc = page.window._document;
 
     const div = try doc.createElement("div", null, page);
@@ -520,11 +558,11 @@ test "browser.markdown: blockquote" {
 }
 
 test "browser.markdown: links" {
-    try testMarkdownHTML("<a href=\"https://lightpanda.io\">Lightpanda</a>", "[Lightpanda](https://lightpanda.io)\n");
+    try testMarkdownHTML("<a href=\"/relative\">Link</a>", "[Link](http://localhost/relative)\n");
 }
 
 test "browser.markdown: images" {
-    try testMarkdownHTML("<img src=\"logo.png\" alt=\"Logo\">", "![Logo](logo.png)\n");
+    try testMarkdownHTML("<img src=\"logo.png\" alt=\"Logo\">", "![Logo](http://localhost/logo.png)\n");
 }
 
 test "browser.markdown: headings" {
@@ -565,7 +603,7 @@ test "browser.markdown: block link" {
         \\### Title
         \\
         \\Description
-        \\([Link](https://example.com))
+        \\([](https://example.com))
         \\
     );
 }
@@ -588,8 +626,8 @@ test "browser.markdown: standalone anchors" {
         \\  <a href="2">Link 2</a>
         \\</main>
     ,
-        \\[Link 1](1)
-        \\[Link 2](2)
+        \\[Link 1](http://localhost/1)
+        \\[Link 2](http://localhost/2)
         \\
     );
 }
@@ -601,7 +639,58 @@ test "browser.markdown: mixed anchors in main" {
         \\  Welcome <a href="1">Link 1</a>.
         \\</main>
     ,
-        \\Welcome [Link 1](1). 
+        \\Welcome [Link 1](http://localhost/1). 
         \\
     );
 }
+
+test "browser.markdown: skip empty links" {
+    try testMarkdownHTML(
+        \\<a href="/"></a>
+        \\<a href="/"><svg></svg></a>
+    ,
+        \\[](http://localhost/)
+        \\[](http://localhost/)
+        \\
+    );
+}
+
+test "browser.markdown: resolve links" {
+    const testing = @import("../testing.zig");
+    const page = try testing.test_session.createPage();
+    defer testing.test_session.removePage();
+    page.url = "https://example.com/a/index.html";
+
+    const doc = page.window._document;
+    const div = try doc.createElement("div", null, page);
+    try page.parseHtmlAsChildren(div.asNode(),
+        \\<a href="b">Link</a>
+        \\<img src="../c.png" alt="Img">
+        \\<a href="/my page">Space</a>
+    );
+
+    var aw: std.Io.Writer.Allocating = .init(testing.allocator);
+    defer aw.deinit();
+    try dump(div.asNode(), .{}, &aw.writer, page);
+
+    try testing.expectString(
+        \\[Link](https://example.com/a/b)
+        \\![Img](https://example.com/c.png) 
+        \\[Space](https://example.com/my%20page)
+        \\
+    , aw.written());
+}
+
+test "browser.markdown: anchor fallback label" {
+    try testMarkdownHTML(
+        \\<a href="/discord" aria-label="Discord Server"><svg></svg></a>
+    , "[Discord Server](http://localhost/discord)\n");
+
+    try testMarkdownHTML(
+        \\<a href="/search" title="Search Site"><svg></svg></a>
+    , "[Search Site](http://localhost/search)\n");
+
+    try testMarkdownHTML(
+        \\<a href="/no-label"><svg></svg></a>
+    , "[](http://localhost/no-label)\n");
+}

src/browser/webapi/TreeWalker.zig

@@ -31,6 +31,7 @@ const Mode = enum {
 
 pub fn TreeWalker(comptime mode: Mode) type {
     return struct {
+        _current: ?*Node = null,
         _next: ?*Node,
         _root: *Node,
 
@@ -47,37 +48,46 @@ pub fn TreeWalker(comptime mode: Mode) type {
 
         pub fn next(self: *Self) ?*Node {
             const node = self._next orelse return null;
+            self._current = node;
 
             if (comptime mode == .children) {
-                self._next = Node.linkToNodeOrNull(node._child_link.next);
+                self._next = node.nextSibling();
                 return node;
             }
 
-            if (node._children) |children| {
-                self._next = children.first();
-            } else if (node._child_link.next) |n| {
-                self._next = Node.linkToNode(n);
+            if (node.firstChild()) |child| {
+                self._next = child;
             } else {
-                // No children, no next sibling - walk up until we find a next sibling or hit root
-                var current = node._parent;
-                while (current) |parent| {
-                    if (parent == self._root) {
-                        self._next = null;
-                        break;
+                var current: *Node = node;
+                while (current != self._root) {
+                    if (current.nextSibling()) |sibling| {
+                        self._next = sibling;
+                        return node;
                     }
-                    if (parent._child_link.next) |next_sibling| {
-                        self._next = Node.linkToNode(next_sibling);
-                        break;
-                    }
-                    current = parent._parent;
-                } else {
-                    self._next = null;
+                    current = current._parent orelse break;
                 }
+                self._next = null;
             }
             return node;
         }
 
+        pub fn skipChildren(self: *Self) void {
+            if (comptime mode == .children) return;
+            const current_node = self._current orelse return;
+
+            var current: *Node = current_node;
+            while (current != self._root) {
+                if (current.nextSibling()) |sibling| {
+                    self._next = sibling;
+                    return;
+                }
+                current = current._parent orelse break;
+            }
+            self._next = null;
+        }
+
         pub fn reset(self: *Self) void {
+            self._current = null;
             self._next = firstNext(self._root);
         }
 
@@ -147,3 +157,38 @@ pub fn TreeWalker(comptime mode: Mode) type {
         };
     };
 }
+
+test "TreeWalker: skipChildren" {
+    const testing = @import("../../testing.zig");
+    const page = try testing.test_session.createPage();
+    defer testing.test_session.removePage();
+    const doc = page.window._document;
+
+    // <div>
+    //   <span>
+    //     <b>A</b>
+    //   </span>
+    //   <p>B</p>
+    // </div>
+    const div = try doc.createElement("div", null, page);
+    const span = try doc.createElement("span", null, page);
+    const b = try doc.createElement("b", null, page);
+    const p = try doc.createElement("p", null, page);
+    _ = try span.asNode().appendChild(b.asNode(), page);
+    _ = try div.asNode().appendChild(span.asNode(), page);
+    _ = try div.asNode().appendChild(p.asNode(), page);
+
+    var tw = Full.init(div.asNode(), .{});
+
+    // root (div)
+    try testing.expect(tw.next() == div.asNode());
+
+    // span
+    try testing.expect(tw.next() == span.asNode());
+
+    // skip children of span (should jump over <b> to <p>)
+    tw.skipChildren();
+    try testing.expect(tw.next() == p.asNode());
+
+    try testing.expect(tw.next() == null);
+}

src/browser/webapi/encoding/TextDecoderStream.zig

@@ -72,6 +72,14 @@ pub fn init(label_: ?[]const u8, opts_: ?InitOpts, page: *Page) !TextDecoderStre
     };
 }
 
+pub fn acquireRef(self: *TextDecoderStream) void {
+    self._transform.acquireRef();
+}
+
+pub fn deinit(self: *TextDecoderStream, shutdown: bool, page: *Page) void {
+    self._transform.deinit(shutdown, page);
+}
+
 fn decodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value, ignoreBOM: bool) !void {
     // chunk should be a Uint8Array; decode it as UTF-8 string
     const typed_array = try chunk.toZig(js.TypedArray(u8));
@@ -111,6 +119,8 @@ pub const JsApi = struct {
         pub const name = "TextDecoderStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
+        pub const weak = true;
+        pub const finalizer = bridge.finalizer(TextDecoderStream.deinit);
     };
 
     pub const constructor = bridge.constructor(TextDecoderStream.init, .{});

src/browser/webapi/encoding/TextEncoderStream.zig

@@ -34,6 +34,14 @@ pub fn init(page: *Page) !TextEncoderStream {
     };
 }
 
+pub fn acquireRef(self: *TextEncoderStream) void {
+    self._transform.acquireRef();
+}
+
+pub fn deinit(self: *TextEncoderStream, shutdown: bool, page: *Page) void {
+    self._transform.deinit(shutdown, page);
+}
+
 fn encodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value) !void {
     // chunk should be a JS string; encode it as UTF-8 bytes (Uint8Array)
     const str = chunk.isString() orelse return error.InvalidChunk;
@@ -56,6 +64,8 @@ pub const JsApi = struct {
         pub const name = "TextEncoderStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
+        pub const weak = true;
+        pub const finalizer = bridge.finalizer(TextEncoderStream.deinit);
     };
 
     pub const constructor = bridge.constructor(TextEncoderStream.init, .{});

src/browser/webapi/streams/ReadableStream.zig

@@ -52,6 +52,8 @@ _pull_fn: ?js.Function.Global = null,
 _pulling: bool = false,
 _pull_again: bool = false,
 _cancel: ?Cancel = null,
+_arena: std.mem.Allocator,
+_rc: usize = 0,
 
 const UnderlyingSource = struct {
     start: ?js.Function = null,
@@ -68,13 +70,18 @@ const QueueingStrategy = struct {
 pub fn init(src_: ?UnderlyingSource, strategy_: ?QueueingStrategy, page: *Page) !*ReadableStream {
     const strategy: QueueingStrategy = strategy_ orelse .{};
 
-    const self = try page._factory.create(ReadableStream{
+    const arena = try page.getArena(.{ .debug = "ReadableStream" });
+    errdefer page.releaseArena(arena);
+
+    const self = try arena.create(ReadableStream);
+    self.* = .{
         ._page = page,
         ._state = .readable,
+        ._arena = arena,
         ._reader = null,
         ._controller = undefined,
         ._stored_error = null,
-    });
+    };
 
     self._controller = try ReadableStreamDefaultController.init(self, strategy.highWaterMark, page);
 
@@ -108,6 +115,23 @@ pub fn initWithData(data: []const u8, page: *Page) !*ReadableStream {
     return stream;
 }
 
+pub fn deinit(self: *ReadableStream, _: bool, page: *Page) void {
+    const rc = self._rc;
+    if (comptime IS_DEBUG) {
+        std.debug.assert(rc != 0);
+    }
+
+    if (rc == 1) {
+        page.releaseArena(self._arena);
+    } else {
+        self._rc = rc - 1;
+    }
+}
+
+pub fn acquireRef(self: *ReadableStream) void {
+    self._rc += 1;
+}
+
 pub fn getReader(self: *ReadableStream, page: *Page) !*ReadableStreamDefaultReader {
     if (self.getLocked()) {
         return error.ReaderLocked;
@@ -120,6 +144,12 @@ pub fn getReader(self: *ReadableStream, page: *Page) !*ReadableStreamDefaultRead
 
 pub fn releaseReader(self: *ReadableStream) void {
     self._reader = null;
+
+    const rc = self._rc;
+    if (comptime IS_DEBUG) {
+        std.debug.assert(rc != 0);
+    }
+    self._rc = rc - 1;
 }
 
 pub fn getAsyncIterator(self: *ReadableStream, page: *Page) !*AsyncIterator {
@@ -367,6 +397,8 @@ pub const JsApi = struct {
         pub const name = "ReadableStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
+        pub const weak = true;
+        pub const finalizer = bridge.finalizer(ReadableStream.deinit);
     };
 
     pub const constructor = bridge.constructor(ReadableStream.init, .{});
@@ -390,6 +422,14 @@ pub const AsyncIterator = struct {
         });
     }
 
+    pub fn acquireRef(self: *AsyncIterator) void {
+        self._stream.acquireRef();
+    }
+
+    pub fn deinit(self: *AsyncIterator, shutdown: bool, page: *Page) void {
+        self._stream.deinit(shutdown, page);
+    }
+
     pub fn next(self: *AsyncIterator, page: *Page) !js.Promise {
         return self._reader.read(page);
     }
@@ -406,6 +446,8 @@ pub const AsyncIterator = struct {
             pub const name = "ReadableStreamAsyncIterator";
             pub const prototype_chain = bridge.prototypeChain();
             pub var class_id: bridge.ClassId = undefined;
+            pub const weak = true;
+            pub const finalizer = bridge.finalizer(AsyncIterator.deinit);
         };
 
         pub const next = bridge.function(ReadableStream.AsyncIterator.next, .{});

src/browser/webapi/streams/ReadableStreamDefaultController.zig

@@ -27,6 +27,8 @@ const ReadableStreamDefaultReader = @import("ReadableStreamDefaultReader.zig");
 
 const IS_DEBUG = @import("builtin").mode == .Debug;
 
+/// ReadableStreamDefaultController uses ReadableStream's arena to make
+/// allocation. Indeed, the controller is owned by its ReadableStream.
 const ReadableStreamDefaultController = @This();
 
 pub const Chunk = union(enum) {
@@ -46,7 +48,6 @@ pub const Chunk = union(enum) {
 
 _page: *Page,
 _stream: *ReadableStream,
-_arena: std.mem.Allocator,
 _queue: std.ArrayList(Chunk),
 _pending_reads: std.ArrayList(js.PromiseResolver.Global),
 _high_water_mark: u32,
@@ -56,15 +57,22 @@ pub fn init(stream: *ReadableStream, high_water_mark: u32, page: *Page) !*Readab
         ._page = page,
         ._queue = .empty,
         ._stream = stream,
-        ._arena = page.arena,
         ._pending_reads = .empty,
         ._high_water_mark = high_water_mark,
     });
 }
 
+pub fn acquireRef(self: *ReadableStreamDefaultController) void {
+    self._stream.acquireRef();
+}
+
+pub fn deinit(self: *ReadableStreamDefaultController, shutdown: bool, page: *Page) void {
+    self._stream.deinit(shutdown, page);
+}
+
 pub fn addPendingRead(self: *ReadableStreamDefaultController, page: *Page) !js.Promise {
     const resolver = page.js.local.?.createPromiseResolver();
-    try self._pending_reads.append(self._arena, try resolver.persist());
+    try self._pending_reads.append(self._stream._arena, try resolver.persist());
     return resolver.promise();
 }
 
@@ -74,8 +82,8 @@ pub fn enqueue(self: *ReadableStreamDefaultController, chunk: Chunk) !void {
     }
 
     if (self._pending_reads.items.len == 0) {
-        const chunk_copy = try chunk.dupe(self._page.arena);
-        return self._queue.append(self._arena, chunk_copy);
+        const chunk_copy = try chunk.dupe(self._stream._arena);
+        return self._queue.append(self._stream._arena, chunk_copy);
     }
 
     // I know, this is ouch! But we expect to have very few (if any)
@@ -109,7 +117,7 @@ pub fn enqueueValue(self: *ReadableStreamDefaultController, value: js.Value) !vo
 
     if (self._pending_reads.items.len == 0) {
         const persisted = try value.persist();
-        try self._queue.append(self._arena, .{ .js_value = persisted });
+        try self._queue.append(self._stream._arena, .{ .js_value = persisted });
         return;
     }
 
@@ -170,7 +178,7 @@ pub fn doError(self: *ReadableStreamDefaultController, err: []const u8) !void {
     }
 
     self._stream._state = .errored;
-    self._stream._stored_error = try self._page.arena.dupe(u8, err);
+    self._stream._stored_error = try self._stream._arena.dupe(u8, err);
 
     // Reject all pending reads
     for (self._pending_reads.items) |resolver| {
@@ -210,6 +218,8 @@ pub const JsApi = struct {
         pub const name = "ReadableStreamDefaultController";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
+        pub const weak = true;
+        pub const finalizer = bridge.finalizer(ReadableStreamDefaultController.deinit);
     };
 
     pub const enqueue = bridge.function(ReadableStreamDefaultController.enqueueValue, .{});

src/browser/webapi/streams/ReadableStreamDefaultReader.zig

@@ -19,6 +19,8 @@
 const std = @import("std");
 const js = @import("../../js/js.zig");
 
+const IS_DEBUG = @import("builtin").mode == .Debug;
+
 const Page = @import("../../Page.zig");
 const ReadableStream = @import("ReadableStream.zig");
 const ReadableStreamDefaultController = @import("ReadableStreamDefaultController.zig");
@@ -35,6 +37,21 @@ pub fn init(stream: *ReadableStream, page: *Page) !*ReadableStreamDefaultReader
     });
 }
 
+pub fn acquireRef(self: *ReadableStreamDefaultReader) void {
+    const stream = self._stream orelse {
+        if (comptime IS_DEBUG) {
+            std.debug.assert(false);
+        }
+        return;
+    };
+    stream.acquireRef();
+}
+
+pub fn deinit(self: *ReadableStreamDefaultReader, shutdown: bool, page: *Page) void {
+    const stream = self._stream orelse return;
+    stream.deinit(shutdown, page);
+}
+
 pub const ReadResult = struct {
     done: bool,
     value: Chunk,
@@ -110,6 +127,8 @@ pub const JsApi = struct {
         pub const name = "ReadableStreamDefaultReader";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
+        pub const weak = true;
+        pub const finalizer = bridge.finalizer(ReadableStreamDefaultReader.deinit);
     };
 
     pub const read = bridge.function(ReadableStreamDefaultReader.read, .{});

src/browser/webapi/streams/TransformStream.zig

@@ -85,6 +85,14 @@ pub fn initWithZigTransform(zig_transform: ZigTransformFn, page: *Page) !*Transf
     return self;
 }
 
+pub fn acquireRef(self: *TransformStream) void {
+    self._readable.acquireRef();
+}
+
+pub fn deinit(self: *TransformStream, shutdown: bool, page: *Page) void {
+    self._readable.deinit(shutdown, page);
+}
+
 pub fn transformWrite(self: *TransformStream, chunk: js.Value, page: *Page) !void {
     if (self._controller._zig_transform_fn) |zig_fn| {
         // Zig-level transform (used by TextEncoderStream etc.)
@@ -130,6 +138,8 @@ pub const JsApi = struct {
         pub const name = "TransformStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
+        pub const weak = true;
+        pub const finalizer = bridge.finalizer(TransformStream.deinit);
     };
 
     pub const constructor = bridge.constructor(TransformStream.init, .{});
@@ -165,6 +175,14 @@ pub const TransformStreamDefaultController = struct {
         });
     }
 
+    pub fn acquireRef(self: *TransformStreamDefaultController) void {
+        self._stream.acquireRef();
+    }
+
+    pub fn deinit(self: *TransformStreamDefaultController, shutdown: bool, page: *Page) void {
+        self._stream.deinit(shutdown, page);
+    }
+
     pub fn enqueue(self: *TransformStreamDefaultController, chunk: ReadableStreamDefaultController.Chunk) !void {
         try self._stream._readable._controller.enqueue(chunk);
     }
@@ -189,6 +207,8 @@ pub const TransformStreamDefaultController = struct {
             pub const name = "TransformStreamDefaultController";
             pub const prototype_chain = bridge.prototypeChain();
             pub var class_id: bridge.ClassId = undefined;
+            pub const weak = true;
+            pub const finalizer = bridge.finalizer(TransformStreamDefaultController.deinit);
         };
 
         pub const enqueue = bridge.function(TransformStreamDefaultController.enqueueValue, .{});

src/crypto.zig

@@ -228,6 +228,8 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8
 
 pub const NID_X25519 = @as(c_int, 948);
 pub const EVP_PKEY_X25519 = NID_X25519;
+pub const NID_ED25519 = 949;
+pub const EVP_PKEY_ED25519 = NID_ED25519;
 
 pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
 pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
@@ -236,3 +238,11 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void;
 pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int;
 pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int;
 pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int;
+pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void;
+
+pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int;
+pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int;
+pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX;
+pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void;
+pub const struct_evp_md_ctx_st = opaque {};
+pub const EVP_MD_CTX = struct_evp_md_ctx_st;

src/http/Client.zig

@@ -29,6 +29,8 @@ const Notification = @import("../Notification.zig");
 const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
 const Robots = @import("../browser/Robots.zig");
 const RobotStore = Robots.RobotStore;
+const WebBotAuth = @import("../browser/WebBotAuth.zig");
+
 const posix = std.posix;
 
 const Allocator = std.mem.Allocator;
@@ -83,6 +85,9 @@ robot_store: *RobotStore,
 // Allows us to fetch the robots.txt just once.
 pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empty,
 
+// Reference to the App-owned WebBotAuth.
+web_bot_auth: *const ?WebBotAuth,
+
 // Once we have a handle/easy to process a request with, we create a Transfer
 // which contains the Request as well as any state we need to process the
 // request. These wil come and go with each request.
@@ -121,7 +126,13 @@ pub const CDPClient = struct {
 
 const TransferQueue = std.DoublyLinkedList;
 
-pub fn init(allocator: Allocator, ca_blob: ?Net.Blob, robot_store: *RobotStore, config: *const Config) !*Client {
+pub fn init(
+    allocator: Allocator,
+    ca_blob: ?Net.Blob,
+    robot_store: *RobotStore,
+    web_bot_auth: *const ?WebBotAuth,
+    config: *const Config,
+) !*Client {
     var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
     errdefer transfer_pool.deinit();
 
@@ -145,6 +156,7 @@ pub fn init(allocator: Allocator, ca_blob: ?Net.Blob, robot_store: *RobotStore,
         .handles = handles,
         .allocator = allocator,
         .robot_store = robot_store,
+        .web_bot_auth = web_bot_auth,
         .http_proxy = http_proxy,
         .use_proxy = http_proxy != null,
         .config = config,
@@ -709,6 +721,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
         try conn.secretHeaders(&header_list, &self.config.http_headers); // Add headers that must be hidden from intercepts
         try conn.setHeaders(&header_list);
 
+        // If we have WebBotAuth, sign our request.
+        if (self.web_bot_auth.*) |wba| {
+            const authority = URL.getHost(req.url);
+            try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
+        }
+
         // Add cookies.
         if (header_list.cookies) |cookies| {
             try conn.setCookies(cookies);
@@ -1302,7 +1320,7 @@ pub const Transfer = struct {
             }
             transfer._redirecting = false;
 
-            if (status == 401 or status == 407) {
+            if ((status == 401 or status == 407) and transfer.client.use_proxy) {
                 // The auth challenge must be parsed from a following
                 // WWW-Authenticate or Proxy-Authenticate header.
                 transfer._auth_challenge = .{

src/http/Http.zig

@@ -29,6 +29,7 @@ pub const Headers = Net.Headers;
 
 const Config = @import("../Config.zig");
 const RobotStore = @import("../browser/Robots.zig").RobotStore;
+const WebBotAuth = @import("../browser/WebBotAuth.zig");
 
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
@@ -45,8 +46,14 @@ allocator: Allocator,
 config: *const Config,
 ca_blob: ?Net.Blob,
 robot_store: *RobotStore,
+web_bot_auth: *const ?WebBotAuth,
 
-pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Config) !Http {
+pub fn init(
+    allocator: Allocator,
+    robot_store: *RobotStore,
+    web_bot_auth: *const ?WebBotAuth,
+    config: *const Config,
+) !Http {
     try Net.globalInit();
     errdefer Net.globalDeinit();
 
@@ -68,6 +75,7 @@ pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Confi
         .config = config,
         .ca_blob = ca_blob,
         .robot_store = robot_store,
+        .web_bot_auth = web_bot_auth,
     };
 }
 
@@ -81,7 +89,7 @@ pub fn deinit(self: *Http) void {
 }
 
 pub fn createClient(self: *Http, allocator: Allocator) !*Client {
-    return Client.init(allocator, self.ca_blob, self.robot_store, self.config);
+    return Client.init(allocator, self.ca_blob, self.robot_store, self.web_bot_auth, self.config);
 }
 
 pub fn newConnection(self: *Http) !Net.Connection {

src/mcp/Server.zig

@@ -24,22 +24,29 @@ mutex: std.Thread.Mutex = .{},
 aw: std.io.Writer.Allocating,
 
 pub fn init(allocator: std.mem.Allocator, app: *App, writer: *std.io.Writer) !*Self {
+    const http_client = try app.http.createClient(allocator);
+    errdefer http_client.deinit();
+
+    const notification = try lp.Notification.init(allocator);
+    errdefer notification.deinit();
+
     const self = try allocator.create(Self);
     errdefer allocator.destroy(self);
 
-    self.allocator = allocator;
-    self.app = app;
-    self.writer = writer;
-    self.aw = .init(allocator);
+    var browser = try lp.Browser.init(app, .{ .http_client = http_client });
+    errdefer browser.deinit();
 
-    self.http_client = try app.http.createClient(allocator);
-    errdefer self.http_client.deinit();
-
-    self.notification = try .init(allocator);
-    errdefer self.notification.deinit();
-
-    self.browser = try lp.Browser.init(app, .{ .http_client = self.http_client });
-    errdefer self.browser.deinit();
+    self.* = .{
+        .allocator = allocator,
+        .app = app,
+        .writer = writer,
+        .browser = browser,
+        .aw = .init(allocator),
+        .http_client = http_client,
+        .notification = notification,
+        .session = undefined,
+        .page = undefined,
+    };
 
     self.session = try self.browser.newSession(self.notification);
     self.page = try self.session.createPage();