From 4d7b7d1d42acbf70750eee9091fb5a43cc7e95fa Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 19 Feb 2026 12:05:45 -0800 Subject: [PATCH 01/13] add web bot auth args --- src/Config.zig | 52 ++++++++++++++++++++++++++++++++++++++ src/network/WebBotAuth.zig | 23 +++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 src/network/WebBotAuth.zig diff --git a/src/Config.zig b/src/Config.zig index 73c7f3a7..680a99a8 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator; const log = @import("log.zig"); const dump = @import("browser/dump.zig"); +const WebBotAuth = @import("browser/WebBotAuth.zig"); + pub const RunMode = enum { help, fetch, @@ -161,6 +163,17 @@ pub fn cdpTimeout(self: *const Config) usize { }; } +pub fn webBotAuth(self: *const Config) ?WebBotAuth { + return switch (self.mode) { + inline .serve, .fetch, .mcp => |opts| WebBotAuth{ + .key_file = opts.common.web_bot_auth_key_file orelse return null, + .keyid = opts.common.web_bot_auth_keyid orelse return null, + .domain = opts.common.web_bot_auth_domain orelse return null, + }, + .help, .version => null, + }; +} + pub fn maxConnections(self: *const Config) u16 { return switch (self.mode) { .serve => |opts| opts.cdp_max_connections, @@ -227,6 +240,10 @@ pub const Common = struct { log_format: ?log.Format = null, log_filter_scopes: ?[]log.Scope = null, user_agent_suffix: ?[]const u8 = null, + + web_bot_auth_key_file: ?[]const u8 = null, + web_bot_auth_keyid: ?[]const u8 = null, + web_bot_auth_domain: ?[]const u8 = null, }; /// Pre-formatted HTTP headers for reuse across Http and Client. @@ -334,6 +351,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void { \\--user_agent_suffix \\ Suffix to append to the Lightpanda/X.Y User-Agent \\ + \\--web_bot_auth_key_file + \\ Path to the Ed25519 private key PEM file. + \\ + \\--web_bot_auth_keyid + \\ The JWK thumbprint of your public key. + \\ + \\--web_bot_auth_directory + \\ Your domain e.g. yourdomain.com ; // MAX_HELP_LEN| @@ -855,5 +880,32 @@ fn parseCommonArg( return true; } + if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" }); + return error.InvalidArgument; + }; + common.web_bot_auth_key_file = try allocator.dupe(u8, str); + return true; + } + + if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" }); + return error.InvalidArgument; + }; + common.web_bot_auth_keyid = try allocator.dupe(u8, str); + return true; + } + + if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" }); + return error.InvalidArgument; + }; + common.web_bot_auth_domain = try allocator.dupe(u8, str); + return true; + } + return false; } diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig new file mode 100644 index 00000000..4af598fb --- /dev/null +++ b/src/network/WebBotAuth.zig @@ -0,0 +1,23 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const WebBotAuth = @This(); + +key_file: []const u8, +keyid: []const u8, +domain: []const u8, From 6cd8202310d2c3b6815f92bf001ec1c16b99e2d0 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 19 Feb 2026 12:29:50 -0800 Subject: [PATCH 02/13] add WebBotAuth and support for ed25119 to crypto --- src/crypto.zig | 10 +++ src/network/WebBotAuth.zig | 149 ++++++++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/src/crypto.zig b/src/crypto.zig index 012ca96e..58f6349f 100644 --- a/src/crypto.zig +++ b/src/crypto.zig @@ -228,6 +228,8 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8 pub const NID_X25519 = @as(c_int, 948); pub const EVP_PKEY_X25519 = NID_X25519; +pub const NID_ED25519 = 949; +pub const EVP_PKEY_ED25519 = NID_ED25519; pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY; pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY; @@ -236,3 +238,11 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void; pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int; pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int; pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int; +pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void; + +pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int; +pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int; +pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX; +pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void; +pub const struct_evp_md_ctx_st = opaque {}; +pub const EVP_MD_CTX = struct_evp_md_ctx_st; diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig index 4af598fb..842f02d2 100644 --- a/src/network/WebBotAuth.zig +++ b/src/network/WebBotAuth.zig @@ -16,8 +16,153 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +const std = @import("std"); +const crypto = @import("../crypto.zig"); + +const Http = @import("../http/Http.zig"); + const WebBotAuth = @This(); -key_file: []const u8, +pkey: *crypto.EVP_PKEY, keyid: []const u8, -domain: []const u8, +directory_url: [:0]const u8, + +pub const Config = struct { + key_file: []const u8, + keyid: []const u8, + domain: []const u8, +}; + +fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY { + const begin = "-----BEGIN PRIVATE KEY-----"; + const end = "-----END PRIVATE KEY-----"; + const start_idx = std.mem.indexOf(u8, pem, begin) orelse return error.InvalidPem; + const end_idx = std.mem.indexOf(u8, pem, end) orelse return error.InvalidPem; + + const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace); + + // strip newlines from b64 + var clean: [4096]u8 = undefined; + var clean_len: usize = 0; + for (b64) |ch| { + if (ch != '\n' and ch != '\r') { + clean[clean_len] = ch; + clean_len += 1; + } + } + + var der: [128]u8 = undefined; + const decoded_len = try std.base64.standard.Decoder.calcSizeForSlice(clean[0..clean_len]); + try std.base64.standard.Decoder.decode(der[0..decoded_len], clean[0..clean_len]); + + // Ed25519 PKCS#8: key bytes are at offset 16, 32 bytes long + const key_bytes = der[16..48]; + + const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32); + return pkey orelse error.InvalidKey; +} + +fn signEd25519(pkey: *crypto.EVP_PKEY, message: []const u8, out: *[64]u8) !void { + const ctx = crypto.EVP_MD_CTX_new() orelse return error.OutOfMemory; + defer crypto.EVP_MD_CTX_free(ctx); + + if (crypto.EVP_DigestSignInit(ctx, null, null, null, pkey) != 1) + return error.SignInit; + + var sig_len: usize = 64; + if (crypto.EVP_DigestSign(ctx, out.ptr, &sig_len, message.ptr, message.len) != 1) + return error.SignFailed; +} + +pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAuth { + const pem = try std.fs.cwd().readFileAlloc(allocator, config.key_file, 1024 * 4); + defer allocator.free(pem); + + const pkey = try parsePemPrivateKey(pem); + errdefer crypto.EVP_PKEY_free(pkey); + + const directory_url = try std.fmt.allocPrintSentinel( + allocator, + "https://{s}/.well-known/http-message-signatures-directory", + .{config.domain}, + 0, + ); + errdefer allocator.free(directory_url); + + return .{ + .pkey = pkey, + // Owned by the Config so it's okay. + .keyid = config.keyid, + .directory_url = directory_url, + }; +} + +pub fn signRequest( + self: *const WebBotAuth, + allocator: std.mem.Allocator, + headers: *Http.Headers, + authority: []const u8, +) !void { + const now = std.time.timestamp(); + const expires = now + 60; + + // build the signature-input value (without the sig1= label) + const sig_input_value = try std.fmt.allocPrint( + allocator, + "(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"", + .{ now, expires, self.keyid }, + ); + defer allocator.free(sig_input_value); + + // build the canonical string to sign + const canonical = try std.fmt.allocPrint( + allocator, + "\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}", + .{ authority, self.directory_url, sig_input_value }, + ); + defer allocator.free(canonical); + + // sign it + var sig: [64]u8 = undefined; + try signEd25519(self.pkey, canonical, &sig); + + // base64 encode + const encoded_len = std.base64.standard.Encoder.calcSize(sig.len); + const encoded = try allocator.alloc(u8, encoded_len); + defer allocator.free(encoded); + _ = std.base64.standard.Encoder.encode(encoded, &sig); + + // build the 3 headers and add them + const sig_agent = try std.fmt.allocPrintSentinel( + allocator, + "Signature-Agent: \"{s}\"", + .{self.directory_url}, + 0, + ); + defer allocator.free(sig_agent); + + const sig_input = try std.fmt.allocPrintSentinel( + allocator, + "Signature-Input: sig1={s}", + .{sig_input_value}, + 0, + ); + defer allocator.free(sig_input); + + const signature = try std.fmt.allocPrintSentinel( + allocator, + "Signature: sig1=:{s}:", + .{encoded}, + 0, + ); + defer allocator.free(signature); + + try headers.add(sig_agent); + try headers.add(sig_input); + try headers.add(signature); +} + +pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void { + crypto.EVP_PKEY_free(self.pkey); + allocator.free(self.directory_url); +} From 02198de455710094c36938577ff029ad2a39b82d Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 19 Feb 2026 12:30:03 -0800 Subject: [PATCH 03/13] add support for WebBotAuth in Client --- src/Config.zig | 6 +++--- src/browser/HttpClient.zig | 7 +++++++ src/network/Runtime.zig | 11 +++++++++++ src/network/WebBotAuth.zig | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Config.zig b/src/Config.zig index 680a99a8..21fdeed4 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -23,7 +23,7 @@ const Allocator = std.mem.Allocator; const log = @import("log.zig"); const dump = @import("browser/dump.zig"); -const WebBotAuth = @import("browser/WebBotAuth.zig"); +const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config; pub const RunMode = enum { help, @@ -163,9 +163,9 @@ pub fn cdpTimeout(self: *const Config) usize { }; } -pub fn webBotAuth(self: *const Config) ?WebBotAuth { +pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig { return switch (self.mode) { - inline .serve, .fetch, .mcp => |opts| WebBotAuth{ + inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{ .key_file = opts.common.web_bot_auth_key_file orelse return null, .keyid = opts.common.web_bot_auth_keyid orelse return null, .domain = opts.common.web_bot_auth_domain orelse return null, diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 41b5ea39..05d5ddde 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -30,6 +30,7 @@ const Notification = @import("../Notification.zig"); const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar; const Robots = @import("../network/Robots.zig"); const RobotStore = Robots.RobotStore; +const WebBotAuth = @import("../network/WebBotAuth.zig"); const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; @@ -702,6 +703,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts try conn.setHeaders(&header_list); + // If we have WebBotAuth, sign our request. + if (self.network.web_bot_auth) |*wba| { + const authority = URL.getHost(req.url); + try wba.signRequest(self.allocator, &header_list, authority); + } + // Add cookies. if (header_list.cookies) |cookies| { try conn.setCookies(cookies); diff --git a/src/network/Runtime.zig b/src/network/Runtime.zig index b3cd6a06..d6a08f59 100644 --- a/src/network/Runtime.zig +++ b/src/network/Runtime.zig @@ -28,6 +28,7 @@ const libcurl = @import("../sys/libcurl.zig"); const net_http = @import("http.zig"); const RobotStore = @import("Robots.zig").RobotStore; +const WebBotAuth = @import("WebBotAuth.zig"); const Runtime = @This(); @@ -42,6 +43,7 @@ allocator: Allocator, config: *const Config, ca_blob: ?net_http.Blob, robot_store: RobotStore, +web_bot_auth: ?WebBotAuth, connections: []net_http.Connection, available: std.DoublyLinkedList = .{}, @@ -205,6 +207,11 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime { available.append(&connections[i].node); } + const web_bot_auth = if (config.webBotAuth()) |wba_cfg| + try WebBotAuth.fromConfig(allocator, &wba_cfg) + else + null; + return .{ .allocator = allocator, .config = config, @@ -212,6 +219,7 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime { .robot_store = RobotStore.init(allocator), .connections = connections, .available = available, + .web_bot_auth = web_bot_auth, .pollfds = pollfds, .wakeup_pipe = pipe, }; @@ -238,6 +246,9 @@ pub fn deinit(self: *Runtime) void { self.allocator.free(self.connections); self.robot_store.deinit(); + if (self.web_bot_auth) |wba| { + wba.deinit(self.allocator); + } globalDeinit(); } diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig index 842f02d2..0abdad82 100644 --- a/src/network/WebBotAuth.zig +++ b/src/network/WebBotAuth.zig @@ -19,7 +19,7 @@ const std = @import("std"); const crypto = @import("../crypto.zig"); -const Http = @import("../http/Http.zig"); +const Http = @import("../network/http.zig"); const WebBotAuth = @This(); From c38d9a309815f164ca8db53120af7fed95831ed0 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Fri, 27 Feb 2026 09:00:27 -0800 Subject: [PATCH 04/13] auth challenge only on use_proxy --- src/browser/HttpClient.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 05d5ddde..2e832077 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -1338,7 +1338,7 @@ pub const Transfer = struct { } transfer._redirecting = false; - if (status == 401 or status == 407) { + if ((status == 401 or status == 407) and transfer.client.use_proxy) { // The auth challenge must be parsed from a following // WWW-Authenticate or Proxy-Authenticate header. transfer._auth_challenge = .{ From 9971816711d3eb8501d4ee38dc34f4f4f87cb9a1 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 4 Mar 2026 05:48:01 -0800 Subject: [PATCH 05/13] use transfer arena to sign webbotauth request --- src/browser/HttpClient.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 2e832077..181b3923 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -706,7 +706,7 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr // If we have WebBotAuth, sign our request. if (self.network.web_bot_auth) |*wba| { const authority = URL.getHost(req.url); - try wba.signRequest(self.allocator, &header_list, authority); + try wba.signRequest(transfer.arena.allocator(), &header_list, authority); } // Add cookies. From a1fb11ae339ba561233ae86628f67763c9147ed7 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 4 Mar 2026 05:52:32 -0800 Subject: [PATCH 06/13] make pem private key buffers smaller with comments --- src/network/WebBotAuth.zig | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig index 0abdad82..3513c206 100644 --- a/src/network/WebBotAuth.zig +++ b/src/network/WebBotAuth.zig @@ -41,8 +41,8 @@ fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY { const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace); - // strip newlines from b64 - var clean: [4096]u8 = undefined; + // Ed25519 PKCS#8 DER is always 48 bytes, which base64-encodes to exactly 64 chars + var clean: [64]u8 = undefined; var clean_len: usize = 0; for (b64) |ch| { if (ch != '\n' and ch != '\r') { @@ -51,11 +51,12 @@ fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY { } } - var der: [128]u8 = undefined; + // decode base64 into 48-byte DER buffer + var der: [48]u8 = undefined; const decoded_len = try std.base64.standard.Decoder.calcSizeForSlice(clean[0..clean_len]); try std.base64.standard.Decoder.decode(der[0..decoded_len], clean[0..clean_len]); - // Ed25519 PKCS#8: key bytes are at offset 16, 32 bytes long + // Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16. const key_bytes = der[16..48]; const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32); From 1ed61d4783283b6f9170f9f1a935a7d98747803d Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 4 Mar 2026 05:59:02 -0800 Subject: [PATCH 07/13] simplify parsePemPrivateKey --- src/network/WebBotAuth.zig | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig index 3513c206..bd3f4946 100644 --- a/src/network/WebBotAuth.zig +++ b/src/network/WebBotAuth.zig @@ -41,20 +41,9 @@ fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY { const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace); - // Ed25519 PKCS#8 DER is always 48 bytes, which base64-encodes to exactly 64 chars - var clean: [64]u8 = undefined; - var clean_len: usize = 0; - for (b64) |ch| { - if (ch != '\n' and ch != '\r') { - clean[clean_len] = ch; - clean_len += 1; - } - } - // decode base64 into 48-byte DER buffer var der: [48]u8 = undefined; - const decoded_len = try std.base64.standard.Decoder.calcSizeForSlice(clean[0..clean_len]); - try std.base64.standard.Decoder.decode(der[0..decoded_len], clean[0..clean_len]); + try std.base64.standard.Decoder.decode(der[0..48], b64); // Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16. const key_bytes = der[16..48]; From d365240f91ed825273c20cf29f91d05a9fcb174b Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 4 Mar 2026 06:01:51 -0800 Subject: [PATCH 08/13] fix cli argument for WebBotAuth domain --- src/Config.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.zig b/src/Config.zig index 21fdeed4..629df32b 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -357,7 +357,7 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void { \\--web_bot_auth_keyid \\ The JWK thumbprint of your public key. \\ - \\--web_bot_auth_directory + \\--web_bot_auth_domain \\ Your domain e.g. yourdomain.com ; From fca29a8be2b48da1f373c239f68c21786e5a7e20 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 5 Mar 2026 21:38:16 -0800 Subject: [PATCH 09/13] add WebBotAuth unit tests --- src/network/WebBotAuth.zig | 126 +++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig index bd3f4946..418ee3cb 100644 --- a/src/network/WebBotAuth.zig +++ b/src/network/WebBotAuth.zig @@ -156,3 +156,129 @@ pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void { crypto.EVP_PKEY_free(self.pkey); allocator.free(self.directory_url); } + +test "parsePemPrivateKey: valid Ed25519 PKCS#8 PEM" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); +} + +test "parsePemPrivateKey: missing BEGIN marker returns error" { + const bad_pem = "-----END PRIVATE KEY-----\n"; + try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem)); +} + +test "parsePemPrivateKey: missing END marker returns error" { + const bad_pem = "-----BEGIN PRIVATE KEY-----\nMC4CAQA=\n"; + try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem)); +} + +test "signEd25519: signature length is always 64 bytes" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); + + var sig: [64]u8 = @splat(0); + try signEd25519(pkey, "hello world", &sig); + + var all_zero = true; + for (sig) |b| if (b != 0) { + all_zero = false; + break; + }; + try std.testing.expect(!all_zero); +} + +test "signEd25519: same key + message produces same signature (deterministic)" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); + + var sig1: [64]u8 = undefined; + var sig2: [64]u8 = undefined; + try signEd25519(pkey, "deterministic test", &sig1); + try signEd25519(pkey, "deterministic test", &sig2); + + try std.testing.expectEqualSlices(u8, &sig1, &sig2); +} + +test "signEd25519: same key + diff message produces different signature (deterministic)" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); + + var sig1: [64]u8 = undefined; + var sig2: [64]u8 = undefined; + try signEd25519(pkey, "msg 1", &sig1); + try signEd25519(pkey, "msg 2", &sig2); + + try std.testing.expect(!std.mem.eql(u8, &sig1, &sig2)); +} + +test "signRequest: adds headers with correct names" { + const allocator = std.testing.allocator; + + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + + const directory_url = try allocator.dupeZ( + u8, + "https://example.com/.well-known/http-message-signatures-directory", + ); + + var auth = WebBotAuth{ + .pkey = pkey, + .keyid = "test-key-id", + .directory_url = directory_url, + }; + defer auth.deinit(allocator); + + var headers = try Http.Headers.init("User-Agent: Test-Agent"); + defer headers.deinit(); + + try auth.signRequest(allocator, &headers, "example.com"); + + var it = headers.iterator(); + var found_sig_agent = false; + var found_sig_input = false; + var found_signature = false; + var count: usize = 0; + + while (it.next()) |h| { + count += 1; + if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true; + if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true; + if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true; + } + + try std.testing.expect(count >= 3); + try std.testing.expect(found_sig_agent); + try std.testing.expect(found_sig_input); + try std.testing.expect(found_signature); +} From c6c0492c33e700a32fd47a67cff767972f08e8c0 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Tue, 10 Mar 2026 20:21:43 -0700 Subject: [PATCH 10/13] fix request authentication with web bot auth --- src/browser/HttpClient.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 181b3923..5b71b3b0 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -1338,7 +1338,7 @@ pub const Transfer = struct { } transfer._redirecting = false; - if ((status == 401 or status == 407) and transfer.client.use_proxy) { + if (status == 401 or status == 407) { // The auth challenge must be parsed from a following // WWW-Authenticate or Proxy-Authenticate header. transfer._auth_challenge = .{ From 172481dd7256e6013b1011a9a648f8de8e9351d5 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Tue, 10 Mar 2026 20:24:12 -0700 Subject: [PATCH 11/13] add e2e tests w/ web bot auth --- .github/workflows/e2e-test.yml | 92 ++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 74eda203..95fabb7f 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -117,6 +117,98 @@ jobs: BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js kill `cat LPD.pid` `cat PROXY.id` + # e2e tests w/ web-bot-auth configuration on. + wba-demo-scripts: + name: wba-demo-scripts + needs: zig-build-release + + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + with: + repository: 'lightpanda-io/demo' + fetch-depth: 0 + + - run: npm install + + - name: download artifact + uses: actions/download-artifact@v4 + with: + name: lightpanda-build-release + + - run: chmod a+x ./lightpanda + + - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + + - name: run end to end tests + run: | + ./lightpanda serve \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ + & echo $! > LPD.pid + go run runner/main.go + kill `cat LPD.pid` + + - name: build proxy + run: | + cd proxy + go build + + - name: run end to end tests through proxy + run: | + ./proxy/proxy & echo $! > PROXY.id + ./lightpanda serve \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ + --http_proxy 'http://127.0.0.1:3000' \ + & echo $! > LPD.pid + go run runner/main.go + kill `cat LPD.pid` `cat PROXY.id` + + - name: run request interception through proxy + run: | + export PROXY_USERNAME=username PROXY_PASSWORD=password + ./proxy/proxy & echo $! > PROXY.id + ./lightpanda serve & echo $! > LPD.pid + URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js + BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js + kill `cat LPD.pid` `cat PROXY.id` + + wba-test: + name: wba-test + needs: zig-build-release + + env: + LIGHTPANDA_DISABLE_TELEMETRY: true + + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: download artifact + uses: actions/download-artifact@v4 + with: + name: lightpanda-build-release + + - run: chmod a+x ./lightpanda + + - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + + - run: | + ./lightpanda fetch https://crawltest.com/cdn-cgi/web-bot-auth \ + --log_level error \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ + --dump markdown \ + | tee output.log + + - run: cat output.log | grep -q "unknown public key or unknown verified bot ID for keyid" + cdp-and-hyperfine-bench: name: cdp-and-hyperfine-bench needs: zig-build-release From bf0be60b890f87da1de7fcd9add7cc8a227a292b Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Tue, 10 Mar 2026 21:44:41 -0700 Subject: [PATCH 12/13] use new validator for e2e test --- .github/workflows/e2e-test.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 95fabb7f..fa13a613 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -186,9 +186,16 @@ jobs: LIGHTPANDA_DISABLE_TELEMETRY: true runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 5 steps: + - uses: actions/checkout@v4 + with: + repository: 'lightpanda-io/demo' + fetch-depth: 0 + + - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + - name: download artifact uses: actions/download-artifact@v4 with: @@ -196,18 +203,18 @@ jobs: - run: chmod a+x ./lightpanda - - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + - name: run wba test + run: | + node webbotauth/validator.js & + VALIDATOR_PID=$! + sleep 1 - - run: | - ./lightpanda fetch https://crawltest.com/cdn-cgi/web-bot-auth \ - --log_level error \ + ./lightpanda fetch http://127.0.0.1:8989/ \ --web_bot_auth_key_file private_key.pem \ --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ - --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ - --dump markdown \ - | tee output.log + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} - - run: cat output.log | grep -q "unknown public key or unknown verified bot ID for keyid" + wait $VALIDATOR_PID cdp-and-hyperfine-bench: name: cdp-and-hyperfine-bench From 8db64772b78a847cec95b7fd806e92a0b6ec84fa Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Thu, 12 Mar 2026 09:02:48 -0700 Subject: [PATCH 13/13] add URL getHost test --- src/browser/URL.zig | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/browser/URL.zig b/src/browser/URL.zig index 19b87333..996f69d2 100644 --- a/src/browser/URL.zig +++ b/src/browser/URL.zig @@ -1400,3 +1400,12 @@ test "URL: unescape" { try testing.expectEqual("hello%2", result); } } + +test "URL: getHost" { + try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://example.com:8080/path")); + try testing.expectEqualSlices(u8, "example.com", getHost("https://example.com/path")); + try testing.expectEqualSlices(u8, "example.com:443", getHost("https://example.com:443/")); + try testing.expectEqualSlices(u8, "example.com", getHost("https://user:pass@example.com/page")); + try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://user:pass@example.com:8080/page")); + try testing.expectEqualSlices(u8, "", getHost("not-a-url")); +}