diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 74eda203..fa13a613 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -117,6 +117,105 @@ jobs: BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js kill `cat LPD.pid` `cat PROXY.id` + # e2e tests w/ web-bot-auth configuration on. + wba-demo-scripts: + name: wba-demo-scripts + needs: zig-build-release + + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + with: + repository: 'lightpanda-io/demo' + fetch-depth: 0 + + - run: npm install + + - name: download artifact + uses: actions/download-artifact@v4 + with: + name: lightpanda-build-release + + - run: chmod a+x ./lightpanda + + - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + + - name: run end to end tests + run: | + ./lightpanda serve \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ + & echo $! > LPD.pid + go run runner/main.go + kill `cat LPD.pid` + + - name: build proxy + run: | + cd proxy + go build + + - name: run end to end tests through proxy + run: | + ./proxy/proxy & echo $! > PROXY.id + ./lightpanda serve \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ + --http_proxy 'http://127.0.0.1:3000' \ + & echo $! > LPD.pid + go run runner/main.go + kill `cat LPD.pid` `cat PROXY.id` + + - name: run request interception through proxy + run: | + export PROXY_USERNAME=username PROXY_PASSWORD=password + ./proxy/proxy & echo $! > PROXY.id + ./lightpanda serve & echo $! > LPD.pid + URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js + BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js + kill `cat LPD.pid` `cat PROXY.id` + + wba-test: + name: wba-test + needs: zig-build-release + + env: + LIGHTPANDA_DISABLE_TELEMETRY: true + + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + with: + repository: 'lightpanda-io/demo' + fetch-depth: 0 + + - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + + - name: download artifact + uses: actions/download-artifact@v4 + with: + name: lightpanda-build-release + + - run: chmod a+x ./lightpanda + + - name: run wba test + run: | + node webbotauth/validator.js & + VALIDATOR_PID=$! + sleep 1 + + ./lightpanda fetch http://127.0.0.1:8989/ \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} + + wait $VALIDATOR_PID + cdp-and-hyperfine-bench: name: cdp-and-hyperfine-bench needs: zig-build-release diff --git a/src/Config.zig b/src/Config.zig index 73c7f3a7..629df32b 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator; const log = @import("log.zig"); const dump = @import("browser/dump.zig"); +const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config; + pub const RunMode = enum { help, fetch, @@ -161,6 +163,17 @@ pub fn cdpTimeout(self: *const Config) usize { }; } +pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig { + return switch (self.mode) { + inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{ + .key_file = opts.common.web_bot_auth_key_file orelse return null, + .keyid = opts.common.web_bot_auth_keyid orelse return null, + .domain = opts.common.web_bot_auth_domain orelse return null, + }, + .help, .version => null, + }; +} + pub fn maxConnections(self: *const Config) u16 { return switch (self.mode) { .serve => |opts| opts.cdp_max_connections, @@ -227,6 +240,10 @@ pub const Common = struct { log_format: ?log.Format = null, log_filter_scopes: ?[]log.Scope = null, user_agent_suffix: ?[]const u8 = null, + + web_bot_auth_key_file: ?[]const u8 = null, + web_bot_auth_keyid: ?[]const u8 = null, + web_bot_auth_domain: ?[]const u8 = null, }; /// Pre-formatted HTTP headers for reuse across Http and Client. @@ -334,6 +351,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void { \\--user_agent_suffix \\ Suffix to append to the Lightpanda/X.Y User-Agent \\ + \\--web_bot_auth_key_file + \\ Path to the Ed25519 private key PEM file. + \\ + \\--web_bot_auth_keyid + \\ The JWK thumbprint of your public key. + \\ + \\--web_bot_auth_domain + \\ Your domain e.g. yourdomain.com ; // MAX_HELP_LEN| @@ -855,5 +880,32 @@ fn parseCommonArg( return true; } + if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" }); + return error.InvalidArgument; + }; + common.web_bot_auth_key_file = try allocator.dupe(u8, str); + return true; + } + + if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" }); + return error.InvalidArgument; + }; + common.web_bot_auth_keyid = try allocator.dupe(u8, str); + return true; + } + + if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" }); + return error.InvalidArgument; + }; + common.web_bot_auth_domain = try allocator.dupe(u8, str); + return true; + } + return false; } diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index f716904c..a7e4cc8a 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -30,6 +30,7 @@ const Notification = @import("../Notification.zig"); const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar; const Robots = @import("../network/Robots.zig"); const RobotStore = Robots.RobotStore; +const WebBotAuth = @import("../network/WebBotAuth.zig"); const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; @@ -702,6 +703,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts try conn.setHeaders(&header_list); + // If we have WebBotAuth, sign our request. + if (self.network.web_bot_auth) |*wba| { + const authority = URL.getHost(req.url); + try wba.signRequest(transfer.arena.allocator(), &header_list, authority); + } + // Add cookies. if (header_list.cookies) |cookies| { try conn.setCookies(cookies); diff --git a/src/browser/URL.zig b/src/browser/URL.zig index f8d7dd90..1c26a981 100644 --- a/src/browser/URL.zig +++ b/src/browser/URL.zig @@ -1405,3 +1405,12 @@ test "URL: unescape" { try testing.expectEqual("hello%2", result); } } + +test "URL: getHost" { + try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://example.com:8080/path")); + try testing.expectEqualSlices(u8, "example.com", getHost("https://example.com/path")); + try testing.expectEqualSlices(u8, "example.com:443", getHost("https://example.com:443/")); + try testing.expectEqualSlices(u8, "example.com", getHost("https://user:pass@example.com/page")); + try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://user:pass@example.com:8080/page")); + try testing.expectEqualSlices(u8, "", getHost("not-a-url")); +} diff --git a/src/crypto.zig b/src/crypto.zig index 012ca96e..58f6349f 100644 --- a/src/crypto.zig +++ b/src/crypto.zig @@ -228,6 +228,8 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8 pub const NID_X25519 = @as(c_int, 948); pub const EVP_PKEY_X25519 = NID_X25519; +pub const NID_ED25519 = 949; +pub const EVP_PKEY_ED25519 = NID_ED25519; pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY; pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY; @@ -236,3 +238,11 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void; pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int; pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int; pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int; +pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void; + +pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int; +pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int; +pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX; +pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void; +pub const struct_evp_md_ctx_st = opaque {}; +pub const EVP_MD_CTX = struct_evp_md_ctx_st; diff --git a/src/network/Runtime.zig b/src/network/Runtime.zig index b3cd6a06..d6a08f59 100644 --- a/src/network/Runtime.zig +++ b/src/network/Runtime.zig @@ -28,6 +28,7 @@ const libcurl = @import("../sys/libcurl.zig"); const net_http = @import("http.zig"); const RobotStore = @import("Robots.zig").RobotStore; +const WebBotAuth = @import("WebBotAuth.zig"); const Runtime = @This(); @@ -42,6 +43,7 @@ allocator: Allocator, config: *const Config, ca_blob: ?net_http.Blob, robot_store: RobotStore, +web_bot_auth: ?WebBotAuth, connections: []net_http.Connection, available: std.DoublyLinkedList = .{}, @@ -205,6 +207,11 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime { available.append(&connections[i].node); } + const web_bot_auth = if (config.webBotAuth()) |wba_cfg| + try WebBotAuth.fromConfig(allocator, &wba_cfg) + else + null; + return .{ .allocator = allocator, .config = config, @@ -212,6 +219,7 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime { .robot_store = RobotStore.init(allocator), .connections = connections, .available = available, + .web_bot_auth = web_bot_auth, .pollfds = pollfds, .wakeup_pipe = pipe, }; @@ -238,6 +246,9 @@ pub fn deinit(self: *Runtime) void { self.allocator.free(self.connections); self.robot_store.deinit(); + if (self.web_bot_auth) |wba| { + wba.deinit(self.allocator); + } globalDeinit(); } diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig new file mode 100644 index 00000000..418ee3cb --- /dev/null +++ b/src/network/WebBotAuth.zig @@ -0,0 +1,284 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const crypto = @import("../crypto.zig"); + +const Http = @import("../network/http.zig"); + +const WebBotAuth = @This(); + +pkey: *crypto.EVP_PKEY, +keyid: []const u8, +directory_url: [:0]const u8, + +pub const Config = struct { + key_file: []const u8, + keyid: []const u8, + domain: []const u8, +}; + +fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY { + const begin = "-----BEGIN PRIVATE KEY-----"; + const end = "-----END PRIVATE KEY-----"; + const start_idx = std.mem.indexOf(u8, pem, begin) orelse return error.InvalidPem; + const end_idx = std.mem.indexOf(u8, pem, end) orelse return error.InvalidPem; + + const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace); + + // decode base64 into 48-byte DER buffer + var der: [48]u8 = undefined; + try std.base64.standard.Decoder.decode(der[0..48], b64); + + // Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16. + const key_bytes = der[16..48]; + + const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32); + return pkey orelse error.InvalidKey; +} + +fn signEd25519(pkey: *crypto.EVP_PKEY, message: []const u8, out: *[64]u8) !void { + const ctx = crypto.EVP_MD_CTX_new() orelse return error.OutOfMemory; + defer crypto.EVP_MD_CTX_free(ctx); + + if (crypto.EVP_DigestSignInit(ctx, null, null, null, pkey) != 1) + return error.SignInit; + + var sig_len: usize = 64; + if (crypto.EVP_DigestSign(ctx, out.ptr, &sig_len, message.ptr, message.len) != 1) + return error.SignFailed; +} + +pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAuth { + const pem = try std.fs.cwd().readFileAlloc(allocator, config.key_file, 1024 * 4); + defer allocator.free(pem); + + const pkey = try parsePemPrivateKey(pem); + errdefer crypto.EVP_PKEY_free(pkey); + + const directory_url = try std.fmt.allocPrintSentinel( + allocator, + "https://{s}/.well-known/http-message-signatures-directory", + .{config.domain}, + 0, + ); + errdefer allocator.free(directory_url); + + return .{ + .pkey = pkey, + // Owned by the Config so it's okay. + .keyid = config.keyid, + .directory_url = directory_url, + }; +} + +pub fn signRequest( + self: *const WebBotAuth, + allocator: std.mem.Allocator, + headers: *Http.Headers, + authority: []const u8, +) !void { + const now = std.time.timestamp(); + const expires = now + 60; + + // build the signature-input value (without the sig1= label) + const sig_input_value = try std.fmt.allocPrint( + allocator, + "(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"", + .{ now, expires, self.keyid }, + ); + defer allocator.free(sig_input_value); + + // build the canonical string to sign + const canonical = try std.fmt.allocPrint( + allocator, + "\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}", + .{ authority, self.directory_url, sig_input_value }, + ); + defer allocator.free(canonical); + + // sign it + var sig: [64]u8 = undefined; + try signEd25519(self.pkey, canonical, &sig); + + // base64 encode + const encoded_len = std.base64.standard.Encoder.calcSize(sig.len); + const encoded = try allocator.alloc(u8, encoded_len); + defer allocator.free(encoded); + _ = std.base64.standard.Encoder.encode(encoded, &sig); + + // build the 3 headers and add them + const sig_agent = try std.fmt.allocPrintSentinel( + allocator, + "Signature-Agent: \"{s}\"", + .{self.directory_url}, + 0, + ); + defer allocator.free(sig_agent); + + const sig_input = try std.fmt.allocPrintSentinel( + allocator, + "Signature-Input: sig1={s}", + .{sig_input_value}, + 0, + ); + defer allocator.free(sig_input); + + const signature = try std.fmt.allocPrintSentinel( + allocator, + "Signature: sig1=:{s}:", + .{encoded}, + 0, + ); + defer allocator.free(signature); + + try headers.add(sig_agent); + try headers.add(sig_input); + try headers.add(signature); +} + +pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void { + crypto.EVP_PKEY_free(self.pkey); + allocator.free(self.directory_url); +} + +test "parsePemPrivateKey: valid Ed25519 PKCS#8 PEM" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); +} + +test "parsePemPrivateKey: missing BEGIN marker returns error" { + const bad_pem = "-----END PRIVATE KEY-----\n"; + try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem)); +} + +test "parsePemPrivateKey: missing END marker returns error" { + const bad_pem = "-----BEGIN PRIVATE KEY-----\nMC4CAQA=\n"; + try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem)); +} + +test "signEd25519: signature length is always 64 bytes" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); + + var sig: [64]u8 = @splat(0); + try signEd25519(pkey, "hello world", &sig); + + var all_zero = true; + for (sig) |b| if (b != 0) { + all_zero = false; + break; + }; + try std.testing.expect(!all_zero); +} + +test "signEd25519: same key + message produces same signature (deterministic)" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); + + var sig1: [64]u8 = undefined; + var sig2: [64]u8 = undefined; + try signEd25519(pkey, "deterministic test", &sig1); + try signEd25519(pkey, "deterministic test", &sig2); + + try std.testing.expectEqualSlices(u8, &sig1, &sig2); +} + +test "signEd25519: same key + diff message produces different signature (deterministic)" { + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + defer crypto.EVP_PKEY_free(pkey); + + var sig1: [64]u8 = undefined; + var sig2: [64]u8 = undefined; + try signEd25519(pkey, "msg 1", &sig1); + try signEd25519(pkey, "msg 2", &sig2); + + try std.testing.expect(!std.mem.eql(u8, &sig1, &sig2)); +} + +test "signRequest: adds headers with correct names" { + const allocator = std.testing.allocator; + + const pem = + \\-----BEGIN PRIVATE KEY----- + \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT + \\-----END PRIVATE KEY----- + \\ + ; + const pkey = try parsePemPrivateKey(pem); + + const directory_url = try allocator.dupeZ( + u8, + "https://example.com/.well-known/http-message-signatures-directory", + ); + + var auth = WebBotAuth{ + .pkey = pkey, + .keyid = "test-key-id", + .directory_url = directory_url, + }; + defer auth.deinit(allocator); + + var headers = try Http.Headers.init("User-Agent: Test-Agent"); + defer headers.deinit(); + + try auth.signRequest(allocator, &headers, "example.com"); + + var it = headers.iterator(); + var found_sig_agent = false; + var found_sig_input = false; + var found_signature = false; + var count: usize = 0; + + while (it.next()) |h| { + count += 1; + if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true; + if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true; + if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true; + } + + try std.testing.expect(count >= 3); + try std.testing.expect(found_sig_agent); + try std.testing.expect(found_sig_input); + try std.testing.expect(found_signature); +}