Merge pull request #1609 from lightpanda-io/web-bot-auth

Web Bot Auth

.github/workflows/e2e-test.yml

@@ -117,6 +117,105 @@ jobs:
           BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 
+  # e2e tests w/ web-bot-auth configuration on.
+  wba-demo-scripts:
+    name: wba-demo-scripts
+    needs: zig-build-release
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: 'lightpanda-io/demo'
+          fetch-depth: 0
+
+      - run: npm install
+
+      - name: download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lightpanda-build-release
+
+      - run: chmod a+x ./lightpanda
+
+      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+      - name: run end to end tests
+        run: |
+          ./lightpanda serve \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            & echo $! > LPD.pid
+          go run runner/main.go
+          kill `cat LPD.pid`
+
+      - name: build proxy
+        run: |
+          cd proxy
+          go build
+
+      - name: run end to end tests through proxy
+        run: |
+          ./proxy/proxy & echo $! > PROXY.id
+          ./lightpanda serve \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+            --http_proxy 'http://127.0.0.1:3000' \
+            & echo $! > LPD.pid
+          go run runner/main.go
+          kill `cat LPD.pid` `cat PROXY.id`
+
+      - name: run request interception through proxy
+        run: |
+          export PROXY_USERNAME=username PROXY_PASSWORD=password
+          ./proxy/proxy & echo $! > PROXY.id
+          ./lightpanda serve & echo $! > LPD.pid
+          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
+          BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
+          kill `cat LPD.pid` `cat PROXY.id`
+
+  wba-test:
+    name: wba-test
+    needs: zig-build-release
+
+    env:
+      LIGHTPANDA_DISABLE_TELEMETRY: true
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: 'lightpanda-io/demo'
+          fetch-depth: 0
+
+      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+      - name: download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lightpanda-build-release
+
+      - run: chmod a+x ./lightpanda
+
+      - name: run wba test
+        run: |
+          node webbotauth/validator.js &
+          VALIDATOR_PID=$!
+          sleep 1
+
+          ./lightpanda fetch http://127.0.0.1:8989/ \
+            --web_bot_auth_key_file private_key.pem \
+            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }}
+
+          wait $VALIDATOR_PID
+
   cdp-and-hyperfine-bench:
     name: cdp-and-hyperfine-bench
     needs: zig-build-release

src/Config.zig

@@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator;
 const log = @import("log.zig");
 const dump = @import("browser/dump.zig");
 
+const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;
+
 pub const RunMode = enum {
     help,
     fetch,
@@ -161,6 +163,17 @@ pub fn cdpTimeout(self: *const Config) usize {
     };
 }
 
+pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
+    return switch (self.mode) {
+        inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
+            .key_file = opts.common.web_bot_auth_key_file orelse return null,
+            .keyid = opts.common.web_bot_auth_keyid orelse return null,
+            .domain = opts.common.web_bot_auth_domain orelse return null,
+        },
+        .help, .version => null,
+    };
+}
+
 pub fn maxConnections(self: *const Config) u16 {
     return switch (self.mode) {
         .serve => |opts| opts.cdp_max_connections,
@@ -227,6 +240,10 @@ pub const Common = struct {
     log_format: ?log.Format = null,
     log_filter_scopes: ?[]log.Scope = null,
     user_agent_suffix: ?[]const u8 = null,
+
+    web_bot_auth_key_file: ?[]const u8 = null,
+    web_bot_auth_keyid: ?[]const u8 = null,
+    web_bot_auth_domain: ?[]const u8 = null,
 };
 
 /// Pre-formatted HTTP headers for reuse across Http and Client.
@@ -334,6 +351,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
         \\--user_agent_suffix
         \\                Suffix to append to the Lightpanda/X.Y User-Agent
         \\
+        \\--web_bot_auth_key_file
+        \\                Path to the Ed25519 private key PEM file.
+        \\
+        \\--web_bot_auth_keyid
+        \\                The JWK thumbprint of your public key.
+        \\
+        \\--web_bot_auth_domain
+        \\                Your domain e.g. yourdomain.com
     ;
 
     //                                                                     MAX_HELP_LEN|
@@ -855,5 +880,32 @@ fn parseCommonArg(
         return true;
     }
 
+    if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_key_file = try allocator.dupe(u8, str);
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_keyid = try allocator.dupe(u8, str);
+        return true;
+    }
+
+    if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) {
+        const str = args.next() orelse {
+            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" });
+            return error.InvalidArgument;
+        };
+        common.web_bot_auth_domain = try allocator.dupe(u8, str);
+        return true;
+    }
+
     return false;
 }

src/browser/HttpClient.zig

@@ -30,6 +30,7 @@ const Notification = @import("../Notification.zig");
 const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
 const Robots = @import("../network/Robots.zig");
 const RobotStore = Robots.RobotStore;
+const WebBotAuth = @import("../network/WebBotAuth.zig");
 
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
@@ -702,6 +703,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
         try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts
         try conn.setHeaders(&header_list);
 
+        // If we have WebBotAuth, sign our request.
+        if (self.network.web_bot_auth) |*wba| {
+            const authority = URL.getHost(req.url);
+            try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
+        }
+
         // Add cookies.
         if (header_list.cookies) |cookies| {
             try conn.setCookies(cookies);

src/browser/URL.zig

@@ -1405,3 +1405,12 @@ test "URL: unescape" {
         try testing.expectEqual("hello%2", result);
     }
 }
+
+test "URL: getHost" {
+    try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://example.com:8080/path"));
+    try testing.expectEqualSlices(u8, "example.com", getHost("https://example.com/path"));
+    try testing.expectEqualSlices(u8, "example.com:443", getHost("https://example.com:443/"));
+    try testing.expectEqualSlices(u8, "example.com", getHost("https://user:pass@example.com/page"));
+    try testing.expectEqualSlices(u8, "example.com:8080", getHost("https://user:pass@example.com:8080/page"));
+    try testing.expectEqualSlices(u8, "", getHost("not-a-url"));
+}

src/crypto.zig

@@ -228,6 +228,8 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8
 
 pub const NID_X25519 = @as(c_int, 948);
 pub const EVP_PKEY_X25519 = NID_X25519;
+pub const NID_ED25519 = 949;
+pub const EVP_PKEY_ED25519 = NID_ED25519;
 
 pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
 pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
@@ -236,3 +238,11 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void;
 pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int;
 pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int;
 pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int;
+pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void;
+
+pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int;
+pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int;
+pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX;
+pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void;
+pub const struct_evp_md_ctx_st = opaque {};
+pub const EVP_MD_CTX = struct_evp_md_ctx_st;

src/network/Runtime.zig

@@ -28,6 +28,7 @@ const libcurl = @import("../sys/libcurl.zig");
 
 const net_http = @import("http.zig");
 const RobotStore = @import("Robots.zig").RobotStore;
+const WebBotAuth = @import("WebBotAuth.zig");
 
 const Runtime = @This();
 
@@ -42,6 +43,7 @@ allocator: Allocator,
 config: *const Config,
 ca_blob: ?net_http.Blob,
 robot_store: RobotStore,
+web_bot_auth: ?WebBotAuth,
 
 connections: []net_http.Connection,
 available: std.DoublyLinkedList = .{},
@@ -205,6 +207,11 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime {
         available.append(&connections[i].node);
     }
 
+    const web_bot_auth = if (config.webBotAuth()) |wba_cfg|
+        try WebBotAuth.fromConfig(allocator, &wba_cfg)
+    else
+        null;
+
     return .{
         .allocator = allocator,
         .config = config,
@@ -212,6 +219,7 @@ pub fn init(allocator: Allocator, config: *const Config) !Runtime {
         .robot_store = RobotStore.init(allocator),
         .connections = connections,
         .available = available,
+        .web_bot_auth = web_bot_auth,
         .pollfds = pollfds,
         .wakeup_pipe = pipe,
     };
@@ -238,6 +246,9 @@ pub fn deinit(self: *Runtime) void {
     self.allocator.free(self.connections);
 
     self.robot_store.deinit();
+    if (self.web_bot_auth) |wba| {
+        wba.deinit(self.allocator);
+    }
 
     globalDeinit();
 }

src/network/WebBotAuth.zig

@@ -0,0 +1,284 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const crypto = @import("../crypto.zig");
+
+const Http = @import("../network/http.zig");
+
+const WebBotAuth = @This();
+
+pkey: *crypto.EVP_PKEY,
+keyid: []const u8,
+directory_url: [:0]const u8,
+
+pub const Config = struct {
+    key_file: []const u8,
+    keyid: []const u8,
+    domain: []const u8,
+};
+
+fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY {
+    const begin = "-----BEGIN PRIVATE KEY-----";
+    const end = "-----END PRIVATE KEY-----";
+    const start_idx = std.mem.indexOf(u8, pem, begin) orelse return error.InvalidPem;
+    const end_idx = std.mem.indexOf(u8, pem, end) orelse return error.InvalidPem;
+
+    const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace);
+
+    // decode base64 into 48-byte DER buffer
+    var der: [48]u8 = undefined;
+    try std.base64.standard.Decoder.decode(der[0..48], b64);
+
+    // Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16.
+    const key_bytes = der[16..48];
+
+    const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32);
+    return pkey orelse error.InvalidKey;
+}
+
+fn signEd25519(pkey: *crypto.EVP_PKEY, message: []const u8, out: *[64]u8) !void {
+    const ctx = crypto.EVP_MD_CTX_new() orelse return error.OutOfMemory;
+    defer crypto.EVP_MD_CTX_free(ctx);
+
+    if (crypto.EVP_DigestSignInit(ctx, null, null, null, pkey) != 1)
+        return error.SignInit;
+
+    var sig_len: usize = 64;
+    if (crypto.EVP_DigestSign(ctx, out.ptr, &sig_len, message.ptr, message.len) != 1)
+        return error.SignFailed;
+}
+
+pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAuth {
+    const pem = try std.fs.cwd().readFileAlloc(allocator, config.key_file, 1024 * 4);
+    defer allocator.free(pem);
+
+    const pkey = try parsePemPrivateKey(pem);
+    errdefer crypto.EVP_PKEY_free(pkey);
+
+    const directory_url = try std.fmt.allocPrintSentinel(
+        allocator,
+        "https://{s}/.well-known/http-message-signatures-directory",
+        .{config.domain},
+        0,
+    );
+    errdefer allocator.free(directory_url);
+
+    return .{
+        .pkey = pkey,
+        // Owned by the Config so it's okay.
+        .keyid = config.keyid,
+        .directory_url = directory_url,
+    };
+}
+
+pub fn signRequest(
+    self: *const WebBotAuth,
+    allocator: std.mem.Allocator,
+    headers: *Http.Headers,
+    authority: []const u8,
+) !void {
+    const now = std.time.timestamp();
+    const expires = now + 60;
+
+    // build the signature-input value (without the sig1= label)
+    const sig_input_value = try std.fmt.allocPrint(
+        allocator,
+        "(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"",
+        .{ now, expires, self.keyid },
+    );
+    defer allocator.free(sig_input_value);
+
+    // build the canonical string to sign
+    const canonical = try std.fmt.allocPrint(
+        allocator,
+        "\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}",
+        .{ authority, self.directory_url, sig_input_value },
+    );
+    defer allocator.free(canonical);
+
+    // sign it
+    var sig: [64]u8 = undefined;
+    try signEd25519(self.pkey, canonical, &sig);
+
+    // base64 encode
+    const encoded_len = std.base64.standard.Encoder.calcSize(sig.len);
+    const encoded = try allocator.alloc(u8, encoded_len);
+    defer allocator.free(encoded);
+    _ = std.base64.standard.Encoder.encode(encoded, &sig);
+
+    // build the 3 headers and add them
+    const sig_agent = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature-Agent: \"{s}\"",
+        .{self.directory_url},
+        0,
+    );
+    defer allocator.free(sig_agent);
+
+    const sig_input = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature-Input: sig1={s}",
+        .{sig_input_value},
+        0,
+    );
+    defer allocator.free(sig_input);
+
+    const signature = try std.fmt.allocPrintSentinel(
+        allocator,
+        "Signature: sig1=:{s}:",
+        .{encoded},
+        0,
+    );
+    defer allocator.free(signature);
+
+    try headers.add(sig_agent);
+    try headers.add(sig_input);
+    try headers.add(signature);
+}
+
+pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {
+    crypto.EVP_PKEY_free(self.pkey);
+    allocator.free(self.directory_url);
+}
+
+test "parsePemPrivateKey: valid Ed25519 PKCS#8 PEM" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+}
+
+test "parsePemPrivateKey: missing BEGIN marker returns error" {
+    const bad_pem = "-----END PRIVATE KEY-----\n";
+    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
+}
+
+test "parsePemPrivateKey: missing END marker returns error" {
+    const bad_pem = "-----BEGIN PRIVATE KEY-----\nMC4CAQA=\n";
+    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
+}
+
+test "signEd25519: signature length is always 64 bytes" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig: [64]u8 = @splat(0);
+    try signEd25519(pkey, "hello world", &sig);
+
+    var all_zero = true;
+    for (sig) |b| if (b != 0) {
+        all_zero = false;
+        break;
+    };
+    try std.testing.expect(!all_zero);
+}
+
+test "signEd25519: same key + message produces same signature (deterministic)" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig1: [64]u8 = undefined;
+    var sig2: [64]u8 = undefined;
+    try signEd25519(pkey, "deterministic test", &sig1);
+    try signEd25519(pkey, "deterministic test", &sig2);
+
+    try std.testing.expectEqualSlices(u8, &sig1, &sig2);
+}
+
+test "signEd25519: same key + diff message produces different signature (deterministic)" {
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+    defer crypto.EVP_PKEY_free(pkey);
+
+    var sig1: [64]u8 = undefined;
+    var sig2: [64]u8 = undefined;
+    try signEd25519(pkey, "msg 1", &sig1);
+    try signEd25519(pkey, "msg 2", &sig2);
+
+    try std.testing.expect(!std.mem.eql(u8, &sig1, &sig2));
+}
+
+test "signRequest: adds headers with correct names" {
+    const allocator = std.testing.allocator;
+
+    const pem =
+        \\-----BEGIN PRIVATE KEY-----
+        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
+        \\-----END PRIVATE KEY-----
+        \\
+    ;
+    const pkey = try parsePemPrivateKey(pem);
+
+    const directory_url = try allocator.dupeZ(
+        u8,
+        "https://example.com/.well-known/http-message-signatures-directory",
+    );
+
+    var auth = WebBotAuth{
+        .pkey = pkey,
+        .keyid = "test-key-id",
+        .directory_url = directory_url,
+    };
+    defer auth.deinit(allocator);
+
+    var headers = try Http.Headers.init("User-Agent: Test-Agent");
+    defer headers.deinit();
+
+    try auth.signRequest(allocator, &headers, "example.com");
+
+    var it = headers.iterator();
+    var found_sig_agent = false;
+    var found_sig_input = false;
+    var found_signature = false;
+    var count: usize = 0;
+
+    while (it.next()) |h| {
+        count += 1;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true;
+        if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true;
+    }
+
+    try std.testing.expect(count >= 3);
+    try std.testing.expect(found_sig_agent);
+    try std.testing.expect(found_sig_input);
+    try std.testing.expect(found_signature);
+}