diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index 74eda203..675dd36b 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -117,6 +117,105 @@ jobs:
BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
kill `cat LPD.pid` `cat PROXY.id`
+ # e2e tests w/ web-bot-auth configuration on.
+ wba-demo-scripts:
+ name: wba-demo-scripts
+ needs: zig-build-release
+
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ repository: 'lightpanda-io/demo'
+ fetch-depth: 0
+
+ - run: npm install
+
+ - name: download artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: lightpanda-build-release
+
+ - run: chmod a+x ./lightpanda
+
+ - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+ - name: run end to end tests
+ run: |
+ ./lightpanda serve \
+ --web_bot_auth_key_file private_key.pem \
+ --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+ --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+ & echo $! > LPD.pid
+ go run runner/main.go
+ kill `cat LPD.pid`
+
+ - name: build proxy
+ run: |
+ cd proxy
+ go build
+
+ - name: run end to end tests through proxy
+ run: |
+ ./proxy/proxy & echo $! > PROXY.id
+ ./lightpanda serve \
+ --web_bot_auth_key_file private_key.pem \
+ --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+ --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
+ --http_proxy 'http://127.0.0.1:3000' \
+ & echo $! > LPD.pid
+ go run runner/main.go
+ kill `cat LPD.pid` `cat PROXY.id`
+
+ - name: run request interception through proxy
+ run: |
+ export PROXY_USERNAME=username PROXY_PASSWORD=password
+ ./proxy/proxy & echo $! > PROXY.id
+ ./lightpanda serve & echo $! > LPD.pid
+ URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
+ BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
+ kill `cat LPD.pid` `cat PROXY.id`
+
+ wba-test:
+ name: wba-test
+ needs: zig-build-release
+
+ env:
+ LIGHTPANDA_DISABLE_TELEMETRY: true
+
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ repository: 'lightpanda-io/demo'
+ fetch-depth: 0
+
+ - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
+
+ - name: download artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: lightpanda-build-release
+
+ - run: chmod a+x ./lightpanda
+
+ - name: run wba test
+ run: |
+ node webbotauth/validator.js &
+ VALIDATOR_PID=$!
+ sleep 2
+
+ ./lightpanda fetch http://127.0.0.1:8989/ \
+ --web_bot_auth_key_file private_key.pem \
+ --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
+ --web_bot_auth_domain ${{ vars.WBA_DOMAIN }}
+
+ wait $VALIDATOR_PID
+
cdp-and-hyperfine-bench:
name: cdp-and-hyperfine-bench
needs: zig-build-release
diff --git a/src/Config.zig b/src/Config.zig
index 73c7f3a7..629df32b 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator;
const log = @import("log.zig");
const dump = @import("browser/dump.zig");
+const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;
+
pub const RunMode = enum {
help,
fetch,
@@ -161,6 +163,17 @@ pub fn cdpTimeout(self: *const Config) usize {
};
}
+pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
+ return switch (self.mode) {
+ inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
+ .key_file = opts.common.web_bot_auth_key_file orelse return null,
+ .keyid = opts.common.web_bot_auth_keyid orelse return null,
+ .domain = opts.common.web_bot_auth_domain orelse return null,
+ },
+ .help, .version => null,
+ };
+}
+
pub fn maxConnections(self: *const Config) u16 {
return switch (self.mode) {
.serve => |opts| opts.cdp_max_connections,
@@ -227,6 +240,10 @@ pub const Common = struct {
log_format: ?log.Format = null,
log_filter_scopes: ?[]log.Scope = null,
user_agent_suffix: ?[]const u8 = null,
+
+ web_bot_auth_key_file: ?[]const u8 = null,
+ web_bot_auth_keyid: ?[]const u8 = null,
+ web_bot_auth_domain: ?[]const u8 = null,
};
/// Pre-formatted HTTP headers for reuse across Http and Client.
@@ -334,6 +351,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
\\--user_agent_suffix
\\ Suffix to append to the Lightpanda/X.Y User-Agent
\\
+ \\--web_bot_auth_key_file
+ \\ Path to the Ed25519 private key PEM file.
+ \\
+ \\--web_bot_auth_keyid
+ \\ The JWK thumbprint of your public key.
+ \\
+ \\--web_bot_auth_domain
+ \\ Your domain e.g. yourdomain.com
;
// MAX_HELP_LEN|
@@ -855,5 +880,32 @@ fn parseCommonArg(
return true;
}
+ if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) {
+ const str = args.next() orelse {
+ log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" });
+ return error.InvalidArgument;
+ };
+ common.web_bot_auth_key_file = try allocator.dupe(u8, str);
+ return true;
+ }
+
+ if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) {
+ const str = args.next() orelse {
+ log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" });
+ return error.InvalidArgument;
+ };
+ common.web_bot_auth_keyid = try allocator.dupe(u8, str);
+ return true;
+ }
+
+ if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) {
+ const str = args.next() orelse {
+ log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" });
+ return error.InvalidArgument;
+ };
+ common.web_bot_auth_domain = try allocator.dupe(u8, str);
+ return true;
+ }
+
return false;
}
diff --git a/src/SemanticTree.zig b/src/SemanticTree.zig
index 8e64b034..b64f222d 100644
--- a/src/SemanticTree.zig
+++ b/src/SemanticTree.zig
@@ -233,7 +233,7 @@ fn extractSelectOptions(node: *Node, page: *Page, arena: std.mem.Allocator) ![]O
if (child.is(Element)) |el| {
if (el.getTag() == .option) {
if (el.is(Element.Html.Option)) |opt| {
- const text = opt.getText();
+ const text = opt.getText(page);
const value = opt.getValue(page);
const selected = opt.getSelected();
try options.append(arena, .{ .text = text, .value = value, .selected = selected });
@@ -242,7 +242,7 @@ fn extractSelectOptions(node: *Node, page: *Page, arena: std.mem.Allocator) ![]O
var group_it = child.childrenIterator();
while (group_it.next()) |group_child| {
if (group_child.is(Element.Html.Option)) |opt| {
- const text = opt.getText();
+ const text = opt.getText(page);
const value = opt.getValue(page);
const selected = opt.getSelected();
try options.append(arena, .{ .text = text, .value = value, .selected = selected });
diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig
index f716904c..1e74c046 100644
--- a/src/browser/HttpClient.zig
+++ b/src/browser/HttpClient.zig
@@ -30,6 +30,7 @@ const Notification = @import("../Notification.zig");
const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
const Robots = @import("../network/Robots.zig");
const RobotStore = Robots.RobotStore;
+const WebBotAuth = @import("../network/WebBotAuth.zig");
const Allocator = std.mem.Allocator;
const ArenaAllocator = std.heap.ArenaAllocator;
@@ -702,6 +703,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts
try conn.setHeaders(&header_list);
+ // If we have WebBotAuth, sign our request.
+ if (self.network.web_bot_auth) |*wba| {
+ const authority = URL.getHost(req.url);
+ try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
+ }
+
// Add cookies.
if (header_list.cookies) |cookies| {
try conn.setCookies(cookies);
@@ -926,7 +933,7 @@ pub const RequestCookie = struct {
if (arr.items.len > 0) {
try arr.append(temp, 0); //null terminate
- headers.cookies = @ptrCast(arr.items.ptr);
+ headers.cookies = @as([*c]const u8, @ptrCast(arr.items.ptr));
}
}
};
diff --git a/src/browser/Page.zig b/src/browser/Page.zig
index f0c914ad..0599bab5 100644
--- a/src/browser/Page.zig
+++ b/src/browser/Page.zig
@@ -1010,6 +1010,14 @@ pub fn scriptAddedCallback(self: *Page, comptime from_parser: bool, script: *Ele
return;
}
+ if (comptime from_parser) {
+ // parser-inserted scripts have force-async set to false, but only if
+ // they have src or non-empty content
+ if (script._src.len > 0 or script.asNode().firstChild() != null) {
+ script._force_async = false;
+ }
+ }
+
self._script_manager.addFromElement(from_parser, script, "parsing") catch |err| {
log.err(.page, "page.scriptAddedCallback", .{
.err = err,
@@ -2643,6 +2651,8 @@ pub fn _insertNodeRelative(self: *Page, comptime from_parser: bool, parent: *Nod
}
}
+ const parent_is_connected = parent.isConnected();
+
// Tri-state behavior for mutations:
// 1. from_parser=true, parse_mode=document -> no mutations (initial document parse)
// 2. from_parser=true, parse_mode=fragment -> mutations (innerHTML additions)
@@ -2658,6 +2668,15 @@ pub fn _insertNodeRelative(self: *Page, comptime from_parser: bool, parent: *Nod
// When the parser adds the node, nodeIsReady is only called when the
// nodeComplete() callback is executed.
try self.nodeIsReady(false, child);
+
+ // Check if text was added to a script that hasn't started yet.
+ if (child._type == .cdata and parent_is_connected) {
+ if (parent.is(Element.Html.Script)) |script| {
+ if (!script._executed) {
+ try self.nodeIsReady(false, parent);
+ }
+ }
+ }
}
// Notify mutation observers about childList change
@@ -2696,7 +2715,6 @@ pub fn _insertNodeRelative(self: *Page, comptime from_parser: bool, parent: *Nod
}
const parent_in_shadow = parent.is(ShadowRoot) != null or parent.isInShadowTree();
- const parent_is_connected = parent.isConnected();
if (!parent_in_shadow and !parent_is_connected) {
return;
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig
index 6f55f43b..2baeef8d 100644
--- a/src/browser/ScriptManager.zig
+++ b/src/browser/ScriptManager.zig
@@ -159,7 +159,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
//
+
+
+
+
+
+
+
+
+
diff --git a/src/browser/tests/frames/frames.html b/src/browser/tests/frames/frames.html
index 97bed281..90cb038e 100644
--- a/src/browser/tests/frames/frames.html
+++ b/src/browser/tests/frames/frames.html
@@ -108,7 +108,7 @@
{
let f5 = document.createElement('iframe');
f5.id = 'f5';
- f5.src = "support/sub 1.html";
+ f5.src = "support/page.html";
document.documentElement.appendChild(f5);
f5.src = "about:blank";
diff --git a/src/browser/tests/window/window.html b/src/browser/tests/window/window.html
index 5506e327..01025b86 100644
--- a/src/browser/tests/window/window.html
+++ b/src/browser/tests/window/window.html
@@ -125,6 +125,143 @@
testing.expectEqual(screen, window.screen);
+
+