diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 74eda203..675dd36b 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -117,6 +117,105 @@ jobs: BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js kill `cat LPD.pid` `cat PROXY.id` + # e2e tests w/ web-bot-auth configuration on. + wba-demo-scripts: + name: wba-demo-scripts + needs: zig-build-release + + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + with: + repository: 'lightpanda-io/demo' + fetch-depth: 0 + + - run: npm install + + - name: download artifact + uses: actions/download-artifact@v4 + with: + name: lightpanda-build-release + + - run: chmod a+x ./lightpanda + + - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + + - name: run end to end tests + run: | + ./lightpanda serve \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ + & echo $! > LPD.pid + go run runner/main.go + kill `cat LPD.pid` + + - name: build proxy + run: | + cd proxy + go build + + - name: run end to end tests through proxy + run: | + ./proxy/proxy & echo $! > PROXY.id + ./lightpanda serve \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \ + --http_proxy 'http://127.0.0.1:3000' \ + & echo $! > LPD.pid + go run runner/main.go + kill `cat LPD.pid` `cat PROXY.id` + + - name: run request interception through proxy + run: | + export PROXY_USERNAME=username PROXY_PASSWORD=password + ./proxy/proxy & echo $! > PROXY.id + ./lightpanda serve & echo $! > LPD.pid + URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js + BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js + kill `cat LPD.pid` `cat PROXY.id` + + wba-test: + name: wba-test + needs: zig-build-release + + env: + LIGHTPANDA_DISABLE_TELEMETRY: true + + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + with: + repository: 'lightpanda-io/demo' + fetch-depth: 0 + + - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem + + - name: download artifact + uses: actions/download-artifact@v4 + with: + name: lightpanda-build-release + + - run: chmod a+x ./lightpanda + + - name: run wba test + run: | + node webbotauth/validator.js & + VALIDATOR_PID=$! + sleep 2 + + ./lightpanda fetch http://127.0.0.1:8989/ \ + --web_bot_auth_key_file private_key.pem \ + --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \ + --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} + + wait $VALIDATOR_PID + cdp-and-hyperfine-bench: name: cdp-and-hyperfine-bench needs: zig-build-release diff --git a/src/Config.zig b/src/Config.zig index 73c7f3a7..629df32b 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -23,6 +23,8 @@ const Allocator = std.mem.Allocator; const log = @import("log.zig"); const dump = @import("browser/dump.zig"); +const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config; + pub const RunMode = enum { help, fetch, @@ -161,6 +163,17 @@ pub fn cdpTimeout(self: *const Config) usize { }; } +pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig { + return switch (self.mode) { + inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{ + .key_file = opts.common.web_bot_auth_key_file orelse return null, + .keyid = opts.common.web_bot_auth_keyid orelse return null, + .domain = opts.common.web_bot_auth_domain orelse return null, + }, + .help, .version => null, + }; +} + pub fn maxConnections(self: *const Config) u16 { return switch (self.mode) { .serve => |opts| opts.cdp_max_connections, @@ -227,6 +240,10 @@ pub const Common = struct { log_format: ?log.Format = null, log_filter_scopes: ?[]log.Scope = null, user_agent_suffix: ?[]const u8 = null, + + web_bot_auth_key_file: ?[]const u8 = null, + web_bot_auth_keyid: ?[]const u8 = null, + web_bot_auth_domain: ?[]const u8 = null, }; /// Pre-formatted HTTP headers for reuse across Http and Client. @@ -334,6 +351,14 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void { \\--user_agent_suffix \\ Suffix to append to the Lightpanda/X.Y User-Agent \\ + \\--web_bot_auth_key_file + \\ Path to the Ed25519 private key PEM file. + \\ + \\--web_bot_auth_keyid + \\ The JWK thumbprint of your public key. + \\ + \\--web_bot_auth_domain + \\ Your domain e.g. yourdomain.com ; // MAX_HELP_LEN| @@ -855,5 +880,32 @@ fn parseCommonArg( return true; } + if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" }); + return error.InvalidArgument; + }; + common.web_bot_auth_key_file = try allocator.dupe(u8, str); + return true; + } + + if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" }); + return error.InvalidArgument; + }; + common.web_bot_auth_keyid = try allocator.dupe(u8, str); + return true; + } + + if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" }); + return error.InvalidArgument; + }; + common.web_bot_auth_domain = try allocator.dupe(u8, str); + return true; + } + return false; } diff --git a/src/SemanticTree.zig b/src/SemanticTree.zig index 8e64b034..b64f222d 100644 --- a/src/SemanticTree.zig +++ b/src/SemanticTree.zig @@ -233,7 +233,7 @@ fn extractSelectOptions(node: *Node, page: *Page, arena: std.mem.Allocator) ![]O if (child.is(Element)) |el| { if (el.getTag() == .option) { if (el.is(Element.Html.Option)) |opt| { - const text = opt.getText(); + const text = opt.getText(page); const value = opt.getValue(page); const selected = opt.getSelected(); try options.append(arena, .{ .text = text, .value = value, .selected = selected }); @@ -242,7 +242,7 @@ fn extractSelectOptions(node: *Node, page: *Page, arena: std.mem.Allocator) ![]O var group_it = child.childrenIterator(); while (group_it.next()) |group_child| { if (group_child.is(Element.Html.Option)) |opt| { - const text = opt.getText(); + const text = opt.getText(page); const value = opt.getValue(page); const selected = opt.getSelected(); try options.append(arena, .{ .text = text, .value = value, .selected = selected }); diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index f716904c..1e74c046 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -30,6 +30,7 @@ const Notification = @import("../Notification.zig"); const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar; const Robots = @import("../network/Robots.zig"); const RobotStore = Robots.RobotStore; +const WebBotAuth = @import("../network/WebBotAuth.zig"); const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; @@ -702,6 +703,12 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts try conn.setHeaders(&header_list); + // If we have WebBotAuth, sign our request. + if (self.network.web_bot_auth) |*wba| { + const authority = URL.getHost(req.url); + try wba.signRequest(transfer.arena.allocator(), &header_list, authority); + } + // Add cookies. if (header_list.cookies) |cookies| { try conn.setCookies(cookies); @@ -926,7 +933,7 @@ pub const RequestCookie = struct { if (arr.items.len > 0) { try arr.append(temp, 0); //null terminate - headers.cookies = @ptrCast(arr.items.ptr); + headers.cookies = @as([*c]const u8, @ptrCast(arr.items.ptr)); } } }; diff --git a/src/browser/Page.zig b/src/browser/Page.zig index f0c914ad..0599bab5 100644 --- a/src/browser/Page.zig +++ b/src/browser/Page.zig @@ -1010,6 +1010,14 @@ pub fn scriptAddedCallback(self: *Page, comptime from_parser: bool, script: *Ele return; } + if (comptime from_parser) { + // parser-inserted scripts have force-async set to false, but only if + // they have src or non-empty content + if (script._src.len > 0 or script.asNode().firstChild() != null) { + script._force_async = false; + } + } + self._script_manager.addFromElement(from_parser, script, "parsing") catch |err| { log.err(.page, "page.scriptAddedCallback", .{ .err = err, @@ -2643,6 +2651,8 @@ pub fn _insertNodeRelative(self: *Page, comptime from_parser: bool, parent: *Nod } } + const parent_is_connected = parent.isConnected(); + // Tri-state behavior for mutations: // 1. from_parser=true, parse_mode=document -> no mutations (initial document parse) // 2. from_parser=true, parse_mode=fragment -> mutations (innerHTML additions) @@ -2658,6 +2668,15 @@ pub fn _insertNodeRelative(self: *Page, comptime from_parser: bool, parent: *Nod // When the parser adds the node, nodeIsReady is only called when the // nodeComplete() callback is executed. try self.nodeIsReady(false, child); + + // Check if text was added to a script that hasn't started yet. + if (child._type == .cdata and parent_is_connected) { + if (parent.is(Element.Html.Script)) |script| { + if (!script._executed) { + try self.nodeIsReady(false, parent); + } + } + } } // Notify mutation observers about childList change @@ -2696,7 +2715,6 @@ pub fn _insertNodeRelative(self: *Page, comptime from_parser: bool, parent: *Nod } const parent_in_shadow = parent.is(ShadowRoot) != null or parent.isInShadowTree(); - const parent_is_connected = parent.isConnected(); if (!parent_in_shadow and !parent_is_connected) { return; diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 6f55f43b..2baeef8d 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -159,7 +159,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e // + + + + + + + + + diff --git a/src/browser/tests/frames/frames.html b/src/browser/tests/frames/frames.html index 97bed281..90cb038e 100644 --- a/src/browser/tests/frames/frames.html +++ b/src/browser/tests/frames/frames.html @@ -108,7 +108,7 @@ { let f5 = document.createElement('iframe'); f5.id = 'f5'; - f5.src = "support/sub 1.html"; + f5.src = "support/page.html"; document.documentElement.appendChild(f5); f5.src = "about:blank"; diff --git a/src/browser/tests/window/window.html b/src/browser/tests/window/window.html index 5506e327..01025b86 100644 --- a/src/browser/tests/window/window.html +++ b/src/browser/tests/window/window.html @@ -125,6 +125,143 @@ testing.expectEqual(screen, window.screen); + +