Replace epoll to poll

.github/workflows/e2e-test.yml

@@ -117,98 +117,6 @@ jobs:
           BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 
-  # e2e tests w/ web-bot-auth configuration on.
-  wba-demo-scripts:
-    name: wba-demo-scripts
-    needs: zig-build-release
-
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: 'lightpanda-io/demo'
-          fetch-depth: 0
-
-      - run: npm install
-
-      - name: download artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: lightpanda-build-release
-
-      - run: chmod a+x ./lightpanda
-
-      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
-
-      - name: run end to end tests
-        run: |
-          ./lightpanda serve \
-            --web_bot_auth_key_file private_key.pem \
-            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
-            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
-            & echo $! > LPD.pid
-          go run runner/main.go
-          kill `cat LPD.pid`
-
-      - name: build proxy
-        run: |
-          cd proxy
-          go build
-
-      - name: run end to end tests through proxy
-        run: |
-          ./proxy/proxy & echo $! > PROXY.id
-          ./lightpanda serve \
-            --web_bot_auth_key_file private_key.pem \
-            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
-            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
-            --http_proxy 'http://127.0.0.1:3000' \
-            & echo $! > LPD.pid
-          go run runner/main.go
-          kill `cat LPD.pid` `cat PROXY.id`
-
-      - name: run request interception through proxy
-        run: |
-          export PROXY_USERNAME=username PROXY_PASSWORD=password
-          ./proxy/proxy & echo $! > PROXY.id
-          ./lightpanda serve & echo $! > LPD.pid
-          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
-          BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
-          kill `cat LPD.pid` `cat PROXY.id`
-
-  wba-test:
-    name: wba-test
-    needs: zig-build-release
-
-    env:
-      LIGHTPANDA_DISABLE_TELEMETRY: true
-
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    steps:
-      - name: download artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: lightpanda-build-release
-
-      - run: chmod a+x ./lightpanda
-
-      - run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
-
-      - run: |
-          ./lightpanda fetch https://crawltest.com/cdn-cgi/web-bot-auth \
-            --log_level error \
-            --web_bot_auth_key_file private_key.pem \
-            --web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
-            --web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
-            --dump markdown \
-            | tee output.log
-
-      - run: cat output.log | grep -q "unknown public key or unknown verified bot ID for keyid"
-
   cdp-and-hyperfine-bench:
     name: cdp-and-hyperfine-bench
     needs: zig-build-release

Dockerfile

@@ -36,6 +36,10 @@ RUN ZIG=$(grep '\.minimum_zig_version = "' "build.zig.zon" | cut -d'"' -f2) && \
     mv zig-${ARCH}-linux-${ZIG} /usr/local/lib && \
     ln -s /usr/local/lib/zig-${ARCH}-linux-${ZIG}/zig /usr/local/bin/zig
 
+# install deps
+RUN git submodule init && \
+    git submodule update --recursive
+
 # download and install v8
 RUN case $TARGETPLATFORM in \
     "linux/arm64") ARCH="aarch64" ;; \

src/App.zig

@@ -26,7 +26,6 @@ const Snapshot = @import("browser/js/Snapshot.zig");
 const Platform = @import("browser/js/Platform.zig");
 const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
 const RobotStore = @import("browser/Robots.zig").RobotStore;
-const WebBotAuth = @import("browser/WebBotAuth.zig");
 
 pub const Http = @import("http/Http.zig");
 pub const ArenaPool = @import("ArenaPool.zig");
@@ -41,7 +40,6 @@ telemetry: Telemetry,
 allocator: Allocator,
 arena_pool: ArenaPool,
 robots: RobotStore,
-web_bot_auth: ?WebBotAuth,
 app_dir_path: ?[]const u8,
 shutdown: bool = false,
 
@@ -54,14 +52,7 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
 
     app.robots = RobotStore.init(allocator);
 
-    if (config.webBotAuth()) |wba_cfg| {
+    app.http = try Http.init(allocator, &app.robots, config);
-        app.web_bot_auth = try WebBotAuth.fromConfig(allocator, &wba_cfg);
-    } else {
-        app.web_bot_auth = null;
-    }
-    errdefer if (app.web_bot_auth) |wba| wba.deinit(allocator);
-
-    app.http = try Http.init(allocator, &app.robots, &app.web_bot_auth, config);
     errdefer app.http.deinit();
 
     app.platform = try Platform.init();
@@ -93,9 +84,6 @@ pub fn deinit(self: *App) void {
     }
     self.telemetry.deinit();
     self.robots.deinit();
-    if (self.web_bot_auth) |wba| {
-        wba.deinit(allocator);
-    }
     self.http.deinit();
     self.snapshot.deinit();
     self.platform.deinit();

src/Config.zig

@@ -23,8 +23,6 @@ const Allocator = std.mem.Allocator;
 const log = @import("log.zig");
 const dump = @import("browser/dump.zig");
 
-const WebBotAuthConfig = @import("browser/WebBotAuth.zig").Config;
-
 pub const RunMode = enum {
     help,
     fetch,
@@ -155,17 +153,6 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
     };
 }
 
-pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
-    return switch (self.mode) {
-        inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
-            .key_file = opts.common.web_bot_auth_key_file orelse return null,
-            .keyid = opts.common.web_bot_auth_keyid orelse return null,
-            .domain = opts.common.web_bot_auth_domain orelse return null,
-        },
-        .help, .version => null,
-    };
-}
-
 pub fn maxConnections(self: *const Config) u16 {
     return switch (self.mode) {
         .serve => |opts| opts.cdp_max_connections,
@@ -230,10 +217,6 @@ pub const Common = struct {
     log_format: ?log.Format = null,
     log_filter_scopes: ?[]log.Scope = null,
     user_agent_suffix: ?[]const u8 = null,
-
-    web_bot_auth_key_file: ?[]const u8 = null,
-    web_bot_auth_keyid: ?[]const u8 = null,
-    web_bot_auth_domain: ?[]const u8 = null,
 };
 
 /// Pre-formatted HTTP headers for reuse across Http and Client.
@@ -341,14 +324,6 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
         \\--user_agent_suffix
         \\                Suffix to append to the Lightpanda/X.Y User-Agent
         \\
-        \\--web_bot_auth_key_file
-        \\                Path to the Ed25519 private key PEM file.
-        \\
-        \\--web_bot_auth_keyid
-        \\                The JWK thumbprint of your public key.
-        \\
-        \\--web_bot_auth_domain
-        \\                Your domain e.g. yourdomain.com
     ;
 
     //                                                                     MAX_HELP_LEN|
@@ -870,32 +845,5 @@ fn parseCommonArg(
         return true;
     }
 
-    if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) {
-        const str = args.next() orelse {
-            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" });
-            return error.InvalidArgument;
-        };
-        common.web_bot_auth_key_file = try allocator.dupe(u8, str);
-        return true;
-    }
-
-    if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) {
-        const str = args.next() orelse {
-            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" });
-            return error.InvalidArgument;
-        };
-        common.web_bot_auth_keyid = try allocator.dupe(u8, str);
-        return true;
-    }
-
-    if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) {
-        const str = args.next() orelse {
-            log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" });
-            return error.InvalidArgument;
-        };
-        common.web_bot_auth_domain = try allocator.dupe(u8, str);
-        return true;
-    }
-
     return false;
 }

src/Net.zig

@@ -476,9 +476,17 @@ pub const Handles = struct {
     available: HandleList,
     multi: *libcurl.CurlM,
     performing: bool = false,
+    runtime_ctx: ?RuntimeContext = null,
 
     pub const HandleList = std.DoublyLinkedList;
 
+    const RuntimeContext = struct {
+        handles: *Handles,
+        runtime: *Runtime,
+        cdp_fd: ?posix.fd_t,
+        timer_deadline_ms: ?i64 = null,
+    };
+
     pub fn init(
         allocator: Allocator,
         ca_blob: ?libcurl.CurlBlob,
@@ -532,6 +540,9 @@ pub const Handles = struct {
 
     pub fn add(self: *Handles, conn: *const Connection) !void {
         try libcurl.curl_multi_add_handle(self.multi, conn.easy);
+        if (self.runtime_ctx != null) {
+            _ = try self.socketActionTimeout();
+        }
     }
 
     pub fn remove(self: *Handles, conn: *Connection) void {
@@ -554,6 +565,10 @@ pub const Handles = struct {
     }
 
     pub fn perform(self: *Handles) !c_int {
+        if (self.runtime_ctx != null) {
+            return self.socketActionTimeout();
+        }
+
         self.performing = true;
         defer self.performing = false;
 
@@ -579,6 +594,34 @@ pub const Handles = struct {
     }
 
     pub fn poll(self: *Handles, extra_fds: []libcurl.CurlWaitFd, timeout_ms: c_int) !void {
+        if (self.runtime_ctx) |*ctx| {
+            var wait_ms: i32 = @intCast(timeout_ms);
+            try self.runDueTimer();
+
+            if (ctx.timer_deadline_ms) |deadline| {
+                const now = std.time.milliTimestamp();
+                if (deadline > now) {
+                    const remaining: i32 = @intCast(@min(deadline - now, std.math.maxInt(i32)));
+                    wait_ms = @min(wait_ms, remaining);
+                } else {
+                    wait_ms = 0;
+                }
+            }
+
+            const watched_fd: ?posix.fd_t = if (extra_fds.len > 0) extra_fds[0].fd else ctx.cdp_fd;
+            const watched_ready = try ctx.runtime.dispatchFor(wait_ms, watched_fd);
+
+            if (extra_fds.len > 0) {
+                extra_fds[0].revents = .{};
+                if (watched_ready) {
+                    extra_fds[0].revents.pollin = true;
+                }
+            }
+
+            try self.runDueTimer();
+            return;
+        }
+
         try libcurl.curl_multi_poll(self.multi, extra_fds, timeout_ms, null);
     }
 
@@ -598,6 +641,128 @@ pub const Handles = struct {
             else => unreachable,
         };
     }
+
+    pub fn attachRuntime(self: *Handles, runtime: *Runtime, cdp_fd: ?posix.fd_t) !void {
+        self.detachRuntime();
+        self.runtime_ctx = .{
+            .handles = self,
+            .runtime = runtime,
+            .cdp_fd = cdp_fd,
+            .timer_deadline_ms = null,
+        };
+
+        const ctx = &self.runtime_ctx.?;
+        try libcurl.curl_multi_setopt(self.multi, .socket_function, onCurlSocket);
+        try libcurl.curl_multi_setopt(self.multi, .socket_data, ctx);
+        try libcurl.curl_multi_setopt(self.multi, .timer_function, onCurlTimer);
+        try libcurl.curl_multi_setopt(self.multi, .timer_data, ctx);
+        _ = try self.socketActionTimeout();
+    }
+
+    pub fn detachRuntime(self: *Handles) void {
+        if (self.runtime_ctx) |*ctx| {
+            ctx.runtime.removeByCtx(ctx);
+        }
+        libcurl.curl_multi_setopt(self.multi, .socket_function, @as(?*const libcurl.CurlSocketCallback, null)) catch {};
+        libcurl.curl_multi_setopt(self.multi, .socket_data, @as(?*anyopaque, null)) catch {};
+        libcurl.curl_multi_setopt(self.multi, .timer_function, @as(?*const libcurl.CurlTimerCallback, null)) catch {};
+        libcurl.curl_multi_setopt(self.multi, .timer_data, @as(?*anyopaque, null)) catch {};
+        self.runtime_ctx = null;
+    }
+
+    fn runDueTimer(self: *Handles) !void {
+        const ctx = &(self.runtime_ctx orelse return);
+        const deadline = ctx.timer_deadline_ms orelse return;
+        if (std.time.milliTimestamp() < deadline) {
+            return;
+        }
+        ctx.timer_deadline_ms = null;
+        _ = try self.socketActionTimeout();
+    }
+
+    fn socketAction(self: *Handles, fd: posix.fd_t, events: u32) !c_int {
+        const select_mask = runtimeEventsToCurlSelect(events).toC();
+        var running: c_int = undefined;
+        self.performing = true;
+        defer self.performing = false;
+        try libcurl.curl_multi_socket_action(self.multi, @intCast(fd), select_mask, &running);
+        return running;
+    }
+
+    fn socketActionTimeout(self: *Handles) !c_int {
+        var running: c_int = undefined;
+        self.performing = true;
+        defer self.performing = false;
+        try libcurl.curl_multi_socket_action(self.multi, libcurl.CURL_SOCKET_TIMEOUT, 0, &running);
+        return running;
+    }
+
+    fn runtimeEventsToCurlSelect(events: u32) libcurl.CurlSelectMask {
+        return .{
+            .in = (events & Runtime.READABLE) != 0,
+            .out = (events & Runtime.WRITABLE) != 0,
+            .err = (events & Runtime.ERROR) != 0,
+        };
+    }
+
+    fn curlPollToRuntimeEvents(what: libcurl.CurlPoll) u32 {
+        return switch (what) {
+            .in => Runtime.READABLE | Runtime.ERROR,
+            .out => Runtime.WRITABLE | Runtime.ERROR,
+            .inout => Runtime.READABLE | Runtime.WRITABLE | Runtime.ERROR,
+            .remove => 0,
+        };
+    }
+
+    fn onRuntimeFdEvent(ctx_ptr: *anyopaque, event: RuntimeEvent) anyerror!void {
+        const ctx: *RuntimeContext = @ptrCast(@alignCast(ctx_ptr));
+        _ = try ctx.handles.socketAction(event.fd, event.events);
+    }
+
+    fn onCurlSocket(
+        easy: ?*libcurl.Curl,
+        s: libcurl.CurlSocket,
+        what_raw: c_int,
+        userp: ?*anyopaque,
+        socketp: ?*anyopaque,
+    ) callconv(.c) c_int {
+        _ = easy;
+        _ = socketp;
+        const ctx = userp orelse return 0;
+        const runtime_ctx: *RuntimeContext = @ptrCast(@alignCast(ctx));
+
+        const what = std.meta.intToEnum(libcurl.CurlPoll, what_raw) catch return 0;
+        if (what == .remove) {
+            runtime_ctx.runtime.remove(@intCast(s));
+            return 0;
+        }
+
+        runtime_ctx.runtime.add(
+            @intCast(s),
+            curlPollToRuntimeEvents(what),
+            runtime_ctx,
+            onRuntimeFdEvent,
+        ) catch {};
+        return 0;
+    }
+
+    fn onCurlTimer(
+        multi: ?*libcurl.CurlM,
+        timeout_ms: c_long,
+        userp: ?*anyopaque,
+    ) callconv(.c) c_int {
+        _ = multi;
+        const ctx = userp orelse return 0;
+        const runtime_ctx: *RuntimeContext = @ptrCast(@alignCast(ctx));
+
+        if (timeout_ms < 0) {
+            runtime_ctx.timer_deadline_ms = null;
+            return 0;
+        }
+
+        runtime_ctx.timer_deadline_ms = std.time.milliTimestamp() + @as(i64, @intCast(timeout_ms));
+        return 0;
+    }
 };
 
 // TODO: on BSD / Linux, we could just read the PEM file directly.
@@ -1381,6 +1546,148 @@ pub const WsConnection = struct {
     }
 };
 
+pub const RuntimeEvent = struct {
+    fd: posix.fd_t,
+    events: u32,
+};
+
+pub const Runtime = PollRuntime;
+
+var runtime_active = std.atomic.Value(bool).init(false);
+
+const PollRuntime = struct {
+    pub const READABLE: u32 = @intCast(posix.POLL.IN);
+    pub const WRITABLE: u32 = @intCast(posix.POLL.OUT);
+    pub const ERROR: u32 = @intCast(posix.POLL.ERR | posix.POLL.HUP | posix.POLL.NVAL);
+
+    allocator: Allocator,
+    watchers: std.AutoHashMapUnmanaged(posix.fd_t, Watcher) = .empty,
+
+    const Self = @This();
+
+    pub const EventCallback = *const fn (ctx: *anyopaque, event: RuntimeEvent) anyerror!void;
+
+    const Watcher = struct {
+        events: u32,
+        ctx: *anyopaque,
+        cb: EventCallback,
+    };
+
+    pub fn init(allocator: Allocator) !Self {
+        if (runtime_active.swap(true, .acq_rel)) {
+            return error.RuntimeAlreadyActive;
+        }
+        errdefer _ = runtime_active.swap(false, .acq_rel);
+
+        return .{
+            .allocator = allocator,
+        };
+    }
+
+    pub fn deinit(self: *Self) void {
+        self.watchers.deinit(self.allocator);
+        _ = runtime_active.swap(false, .acq_rel);
+    }
+
+    pub fn add(self: *Self, fd: posix.fd_t, events: u32, ctx: *anyopaque, cb: EventCallback) !void {
+        const gop = try self.watchers.getOrPut(self.allocator, fd);
+        gop.value_ptr.* = .{ .events = events, .ctx = ctx, .cb = cb };
+    }
+
+    pub fn remove(self: *Self, fd: posix.fd_t) void {
+        _ = self.watchers.remove(fd);
+    }
+
+    pub fn removeByCtx(self: *Self, ctx: *anyopaque) void {
+        while (true) {
+            var found: ?posix.fd_t = null;
+            var it = self.watchers.iterator();
+            while (it.next()) |entry| {
+                if (entry.value_ptr.ctx == ctx) {
+                    found = entry.key_ptr.*;
+                    break;
+                }
+            }
+            const fd = found orelse break;
+            self.remove(fd);
+        }
+    }
+
+    pub fn run(self: *Self, stop_ctx: *anyopaque, should_stop: *const fn (ctx: *anyopaque) bool) !void {
+        while (!should_stop(stop_ctx)) {
+            _ = try self.dispatch(200);
+        }
+    }
+
+    pub fn dispatch(self: *Self, timeout_ms: i32) !usize {
+        var triggered: usize = 0;
+        _ = try self.dispatchPoll(timeout_ms, null, &triggered);
+        return triggered;
+    }
+
+    pub fn dispatchFor(self: *Self, timeout_ms: i32, watched_fd: ?posix.fd_t) !bool {
+        var triggered: usize = 0;
+        return self.dispatchPoll(timeout_ms, watched_fd, &triggered);
+    }
+
+    fn dispatchPoll(self: *Self, timeout_ms: i32, watched_fd: ?posix.fd_t, triggered: *usize) !bool {
+        var pollfds: std.ArrayList(posix.pollfd) = .empty;
+        defer pollfds.deinit(self.allocator);
+
+        var it = self.watchers.iterator();
+        while (it.next()) |entry| {
+            try pollfds.append(self.allocator, .{
+                .fd = entry.key_ptr.*,
+                .events = @intCast(entry.value_ptr.events),
+                .revents = 0,
+            });
+        }
+
+        if (watched_fd) |wfd| {
+            if (!self.watchers.contains(wfd)) {
+                try pollfds.append(self.allocator, .{
+                    .fd = wfd,
+                    .events = @intCast(Self.READABLE | Self.ERROR),
+                    .revents = 0,
+                });
+            }
+        }
+
+        if (pollfds.items.len == 0) {
+            if (timeout_ms > 0) {
+                std.Thread.sleep(@as(u64, @intCast(timeout_ms)) * std.time.ns_per_ms);
+            }
+            return false;
+        }
+
+        _ = try posix.poll(pollfds.items, timeout_ms);
+
+        var watched_ready = false;
+        for (pollfds.items) |pfd| {
+            if (pfd.revents == 0) continue;
+            triggered.* += 1;
+
+            const revents_u32: u32 = @intCast(pfd.revents);
+            if (watched_fd) |wfd| {
+                if (pfd.fd == wfd) {
+                    watched_ready = true;
+                }
+            }
+
+            const watcher = self.watchers.get(pfd.fd) orelse continue;
+            watcher.cb(watcher.ctx, .{
+                .fd = pfd.fd,
+                .events = revents_u32,
+            }) catch |err| {
+                log.err(.app, "runtime callback", .{ .err = err, .fd = pfd.fd });
+                self.remove(pfd.fd);
+            };
+        }
+
+        return watched_ready;
+    }
+};
+
 const testing = std.testing;
 
 test "mask" {

src/Server.zig

@@ -121,26 +121,52 @@ pub fn run(self: *Server, address: net.Address, timeout_ms: u32) !void {
     try posix.listen(listener, self.app.config.maxPendingConnections());
 
     log.info(.app, "server running", .{ .address = address });
-    while (!self.shutdown.load(.acquire)) {
+
-        const socket = posix.accept(listener, null, null, posix.SOCK.NONBLOCK) catch |err| {
+    var runtime = try Net.Runtime.init(self.allocator);
-            switch (err) {
+    defer runtime.deinit();
-                error.SocketNotListening, error.ConnectionAborted => {
+
-                    log.info(.app, "server stopped", .{});
+    var accept_ctx: AcceptCtx = .{
-                    break;
+        .server = self,
-                },
+        .timeout_ms = timeout_ms,
-                error.WouldBlock => {
-                    std.Thread.sleep(10 * std.time.ns_per_ms);
-                    continue;
-                },
-                else => {
-                    log.err(.app, "CDP accept", .{ .err = err });
-                    std.Thread.sleep(std.time.ns_per_s);
-                    continue;
-                },
-            }
     };
 
-        self.spawnWorker(socket, timeout_ms) catch |err| {
+    try runtime.add(listener, Net.Runtime.READABLE, &accept_ctx, onListenerEvent);
+    defer runtime.remove(listener);
+
+    runtime.run(self, shouldStopRuntime) catch |err| {
+        return err;
+    };
+
+    log.info(.app, "server stopped", .{});
+}
+
+const AcceptCtx = struct {
+    server: *Server,
+    timeout_ms: u32,
+};
+
+fn shouldStopRuntime(ctx: *anyopaque) bool {
+    const self: *Server = @ptrCast(@alignCast(ctx));
+    return self.shutdown.load(.acquire);
+}
+
+fn onListenerEvent(ctx: *anyopaque, event: Net.RuntimeEvent) !void {
+    _ = event;
+    const accept_ctx: *AcceptCtx = @ptrCast(@alignCast(ctx));
+    const self = accept_ctx.server;
+    const listener = self.listener orelse return;
+
+    while (true) {
+        const socket = posix.accept(listener, null, null, posix.SOCK.NONBLOCK) catch |err| switch (err) {
+            error.WouldBlock => return,
+            error.SocketNotListening, error.ConnectionAborted => return,
+            else => {
+                log.err(.app, "CDP accept", .{ .err = err });
+                return;
+            },
+        };
+
+        self.spawnWorker(socket, accept_ctx.timeout_ms) catch |err| {
             log.err(.app, "CDP spawn", .{ .err = err });
             posix.close(socket);
         };

src/browser/Session.zig

@@ -261,7 +261,7 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
                         std.debug.assert(http_client.intercepted == 0);
                     }
 
-                    var ms: u64 = ms_to_next_task orelse blk: {
+                    const ms: u64 = ms_to_next_task orelse blk: {
                         if (wait_ms - ms_remaining < 100) {
                             if (comptime builtin.is_test) {
                                 return .done;
@@ -288,14 +288,8 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
                         // Same as above, except we have a scheduled task,
                         // it just happens to be too far into the future
                         // compared to how long we were told to wait.
-                        if (!browser.hasBackgroundTasks()) {
                         return .done;
                     }
-                        // _we_ have nothing to run, but v8 is working on
-                        // background tasks. We'll wait for them.
-                        browser.waitForBackgroundTasks();
-                        ms = 20;
-                    }
 
                     // We have a task to run in the not-so-distant future.
                     // You might think we can just sleep until that task is

src/browser/WebBotAuth.zig

@@ -1,284 +0,0 @@
-// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <francis@lightpanda.io>
-// Pierre Tachoire <pierre@lightpanda.io>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const crypto = @import("../crypto.zig");
-
-const Http = @import("../http/Http.zig");
-
-const WebBotAuth = @This();
-
-pkey: *crypto.EVP_PKEY,
-keyid: []const u8,
-directory_url: [:0]const u8,
-
-pub const Config = struct {
-    key_file: []const u8,
-    keyid: []const u8,
-    domain: []const u8,
-};
-
-fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY {
-    const begin = "-----BEGIN PRIVATE KEY-----";
-    const end = "-----END PRIVATE KEY-----";
-    const start_idx = std.mem.indexOf(u8, pem, begin) orelse return error.InvalidPem;
-    const end_idx = std.mem.indexOf(u8, pem, end) orelse return error.InvalidPem;
-
-    const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace);
-
-    // decode base64 into 48-byte DER buffer
-    var der: [48]u8 = undefined;
-    try std.base64.standard.Decoder.decode(der[0..48], b64);
-
-    // Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16.
-    const key_bytes = der[16..48];
-
-    const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32);
-    return pkey orelse error.InvalidKey;
-}
-
-fn signEd25519(pkey: *crypto.EVP_PKEY, message: []const u8, out: *[64]u8) !void {
-    const ctx = crypto.EVP_MD_CTX_new() orelse return error.OutOfMemory;
-    defer crypto.EVP_MD_CTX_free(ctx);
-
-    if (crypto.EVP_DigestSignInit(ctx, null, null, null, pkey) != 1)
-        return error.SignInit;
-
-    var sig_len: usize = 64;
-    if (crypto.EVP_DigestSign(ctx, out.ptr, &sig_len, message.ptr, message.len) != 1)
-        return error.SignFailed;
-}
-
-pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAuth {
-    const pem = try std.fs.cwd().readFileAlloc(allocator, config.key_file, 1024 * 4);
-    defer allocator.free(pem);
-
-    const pkey = try parsePemPrivateKey(pem);
-    errdefer crypto.EVP_PKEY_free(pkey);
-
-    const directory_url = try std.fmt.allocPrintSentinel(
-        allocator,
-        "https://{s}/.well-known/http-message-signatures-directory",
-        .{config.domain},
-        0,
-    );
-    errdefer allocator.free(directory_url);
-
-    return .{
-        .pkey = pkey,
-        // Owned by the Config so it's okay.
-        .keyid = config.keyid,
-        .directory_url = directory_url,
-    };
-}
-
-pub fn signRequest(
-    self: *const WebBotAuth,
-    allocator: std.mem.Allocator,
-    headers: *Http.Headers,
-    authority: []const u8,
-) !void {
-    const now = std.time.timestamp();
-    const expires = now + 60;
-
-    // build the signature-input value (without the sig1= label)
-    const sig_input_value = try std.fmt.allocPrint(
-        allocator,
-        "(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"",
-        .{ now, expires, self.keyid },
-    );
-    defer allocator.free(sig_input_value);
-
-    // build the canonical string to sign
-    const canonical = try std.fmt.allocPrint(
-        allocator,
-        "\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}",
-        .{ authority, self.directory_url, sig_input_value },
-    );
-    defer allocator.free(canonical);
-
-    // sign it
-    var sig: [64]u8 = undefined;
-    try signEd25519(self.pkey, canonical, &sig);
-
-    // base64 encode
-    const encoded_len = std.base64.standard.Encoder.calcSize(sig.len);
-    const encoded = try allocator.alloc(u8, encoded_len);
-    defer allocator.free(encoded);
-    _ = std.base64.standard.Encoder.encode(encoded, &sig);
-
-    // build the 3 headers and add them
-    const sig_agent = try std.fmt.allocPrintSentinel(
-        allocator,
-        "Signature-Agent: \"{s}\"",
-        .{self.directory_url},
-        0,
-    );
-    defer allocator.free(sig_agent);
-
-    const sig_input = try std.fmt.allocPrintSentinel(
-        allocator,
-        "Signature-Input: sig1={s}",
-        .{sig_input_value},
-        0,
-    );
-    defer allocator.free(sig_input);
-
-    const signature = try std.fmt.allocPrintSentinel(
-        allocator,
-        "Signature: sig1=:{s}:",
-        .{encoded},
-        0,
-    );
-    defer allocator.free(signature);
-
-    try headers.add(sig_agent);
-    try headers.add(sig_input);
-    try headers.add(signature);
-}
-
-pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {
-    crypto.EVP_PKEY_free(self.pkey);
-    allocator.free(self.directory_url);
-}
-
-test "parsePemPrivateKey: valid Ed25519 PKCS#8 PEM" {
-    const pem =
-        \\-----BEGIN PRIVATE KEY-----
-        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
-        \\-----END PRIVATE KEY-----
-        \\
-    ;
-
-    const pkey = try parsePemPrivateKey(pem);
-    defer crypto.EVP_PKEY_free(pkey);
-}
-
-test "parsePemPrivateKey: missing BEGIN marker returns error" {
-    const bad_pem = "-----END PRIVATE KEY-----\n";
-    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
-}
-
-test "parsePemPrivateKey: missing END marker returns error" {
-    const bad_pem = "-----BEGIN PRIVATE KEY-----\nMC4CAQA=\n";
-    try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
-}
-
-test "signEd25519: signature length is always 64 bytes" {
-    const pem =
-        \\-----BEGIN PRIVATE KEY-----
-        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
-        \\-----END PRIVATE KEY-----
-        \\
-    ;
-    const pkey = try parsePemPrivateKey(pem);
-    defer crypto.EVP_PKEY_free(pkey);
-
-    var sig: [64]u8 = @splat(0);
-    try signEd25519(pkey, "hello world", &sig);
-
-    var all_zero = true;
-    for (sig) |b| if (b != 0) {
-        all_zero = false;
-        break;
-    };
-    try std.testing.expect(!all_zero);
-}
-
-test "signEd25519: same key + message produces same signature (deterministic)" {
-    const pem =
-        \\-----BEGIN PRIVATE KEY-----
-        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
-        \\-----END PRIVATE KEY-----
-        \\
-    ;
-    const pkey = try parsePemPrivateKey(pem);
-    defer crypto.EVP_PKEY_free(pkey);
-
-    var sig1: [64]u8 = undefined;
-    var sig2: [64]u8 = undefined;
-    try signEd25519(pkey, "deterministic test", &sig1);
-    try signEd25519(pkey, "deterministic test", &sig2);
-
-    try std.testing.expectEqualSlices(u8, &sig1, &sig2);
-}
-
-test "signEd25519: same key + diff message produces different signature (deterministic)" {
-    const pem =
-        \\-----BEGIN PRIVATE KEY-----
-        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
-        \\-----END PRIVATE KEY-----
-        \\
-    ;
-    const pkey = try parsePemPrivateKey(pem);
-    defer crypto.EVP_PKEY_free(pkey);
-
-    var sig1: [64]u8 = undefined;
-    var sig2: [64]u8 = undefined;
-    try signEd25519(pkey, "msg 1", &sig1);
-    try signEd25519(pkey, "msg 2", &sig2);
-
-    try std.testing.expect(!std.mem.eql(u8, &sig1, &sig2));
-}
-
-test "signRequest: adds headers with correct names" {
-    const allocator = std.testing.allocator;
-
-    const pem =
-        \\-----BEGIN PRIVATE KEY-----
-        \\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
-        \\-----END PRIVATE KEY-----
-        \\
-    ;
-    const pkey = try parsePemPrivateKey(pem);
-
-    const directory_url = try allocator.dupeZ(
-        u8,
-        "https://example.com/.well-known/http-message-signatures-directory",
-    );
-
-    var auth = WebBotAuth{
-        .pkey = pkey,
-        .keyid = "test-key-id",
-        .directory_url = directory_url,
-    };
-    defer auth.deinit(allocator);
-
-    var headers = try Http.Headers.init("User-Agent: Test-Agent");
-    defer headers.deinit();
-
-    try auth.signRequest(allocator, &headers, "example.com");
-
-    var it = headers.iterator();
-    var found_sig_agent = false;
-    var found_sig_input = false;
-    var found_signature = false;
-    var count: usize = 0;
-
-    while (it.next()) |h| {
-        count += 1;
-        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true;
-        if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true;
-        if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true;
-    }
-
-    try std.testing.expect(count >= 3);
-    try std.testing.expect(found_sig_agent);
-    try std.testing.expect(found_sig_input);
-    try std.testing.expect(found_signature);
-}

src/browser/markdown.zig

@@ -19,8 +19,6 @@
 const std = @import("std");
 
 const Page = @import("Page.zig");
-const URL = @import("URL.zig");
-const TreeWalker = @import("webapi/TreeWalker.zig");
 const CData = @import("webapi/CData.zig");
 const Element = @import("webapi/Element.zig");
 const Node = @import("webapi/Node.zig");
@@ -105,37 +103,20 @@ fn isVisibleElement(el: *Element) bool {
     };
 }
 
-fn getAnchorLabel(el: *Element) ?[]const u8 {
-    return el.getAttributeSafe(comptime .wrap("aria-label")) orelse el.getAttributeSafe(comptime .wrap("title"));
-}
-
 fn isAllWhitespace(text: []const u8) bool {
     return for (text) |c| {
         if (!std.ascii.isWhitespace(c)) break false;
     } else true;
 }
 
-fn hasBlockDescendant(root: *Node) bool {
+fn hasBlockDescendant(node: *Node) bool {
-    var tw = TreeWalker.FullExcludeSelf.Elements.init(root, .{});
+    var it = node.childrenIterator();
-    while (tw.next()) |el| {
+    return while (it.next()) |child| {
-        if (isBlock(el.getTag())) return true;
+        if (child.is(Element)) |el| {
+            if (isBlock(el.getTag())) break true;
+            if (hasBlockDescendant(child)) break true;
         }
-    return false;
+    } else false;
-}
-
-fn hasVisibleContent(root: *Node) bool {
-    var tw = TreeWalker.FullExcludeSelf.init(root, .{});
-    while (tw.next()) |node| {
-        if (isSignificantText(node)) return true;
-        if (node.is(Element)) |el| {
-            if (!isVisibleElement(el)) {
-                tw.skipChildren();
-            } else if (el.getTag() == .img) {
-                return true;
-            }
-        }
-    }
-    return false;
 }
 
 fn ensureNewline(state: *State, writer: *std.Io.Writer) !void {
@@ -297,29 +278,20 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
             }
             try writer.writeAll("](");
             if (el.getAttributeSafe(comptime .wrap("src"))) |src| {
-                const absolute_src = URL.resolve(page.call_arena, page.base(), src, .{ .encode = true }) catch src;
+                try writer.writeAll(src);
-                try writer.writeAll(absolute_src);
             }
             try writer.writeAll(")");
             state.last_char_was_newline = false;
             return;
         },
         .anchor => {
-            const has_content = hasVisibleContent(el.asNode());
-            const label = getAnchorLabel(el);
-            const href_raw = el.getAttributeSafe(comptime .wrap("href"));
-
-            if (!has_content and label == null and href_raw == null) return;
-
             const has_block = hasBlockDescendant(el.asNode());
-            const href = if (href_raw) |h| URL.resolve(page.call_arena, page.base(), h, .{ .encode = true }) catch h else null;
-
             if (has_block) {
                 try renderChildren(el.asNode(), state, writer, page);
-                if (href) |h| {
+                if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
                     if (!state.last_char_was_newline) try writer.writeByte('\n');
-                    try writer.writeAll("([](");
+                    try writer.writeAll("([Link](");
-                    try writer.writeAll(h);
+                    try writer.writeAll(href);
                     try writer.writeAll("))\n");
                     state.last_char_was_newline = true;
                 }
@@ -329,14 +301,10 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
             if (isStandaloneAnchor(el)) {
                 if (!state.last_char_was_newline) try writer.writeByte('\n');
                 try writer.writeByte('[');
-                if (has_content) {
                 try renderChildren(el.asNode(), state, writer, page);
-                } else {
-                    try writer.writeAll(label orelse "");
-                }
                 try writer.writeAll("](");
-                if (href) |h| {
+                if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
-                    try writer.writeAll(h);
+                    try writer.writeAll(href);
                 }
                 try writer.writeAll(")\n");
                 state.last_char_was_newline = true;
@@ -344,14 +312,10 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
             }
 
             try writer.writeByte('[');
-            if (has_content) {
             try renderChildren(el.asNode(), state, writer, page);
-            } else {
-                try writer.writeAll(label orelse "");
-            }
             try writer.writeAll("](");
-            if (href) |h| {
+            if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
-                try writer.writeAll(h);
+                try writer.writeAll(href);
             }
             try writer.writeByte(')');
             state.last_char_was_newline = false;
@@ -488,8 +452,6 @@ fn testMarkdownHTML(html: []const u8, expected: []const u8) !void {
     const testing = @import("../testing.zig");
     const page = try testing.test_session.createPage();
     defer testing.test_session.removePage();
-    page.url = "http://localhost/";
-
     const doc = page.window._document;
 
     const div = try doc.createElement("div", null, page);
@@ -558,11 +520,11 @@ test "browser.markdown: blockquote" {
 }
 
 test "browser.markdown: links" {
-    try testMarkdownHTML("<a href=\"/relative\">Link</a>", "[Link](http://localhost/relative)\n");
+    try testMarkdownHTML("<a href=\"https://lightpanda.io\">Lightpanda</a>", "[Lightpanda](https://lightpanda.io)\n");
 }
 
 test "browser.markdown: images" {
-    try testMarkdownHTML("<img src=\"logo.png\" alt=\"Logo\">", "![Logo](http://localhost/logo.png)\n");
+    try testMarkdownHTML("<img src=\"logo.png\" alt=\"Logo\">", "![Logo](logo.png)\n");
 }
 
 test "browser.markdown: headings" {
@@ -603,7 +565,7 @@ test "browser.markdown: block link" {
         \\### Title
         \\
         \\Description
-        \\([](https://example.com))
+        \\([Link](https://example.com))
         \\
     );
 }
@@ -626,8 +588,8 @@ test "browser.markdown: standalone anchors" {
         \\  <a href="2">Link 2</a>
         \\</main>
     ,
-        \\[Link 1](http://localhost/1)
+        \\[Link 1](1)
-        \\[Link 2](http://localhost/2)
+        \\[Link 2](2)
         \\
     );
 }
@@ -639,58 +601,7 @@ test "browser.markdown: mixed anchors in main" {
         \\  Welcome <a href="1">Link 1</a>.
         \\</main>
     ,
-        \\Welcome [Link 1](http://localhost/1). 
+        \\Welcome [Link 1](1). 
         \\
     );
 }
-
-test "browser.markdown: skip empty links" {
-    try testMarkdownHTML(
-        \\<a href="/"></a>
-        \\<a href="/"><svg></svg></a>
-    ,
-        \\[](http://localhost/)
-        \\[](http://localhost/)
-        \\
-    );
-}
-
-test "browser.markdown: resolve links" {
-    const testing = @import("../testing.zig");
-    const page = try testing.test_session.createPage();
-    defer testing.test_session.removePage();
-    page.url = "https://example.com/a/index.html";
-
-    const doc = page.window._document;
-    const div = try doc.createElement("div", null, page);
-    try page.parseHtmlAsChildren(div.asNode(),
-        \\<a href="b">Link</a>
-        \\<img src="../c.png" alt="Img">
-        \\<a href="/my page">Space</a>
-    );
-
-    var aw: std.Io.Writer.Allocating = .init(testing.allocator);
-    defer aw.deinit();
-    try dump(div.asNode(), .{}, &aw.writer, page);
-
-    try testing.expectString(
-        \\[Link](https://example.com/a/b)
-        \\![Img](https://example.com/c.png) 
-        \\[Space](https://example.com/my%20page)
-        \\
-    , aw.written());
-}
-
-test "browser.markdown: anchor fallback label" {
-    try testMarkdownHTML(
-        \\<a href="/discord" aria-label="Discord Server"><svg></svg></a>
-    , "[Discord Server](http://localhost/discord)\n");
-
-    try testMarkdownHTML(
-        \\<a href="/search" title="Search Site"><svg></svg></a>
-    , "[Search Site](http://localhost/search)\n");
-
-    try testMarkdownHTML(
-        \\<a href="/no-label"><svg></svg></a>
-    , "[](http://localhost/no-label)\n");
-}

src/browser/webapi/TreeWalker.zig

@@ -31,7 +31,6 @@ const Mode = enum {
 
 pub fn TreeWalker(comptime mode: Mode) type {
     return struct {
-        _current: ?*Node = null,
         _next: ?*Node,
         _root: *Node,
 
@@ -48,46 +47,37 @@ pub fn TreeWalker(comptime mode: Mode) type {
 
         pub fn next(self: *Self) ?*Node {
             const node = self._next orelse return null;
-            self._current = node;
 
             if (comptime mode == .children) {
-                self._next = node.nextSibling();
+                self._next = Node.linkToNodeOrNull(node._child_link.next);
                 return node;
             }
 
-            if (node.firstChild()) |child| {
+            if (node._children) |children| {
-                self._next = child;
+                self._next = children.first();
+            } else if (node._child_link.next) |n| {
+                self._next = Node.linkToNode(n);
             } else {
-                var current: *Node = node;
+                // No children, no next sibling - walk up until we find a next sibling or hit root
-                while (current != self._root) {
+                var current = node._parent;
-                    if (current.nextSibling()) |sibling| {
+                while (current) |parent| {
-                        self._next = sibling;
+                    if (parent == self._root) {
-                        return node;
+                        self._next = null;
+                        break;
                     }
-                    current = current._parent orelse break;
+                    if (parent._child_link.next) |next_sibling| {
+                        self._next = Node.linkToNode(next_sibling);
+                        break;
                     }
+                    current = parent._parent;
+                } else {
                     self._next = null;
                 }
+            }
             return node;
         }
 
-        pub fn skipChildren(self: *Self) void {
-            if (comptime mode == .children) return;
-            const current_node = self._current orelse return;
-
-            var current: *Node = current_node;
-            while (current != self._root) {
-                if (current.nextSibling()) |sibling| {
-                    self._next = sibling;
-                    return;
-                }
-                current = current._parent orelse break;
-            }
-            self._next = null;
-        }
-
         pub fn reset(self: *Self) void {
-            self._current = null;
             self._next = firstNext(self._root);
         }
 
@@ -157,38 +147,3 @@ pub fn TreeWalker(comptime mode: Mode) type {
         };
     };
 }
-
-test "TreeWalker: skipChildren" {
-    const testing = @import("../../testing.zig");
-    const page = try testing.test_session.createPage();
-    defer testing.test_session.removePage();
-    const doc = page.window._document;
-
-    // <div>
-    //   <span>
-    //     <b>A</b>
-    //   </span>
-    //   <p>B</p>
-    // </div>
-    const div = try doc.createElement("div", null, page);
-    const span = try doc.createElement("span", null, page);
-    const b = try doc.createElement("b", null, page);
-    const p = try doc.createElement("p", null, page);
-    _ = try span.asNode().appendChild(b.asNode(), page);
-    _ = try div.asNode().appendChild(span.asNode(), page);
-    _ = try div.asNode().appendChild(p.asNode(), page);
-
-    var tw = Full.init(div.asNode(), .{});
-
-    // root (div)
-    try testing.expect(tw.next() == div.asNode());
-
-    // span
-    try testing.expect(tw.next() == span.asNode());
-
-    // skip children of span (should jump over <b> to <p>)
-    tw.skipChildren();
-    try testing.expect(tw.next() == p.asNode());
-
-    try testing.expect(tw.next() == null);
-}

src/browser/webapi/encoding/TextDecoderStream.zig

@@ -72,14 +72,6 @@ pub fn init(label_: ?[]const u8, opts_: ?InitOpts, page: *Page) !TextDecoderStre
     };
 }
 
-pub fn acquireRef(self: *TextDecoderStream) void {
-    self._transform.acquireRef();
-}
-
-pub fn deinit(self: *TextDecoderStream, shutdown: bool, page: *Page) void {
-    self._transform.deinit(shutdown, page);
-}
-
 fn decodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value, ignoreBOM: bool) !void {
     // chunk should be a Uint8Array; decode it as UTF-8 string
     const typed_array = try chunk.toZig(js.TypedArray(u8));
@@ -119,8 +111,6 @@ pub const JsApi = struct {
         pub const name = "TextDecoderStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
-        pub const weak = true;
-        pub const finalizer = bridge.finalizer(TextDecoderStream.deinit);
     };
 
     pub const constructor = bridge.constructor(TextDecoderStream.init, .{});

src/browser/webapi/encoding/TextEncoderStream.zig

@@ -34,14 +34,6 @@ pub fn init(page: *Page) !TextEncoderStream {
     };
 }
 
-pub fn acquireRef(self: *TextEncoderStream) void {
-    self._transform.acquireRef();
-}
-
-pub fn deinit(self: *TextEncoderStream, shutdown: bool, page: *Page) void {
-    self._transform.deinit(shutdown, page);
-}
-
 fn encodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value) !void {
     // chunk should be a JS string; encode it as UTF-8 bytes (Uint8Array)
     const str = chunk.isString() orelse return error.InvalidChunk;
@@ -64,8 +56,6 @@ pub const JsApi = struct {
         pub const name = "TextEncoderStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
-        pub const weak = true;
-        pub const finalizer = bridge.finalizer(TextEncoderStream.deinit);
     };
 
     pub const constructor = bridge.constructor(TextEncoderStream.init, .{});

src/browser/webapi/streams/ReadableStream.zig

@@ -52,8 +52,6 @@ _pull_fn: ?js.Function.Global = null,
 _pulling: bool = false,
 _pull_again: bool = false,
 _cancel: ?Cancel = null,
-_arena: std.mem.Allocator,
-_rc: usize = 0,
 
 const UnderlyingSource = struct {
     start: ?js.Function = null,
@@ -70,18 +68,13 @@ const QueueingStrategy = struct {
 pub fn init(src_: ?UnderlyingSource, strategy_: ?QueueingStrategy, page: *Page) !*ReadableStream {
     const strategy: QueueingStrategy = strategy_ orelse .{};
 
-    const arena = try page.getArena(.{ .debug = "ReadableStream" });
+    const self = try page._factory.create(ReadableStream{
-    errdefer page.releaseArena(arena);
-
-    const self = try arena.create(ReadableStream);
-    self.* = .{
         ._page = page,
         ._state = .readable,
-        ._arena = arena,
         ._reader = null,
         ._controller = undefined,
         ._stored_error = null,
-    };
+    });
 
     self._controller = try ReadableStreamDefaultController.init(self, strategy.highWaterMark, page);
 
@@ -115,23 +108,6 @@ pub fn initWithData(data: []const u8, page: *Page) !*ReadableStream {
     return stream;
 }
 
-pub fn deinit(self: *ReadableStream, _: bool, page: *Page) void {
-    const rc = self._rc;
-    if (comptime IS_DEBUG) {
-        std.debug.assert(rc != 0);
-    }
-
-    if (rc == 1) {
-        page.releaseArena(self._arena);
-    } else {
-        self._rc = rc - 1;
-    }
-}
-
-pub fn acquireRef(self: *ReadableStream) void {
-    self._rc += 1;
-}
-
 pub fn getReader(self: *ReadableStream, page: *Page) !*ReadableStreamDefaultReader {
     if (self.getLocked()) {
         return error.ReaderLocked;
@@ -144,12 +120,6 @@ pub fn getReader(self: *ReadableStream, page: *Page) !*ReadableStreamDefaultRead
 
 pub fn releaseReader(self: *ReadableStream) void {
     self._reader = null;
-
-    const rc = self._rc;
-    if (comptime IS_DEBUG) {
-        std.debug.assert(rc != 0);
-    }
-    self._rc = rc - 1;
 }
 
 pub fn getAsyncIterator(self: *ReadableStream, page: *Page) !*AsyncIterator {
@@ -397,8 +367,6 @@ pub const JsApi = struct {
         pub const name = "ReadableStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
-        pub const weak = true;
-        pub const finalizer = bridge.finalizer(ReadableStream.deinit);
     };
 
     pub const constructor = bridge.constructor(ReadableStream.init, .{});
@@ -422,14 +390,6 @@ pub const AsyncIterator = struct {
         });
     }
 
-    pub fn acquireRef(self: *AsyncIterator) void {
-        self._stream.acquireRef();
-    }
-
-    pub fn deinit(self: *AsyncIterator, shutdown: bool, page: *Page) void {
-        self._stream.deinit(shutdown, page);
-    }
-
     pub fn next(self: *AsyncIterator, page: *Page) !js.Promise {
         return self._reader.read(page);
     }
@@ -446,8 +406,6 @@ pub const AsyncIterator = struct {
             pub const name = "ReadableStreamAsyncIterator";
             pub const prototype_chain = bridge.prototypeChain();
             pub var class_id: bridge.ClassId = undefined;
-            pub const weak = true;
-            pub const finalizer = bridge.finalizer(AsyncIterator.deinit);
         };
 
         pub const next = bridge.function(ReadableStream.AsyncIterator.next, .{});

src/browser/webapi/streams/ReadableStreamDefaultController.zig

@@ -27,8 +27,6 @@ const ReadableStreamDefaultReader = @import("ReadableStreamDefaultReader.zig");
 
 const IS_DEBUG = @import("builtin").mode == .Debug;
 
-/// ReadableStreamDefaultController uses ReadableStream's arena to make
-/// allocation. Indeed, the controller is owned by its ReadableStream.
 const ReadableStreamDefaultController = @This();
 
 pub const Chunk = union(enum) {
@@ -48,6 +46,7 @@ pub const Chunk = union(enum) {
 
 _page: *Page,
 _stream: *ReadableStream,
+_arena: std.mem.Allocator,
 _queue: std.ArrayList(Chunk),
 _pending_reads: std.ArrayList(js.PromiseResolver.Global),
 _high_water_mark: u32,
@@ -57,22 +56,15 @@ pub fn init(stream: *ReadableStream, high_water_mark: u32, page: *Page) !*Readab
         ._page = page,
         ._queue = .empty,
         ._stream = stream,
+        ._arena = page.arena,
         ._pending_reads = .empty,
         ._high_water_mark = high_water_mark,
     });
 }
 
-pub fn acquireRef(self: *ReadableStreamDefaultController) void {
-    self._stream.acquireRef();
-}
-
-pub fn deinit(self: *ReadableStreamDefaultController, shutdown: bool, page: *Page) void {
-    self._stream.deinit(shutdown, page);
-}
-
 pub fn addPendingRead(self: *ReadableStreamDefaultController, page: *Page) !js.Promise {
     const resolver = page.js.local.?.createPromiseResolver();
-    try self._pending_reads.append(self._stream._arena, try resolver.persist());
+    try self._pending_reads.append(self._arena, try resolver.persist());
     return resolver.promise();
 }
 
@@ -82,8 +74,8 @@ pub fn enqueue(self: *ReadableStreamDefaultController, chunk: Chunk) !void {
     }
 
     if (self._pending_reads.items.len == 0) {
-        const chunk_copy = try chunk.dupe(self._stream._arena);
+        const chunk_copy = try chunk.dupe(self._page.arena);
-        return self._queue.append(self._stream._arena, chunk_copy);
+        return self._queue.append(self._arena, chunk_copy);
     }
 
     // I know, this is ouch! But we expect to have very few (if any)
@@ -117,7 +109,7 @@ pub fn enqueueValue(self: *ReadableStreamDefaultController, value: js.Value) !vo
 
     if (self._pending_reads.items.len == 0) {
         const persisted = try value.persist();
-        try self._queue.append(self._stream._arena, .{ .js_value = persisted });
+        try self._queue.append(self._arena, .{ .js_value = persisted });
         return;
     }
 
@@ -178,7 +170,7 @@ pub fn doError(self: *ReadableStreamDefaultController, err: []const u8) !void {
     }
 
     self._stream._state = .errored;
-    self._stream._stored_error = try self._stream._arena.dupe(u8, err);
+    self._stream._stored_error = try self._page.arena.dupe(u8, err);
 
     // Reject all pending reads
     for (self._pending_reads.items) |resolver| {
@@ -218,8 +210,6 @@ pub const JsApi = struct {
         pub const name = "ReadableStreamDefaultController";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
-        pub const weak = true;
-        pub const finalizer = bridge.finalizer(ReadableStreamDefaultController.deinit);
     };
 
     pub const enqueue = bridge.function(ReadableStreamDefaultController.enqueueValue, .{});

src/browser/webapi/streams/ReadableStreamDefaultReader.zig

@@ -19,8 +19,6 @@
 const std = @import("std");
 const js = @import("../../js/js.zig");
 
-const IS_DEBUG = @import("builtin").mode == .Debug;
-
 const Page = @import("../../Page.zig");
 const ReadableStream = @import("ReadableStream.zig");
 const ReadableStreamDefaultController = @import("ReadableStreamDefaultController.zig");
@@ -37,21 +35,6 @@ pub fn init(stream: *ReadableStream, page: *Page) !*ReadableStreamDefaultReader
     });
 }
 
-pub fn acquireRef(self: *ReadableStreamDefaultReader) void {
-    const stream = self._stream orelse {
-        if (comptime IS_DEBUG) {
-            std.debug.assert(false);
-        }
-        return;
-    };
-    stream.acquireRef();
-}
-
-pub fn deinit(self: *ReadableStreamDefaultReader, shutdown: bool, page: *Page) void {
-    const stream = self._stream orelse return;
-    stream.deinit(shutdown, page);
-}
-
 pub const ReadResult = struct {
     done: bool,
     value: Chunk,
@@ -127,8 +110,6 @@ pub const JsApi = struct {
         pub const name = "ReadableStreamDefaultReader";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
-        pub const weak = true;
-        pub const finalizer = bridge.finalizer(ReadableStreamDefaultReader.deinit);
     };
 
     pub const read = bridge.function(ReadableStreamDefaultReader.read, .{});

src/browser/webapi/streams/TransformStream.zig

@@ -85,14 +85,6 @@ pub fn initWithZigTransform(zig_transform: ZigTransformFn, page: *Page) !*Transf
     return self;
 }
 
-pub fn acquireRef(self: *TransformStream) void {
-    self._readable.acquireRef();
-}
-
-pub fn deinit(self: *TransformStream, shutdown: bool, page: *Page) void {
-    self._readable.deinit(shutdown, page);
-}
-
 pub fn transformWrite(self: *TransformStream, chunk: js.Value, page: *Page) !void {
     if (self._controller._zig_transform_fn) |zig_fn| {
         // Zig-level transform (used by TextEncoderStream etc.)
@@ -138,8 +130,6 @@ pub const JsApi = struct {
         pub const name = "TransformStream";
         pub const prototype_chain = bridge.prototypeChain();
         pub var class_id: bridge.ClassId = undefined;
-        pub const weak = true;
-        pub const finalizer = bridge.finalizer(TransformStream.deinit);
     };
 
     pub const constructor = bridge.constructor(TransformStream.init, .{});
@@ -175,14 +165,6 @@ pub const TransformStreamDefaultController = struct {
         });
     }
 
-    pub fn acquireRef(self: *TransformStreamDefaultController) void {
-        self._stream.acquireRef();
-    }
-
-    pub fn deinit(self: *TransformStreamDefaultController, shutdown: bool, page: *Page) void {
-        self._stream.deinit(shutdown, page);
-    }
-
     pub fn enqueue(self: *TransformStreamDefaultController, chunk: ReadableStreamDefaultController.Chunk) !void {
         try self._stream._readable._controller.enqueue(chunk);
     }
@@ -207,8 +189,6 @@ pub const TransformStreamDefaultController = struct {
             pub const name = "TransformStreamDefaultController";
             pub const prototype_chain = bridge.prototypeChain();
             pub var class_id: bridge.ClassId = undefined;
-            pub const weak = true;
-            pub const finalizer = bridge.finalizer(TransformStreamDefaultController.deinit);
         };
 
         pub const enqueue = bridge.function(TransformStreamDefaultController.enqueueValue, .{});

src/crypto.zig

@@ -228,8 +228,6 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8
 
 pub const NID_X25519 = @as(c_int, 948);
 pub const EVP_PKEY_X25519 = NID_X25519;
-pub const NID_ED25519 = 949;
-pub const EVP_PKEY_ED25519 = NID_ED25519;
 
 pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
 pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
@@ -238,11 +236,3 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void;
 pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int;
 pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int;
 pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int;
-pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void;
-
-pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int;
-pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int;
-pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX;
-pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void;
-pub const struct_evp_md_ctx_st = opaque {};
-pub const EVP_MD_CTX = struct_evp_md_ctx_st;

src/http/Client.zig

@@ -29,8 +29,6 @@ const Notification = @import("../Notification.zig");
 const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
 const Robots = @import("../browser/Robots.zig");
 const RobotStore = Robots.RobotStore;
-const WebBotAuth = @import("../browser/WebBotAuth.zig");
-
 const posix = std.posix;
 
 const Allocator = std.mem.Allocator;
@@ -85,9 +83,6 @@ robot_store: *RobotStore,
 // Allows us to fetch the robots.txt just once.
 pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empty,
 
-// Reference to the App-owned WebBotAuth.
-web_bot_auth: *const ?WebBotAuth,
-
 // Once we have a handle/easy to process a request with, we create a Transfer
 // which contains the Request as well as any state we need to process the
 // request. These wil come and go with each request.
@@ -126,13 +121,7 @@ pub const CDPClient = struct {
 
 const TransferQueue = std.DoublyLinkedList;
 
-pub fn init(
+pub fn init(allocator: Allocator, ca_blob: ?Net.Blob, robot_store: *RobotStore, config: *const Config) !*Client {
-    allocator: Allocator,
-    ca_blob: ?Net.Blob,
-    robot_store: *RobotStore,
-    web_bot_auth: *const ?WebBotAuth,
-    config: *const Config,
-) !*Client {
     var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
     errdefer transfer_pool.deinit();
 
@@ -156,7 +145,6 @@ pub fn init(
         .handles = handles,
         .allocator = allocator,
         .robot_store = robot_store,
-        .web_bot_auth = web_bot_auth,
         .http_proxy = http_proxy,
         .use_proxy = http_proxy != null,
         .config = config,
@@ -185,6 +173,14 @@ pub fn newHeaders(self: *const Client) !Net.Headers {
     return Net.Headers.init(self.config.http_headers.user_agent_header);
 }
 
+pub fn attachRuntime(self: *Client, runtime: *Net.Runtime, cdp_fd: ?posix.fd_t) !void {
+    try self.handles.attachRuntime(runtime, cdp_fd);
+}
+
+pub fn detachRuntime(self: *Client) void {
+    self.handles.detachRuntime();
+}
+
 pub fn abort(self: *Client) void {
     self._abort(true, 0);
 }
@@ -721,12 +717,6 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
         try conn.secretHeaders(&header_list, &self.config.http_headers); // Add headers that must be hidden from intercepts
         try conn.setHeaders(&header_list);
 
-        // If we have WebBotAuth, sign our request.
-        if (self.web_bot_auth.*) |wba| {
-            const authority = URL.getHost(req.url);
-            try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
-        }
-
         // Add cookies.
         if (header_list.cookies) |cookies| {
             try conn.setCookies(cookies);
@@ -1320,7 +1310,7 @@ pub const Transfer = struct {
             }
             transfer._redirecting = false;
 
-            if ((status == 401 or status == 407) and transfer.client.use_proxy) {
+            if (status == 401 or status == 407) {
                 // The auth challenge must be parsed from a following
                 // WWW-Authenticate or Proxy-Authenticate header.
                 transfer._auth_challenge = .{

src/http/Http.zig

@@ -29,7 +29,6 @@ pub const Headers = Net.Headers;
 
 const Config = @import("../Config.zig");
 const RobotStore = @import("../browser/Robots.zig").RobotStore;
-const WebBotAuth = @import("../browser/WebBotAuth.zig");
 
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
@@ -46,14 +45,8 @@ allocator: Allocator,
 config: *const Config,
 ca_blob: ?Net.Blob,
 robot_store: *RobotStore,
-web_bot_auth: *const ?WebBotAuth,
 
-pub fn init(
+pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Config) !Http {
-    allocator: Allocator,
-    robot_store: *RobotStore,
-    web_bot_auth: *const ?WebBotAuth,
-    config: *const Config,
-) !Http {
     try Net.globalInit();
     errdefer Net.globalDeinit();
 
@@ -75,7 +68,6 @@ pub fn init(
         .config = config,
         .ca_blob = ca_blob,
         .robot_store = robot_store,
-        .web_bot_auth = web_bot_auth,
     };
 }
 
@@ -89,7 +81,7 @@ pub fn deinit(self: *Http) void {
 }
 
 pub fn createClient(self: *Http, allocator: Allocator) !*Client {
-    return Client.init(allocator, self.ca_blob, self.robot_store, self.web_bot_auth, self.config);
+    return Client.init(allocator, self.ca_blob, self.robot_store, self.config);
 }
 
 pub fn newConnection(self: *Http) !Net.Connection {

src/mcp/Server.zig

@@ -24,29 +24,22 @@ mutex: std.Thread.Mutex = .{},
 aw: std.io.Writer.Allocating,
 
 pub fn init(allocator: std.mem.Allocator, app: *App, writer: *std.io.Writer) !*Self {
-    const http_client = try app.http.createClient(allocator);
-    errdefer http_client.deinit();
-
-    const notification = try lp.Notification.init(allocator);
-    errdefer notification.deinit();
-
     const self = try allocator.create(Self);
     errdefer allocator.destroy(self);
 
-    var browser = try lp.Browser.init(app, .{ .http_client = http_client });
+    self.allocator = allocator;
-    errdefer browser.deinit();
+    self.app = app;
+    self.writer = writer;
+    self.aw = .init(allocator);
 
-    self.* = .{
+    self.http_client = try app.http.createClient(allocator);
-        .allocator = allocator,
+    errdefer self.http_client.deinit();
-        .app = app,
+
-        .writer = writer,
+    self.notification = try .init(allocator);
-        .browser = browser,
+    errdefer self.notification.deinit();
-        .aw = .init(allocator),
+
-        .http_client = http_client,
+    self.browser = try lp.Browser.init(app, .{ .http_client = self.http_client });
-        .notification = notification,
+    errdefer self.browser.deinit();
-        .session = undefined,
-        .page = undefined,
-    };
 
     self.session = try self.browser.newSession(self.notification);
     self.page = try self.session.createPage();

src/sys/libcurl.zig

@@ -39,7 +39,32 @@ pub const CurlOffT = c.curl_off_t;
 pub const CurlDebugFunction = fn (*Curl, CurlInfoType, [*c]u8, usize, *anyopaque) c_int;
 pub const CurlHeaderFunction = fn ([*]const u8, usize, usize, *anyopaque) usize;
 pub const CurlWriteFunction = fn ([*]const u8, usize, usize, *anyopaque) usize;
+pub const CurlSocketCallback = fn (?*Curl, CurlSocket, c_int, ?*anyopaque, ?*anyopaque) callconv(.c) c_int;
+pub const CurlTimerCallback = fn (?*CurlM, c_long, ?*anyopaque) callconv(.c) c_int;
 pub const curl_writefunc_error: usize = c.CURL_WRITEFUNC_ERROR;
+pub const CURL_SOCKET_TIMEOUT: CurlSocket = c.CURL_SOCKET_TIMEOUT;
+
+pub const CurlPoll = enum(c_int) {
+    in = c.CURL_POLL_IN,
+    out = c.CURL_POLL_OUT,
+    inout = c.CURL_POLL_INOUT,
+    remove = c.CURL_POLL_REMOVE,
+};
+
+pub const CurlSelectMask = packed struct(c_int) {
+    in: bool = false,
+    out: bool = false,
+    err: bool = false,
+    _reserved: std.meta.Int(.unsigned, @bitSizeOf(c_int) - 3) = 0,
+
+    pub fn toC(self: @This()) c_int {
+        var mask: c_int = 0;
+        if (self.in) mask |= c.CURL_CSELECT_IN;
+        if (self.out) mask |= c.CURL_CSELECT_OUT;
+        if (self.err) mask |= c.CURL_CSELECT_ERR;
+        return mask;
+    }
+};
 
 pub const CurlGlobalFlags = packed struct(u8) {
     ssl: bool = false,
@@ -156,6 +181,10 @@ pub const CurlOption = enum(c.CURLoption) {
 
 pub const CurlMOption = enum(c.CURLMoption) {
     max_host_connections = c.CURLMOPT_MAX_HOST_CONNECTIONS,
+    socket_function = c.CURLMOPT_SOCKETFUNCTION,
+    socket_data = c.CURLMOPT_SOCKETDATA,
+    timer_function = c.CURLMOPT_TIMERFUNCTION,
+    timer_data = c.CURLMOPT_TIMERDATA,
 };
 
 pub const CurlInfo = enum(c.CURLINFO) {
@@ -675,6 +704,10 @@ pub fn curl_multi_setopt(multi: *CurlM, comptime option: CurlMOption, value: any
             };
             break :blk c.curl_multi_setopt(multi, opt, n);
         },
+        .socket_function => c.curl_multi_setopt(multi, opt, value),
+        .socket_data => c.curl_multi_setopt(multi, opt, value),
+        .timer_function => c.curl_multi_setopt(multi, opt, value),
+        .timer_data => c.curl_multi_setopt(multi, opt, value),
     };
     try errorMCheck(code);
 }
@@ -701,6 +734,15 @@ pub fn curl_multi_poll(
     try errorMCheck(c.curl_multi_poll(multi, raw_fds, @intCast(extra_fds.len), timeout_ms, numfds));
 }
 
+pub fn curl_multi_socket_action(
+    multi: *CurlM,
+    s: CurlSocket,
+    ev_bitmask: c_int,
+    running_handles: *c_int,
+) ErrorMulti!void {
+    try errorMCheck(c.curl_multi_socket_action(multi, s, ev_bitmask, running_handles));
+}
+
 pub fn curl_multi_info_read(multi: *CurlM, msgs_in_queue: *c_int) ?CurlMsg {
     const ptr = c.curl_multi_info_read(multi, msgs_in_queue);
     if (ptr == null) return null;