Merge pull request #675 from lightpanda-io/http_request_notifications

HTTP request notification

.github/workflows/e2e-test.yml

@@ -63,7 +63,7 @@ jobs:
     needs: zig-build-release
 
     env:
-      MAX_MEMORY: 28000
+      MAX_MEMORY: 29000
       MAX_AVG_DURATION: 24
       LIGHTPANDA_DISABLE_TELEMETRY: true
 

src/browser/env.zig

@@ -7,7 +7,7 @@ const storage = @import("storage/storage.zig");
 const generate = @import("../runtime/generate.zig");
 const Renderer = @import("renderer.zig").Renderer;
 const Loop = @import("../runtime/loop.zig").Loop;
-const HttpClient = @import("../http/client.zig").Client;
+const RequestFactory = @import("../http/client.zig").RequestFactory;
 
 const WebApis = struct {
     // Wrapped like this for debug ergonomics.
@@ -54,8 +54,8 @@ pub const SessionState = struct {
     window: *Window,
     renderer: *Renderer,
     arena: std.mem.Allocator,
-    http_client: *HttpClient,
     cookie_jar: *storage.CookieJar,
+    request_factory: RequestFactory,
 
     // dangerous, but set by the JS framework
     // shorter-lived than the arena above, which

src/browser/page.zig

@@ -98,7 +98,7 @@ pub const Page = struct {
                 .renderer = &self.renderer,
                 .loop = browser.app.loop,
                 .cookie_jar = &session.cookie_jar,
-                .http_client = browser.http_client,
+                .request_factory = browser.http_client.requestFactory(browser.notification),
             },
             .scope = try session.executor.startScope(&self.window, &self.state, self, true),
             .module_map = .empty,
@@ -174,6 +174,7 @@ pub const Page = struct {
     pub fn navigate(self: *Page, request_url: URL, opts: NavigateOpts) !void {
         const arena = self.arena;
         const session = self.session;
+        const notification = session.browser.notification;
 
         log.debug("starting GET {s}", .{request_url});
 
@@ -195,10 +196,11 @@ pub const Page = struct {
         // load the data
         var request = try self.newHTTPRequest(.GET, &self.url, .{ .navigation = true });
         defer request.deinit();
+        request.notification = notification;
 
-        session.browser.notification.dispatch(.page_navigate, &.{
+        notification.dispatch(.page_navigate, &.{
+            .opts = opts,
             .url = &self.url,
-            .reason = opts.reason,
             .timestamp = timestamp(),
         });
 
@@ -238,7 +240,7 @@ pub const Page = struct {
             self.raw_data = arr.items;
         }
 
-        session.browser.notification.dispatch(.page_navigated, &.{
+        notification.dispatch(.page_navigated, &.{
             .url = &self.url,
             .timestamp = timestamp(),
         });
@@ -464,7 +466,9 @@ pub const Page = struct {
     }
 
     fn newHTTPRequest(self: *const Page, method: http.Request.Method, url: *const URL, opts: storage.cookie.LookupOpts) !http.Request {
-        var request = try self.state.http_client.request(method, &url.uri);
+        // Don't use the state's request_factory here, since requests made by the
+        // page (i.e. to load <scripts>) should not generate notifications.
+        var request = try self.session.browser.http_client.request(method, &url.uri);
         errdefer request.deinit();
 
         var arr: std.ArrayListUnmanaged(u8) = .{};
@@ -661,7 +665,8 @@ pub const NavigateReason = enum {
     address_bar,
 };
 
-const NavigateOpts = struct {
+pub const NavigateOpts = struct {
+    cdp_id: ?i64 = null,
     reason: NavigateReason = .address_bar,
 };
 

src/browser/xhr/xhr.zig

@@ -80,7 +80,6 @@ const XMLHttpRequestBodyInit = union(enum) {
 pub const XMLHttpRequest = struct {
     proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
     arena: Allocator,
-    client: *http.Client,
     request: ?http.Request = null,
 
     priv_state: PrivState = .new,
@@ -252,7 +251,6 @@ pub const XMLHttpRequest = struct {
             .state = .unsent,
             .url = null,
             .origin_url = session_state.url,
-            .client = session_state.http_client,
             .cookie_jar = session_state.cookie_jar,
         };
     }
@@ -420,7 +418,7 @@ pub const XMLHttpRequest = struct {
         self.send_flag = true;
         self.priv_state = .open;
 
-        self.request = try self.client.request(self.method, &self.url.?.uri);
+        self.request = try session_state.request_factory.create(self.method, &self.url.?.uri);
         var request = &self.request.?;
         errdefer request.deinit();
 

src/cdp/cdp.zig

@@ -69,6 +69,9 @@ pub fn CDPT(comptime TypeProvider: type) type {
         // 1 message at a time.
         message_arena: std.heap.ArenaAllocator,
 
+        // Used for processing notifications within a browser context.
+        notification_arena: std.heap.ArenaAllocator,
+
         const Self = @This();
 
         pub fn init(app: *App, client: TypeProvider.Client) !Self {
@@ -82,6 +85,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 .allocator = allocator,
                 .browser_context = null,
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
+                .notification_arena = std.heap.ArenaAllocator.init(allocator),
             };
         }
 
@@ -91,6 +95,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
             }
             self.browser.deinit();
             self.message_arena.deinit();
+            self.notification_arena.deinit();
         }
 
         pub fn handleMessage(self: *Self, msg: []const u8) bool {
@@ -259,7 +264,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
             });
         }
 
-        fn sendJSON(self: *Self, message: anytype) !void {
+        pub fn sendJSON(self: *Self, message: anytype) !void {
             return self.client.sendJSON(message, .{
                 .emit_null_optional_fields = false,
             });
@@ -283,6 +288,12 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         // Points to the session arena
         arena: Allocator,
 
+        // From the parent's notification_arena.allocator(). Most of the CDP
+        // code paths deal with a cmd which has its own arena (from the
+        // message_arena). But notifications happen outside of the typical CDP
+        // request->response, and thus don't have a cmd and don't have an arena.
+        notification_arena: Allocator,
+
         // Maps to our Page. (There are other types of targets, but we only
         // deal with "pages" for now). Since we only allow 1 open page at a
         // time, we only have 1 target_id.
@@ -336,6 +347,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
                 .node_search_list = undefined,
                 .isolated_world = null,
                 .inspector = inspector,
+                .notification_arena = cdp.notification_arena.allocator(),
             };
             self.node_search_list = Node.Search.List.init(allocator, &self.node_registry);
             errdefer self.deinit();
@@ -397,6 +409,16 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             return if (raw_url.len == 0) null else raw_url;
         }
 
+        pub fn networkEnable(self: *Self) !void {
+            try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart);
+            try self.cdp.browser.notification.register(.http_request_complete, self, onHttpRequestComplete);
+        }
+
+        pub fn networkDisable(self: *Self) void {
+            self.cdp.browser.notification.unregister(.http_request_start, self);
+            self.cdp.browser.notification.unregister(.http_request_complete, self);
+        }
+
         pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             return @import("domains/page.zig").pageRemove(self);
@@ -409,7 +431,8 @@ pub fn BrowserContext(comptime CDP_T: type) type {
 
         pub fn onPageNavigate(ctx: *anyopaque, data: *const Notification.PageNavigate) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
-            return @import("domains/page.zig").pageNavigate(self, data);
+            defer self.resetNotificationArena();
+            return @import("domains/page.zig").pageNavigate(self.notification_arena, self, data);
         }
 
         pub fn onPageNavigated(ctx: *anyopaque, data: *const Notification.PageNavigated) !void {
@@ -417,6 +440,22 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             return @import("domains/page.zig").pageNavigated(self, data);
         }
 
+        pub fn onHttpRequestStart(ctx: *anyopaque, data: *const Notification.RequestStart) !void {
+            const self: *Self = @alignCast(@ptrCast(ctx));
+            defer self.resetNotificationArena();
+            return @import("domains/network.zig").httpRequestStart(self.notification_arena, self, data);
+        }
+
+        pub fn onHttpRequestComplete(ctx: *anyopaque, data: *const Notification.RequestComplete) !void {
+            const self: *Self = @alignCast(@ptrCast(ctx));
+            defer self.resetNotificationArena();
+            return @import("domains/network.zig").httpRequestComplete(self.notification_arena, self, data);
+        }
+
+        fn resetNotificationArena(self: *Self) void {
+            defer _ = self.cdp.notification_arena.reset(.{ .retain_with_limit = 1024 * 64 });
+        }
+
         pub fn callInspector(self: *const Self, msg: []const u8) void {
             self.inspector.send(msg);
             // force running micro tasks after send input to the inspector.

src/cdp/domains/network.zig

@@ -17,15 +17,130 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
+const Notification = @import("../../notification.zig").Notification;
+
+const Allocator = std.mem.Allocator;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
         enable,
+        disable,
         setCacheDisabled,
     }, cmd.input.action) orelse return error.UnknownMethod;
 
     switch (action) {
-        .enable => return cmd.sendResult(null, .{}),
+        .enable => return enable(cmd),
+        .disable => return disable(cmd),
         .setCacheDisabled => return cmd.sendResult(null, .{}),
     }
 }
+
+fn enable(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    try bc.networkEnable();
+    return cmd.sendResult(null, .{});
+}
+
+fn disable(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    bc.networkDisable();
+    return cmd.sendResult(null, .{});
+}
+
+pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notification.RequestStart) !void {
+    // Isn't possible to do a network request within a Browser (which our
+    // notification is tied to), without a page.
+    std.debug.assert(bc.session.page != null);
+
+    var cdp = bc.cdp;
+
+    // all unreachable because we _have_ to have a page.
+    const session_id = bc.session_id orelse unreachable;
+    const target_id = bc.target_id orelse unreachable;
+    const page = bc.session.currentPage() orelse unreachable;
+
+    const document_url = try urlToString(arena, &page.url.uri, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    const request_url = try urlToString(arena, request.url, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    const request_fragment = try urlToString(arena, request.url, .{
+        .fragment = true,
+    });
+
+    var headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
+    try headers.ensureTotalCapacity(arena, request.headers.len);
+    for (request.headers) |header| {
+        headers.putAssumeCapacity(header.name, header.value);
+    }
+
+    // We're missing a bunch of fields, but, for now, this seems like enough
+    try cdp.sendEvent("Network.requestWillBeSent", .{
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}),
+        .frameId = target_id,
+        .loaderId = bc.loader_id,
+        .documentUrl = document_url,
+        .request = .{
+            .url = request_url,
+            .urlFragment = request_fragment,
+            .method = @tagName(request.method),
+            .hasPostData = request.has_body,
+            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
+        },
+    }, .{ .session_id = session_id });
+}
+
+pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notification.RequestComplete) !void {
+    // Isn't possible to do a network request within a Browser (which our
+    // notification is tied to), without a page.
+    std.debug.assert(bc.session.page != null);
+
+    var cdp = bc.cdp;
+
+    // all unreachable because we _have_ to have a page.
+    const session_id = bc.session_id orelse unreachable;
+    const target_id = bc.target_id orelse unreachable;
+
+    const url = try urlToString(arena, request.url, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    var headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
+    try headers.ensureTotalCapacity(arena, request.headers.len);
+    for (request.headers) |header| {
+        headers.putAssumeCapacity(header.name, header.value);
+    }
+
+    // We're missing a bunch of fields, but, for now, this seems like enough
+    try cdp.sendEvent("Network.responseReceived", .{
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}),
+        .frameId = target_id,
+        .loaderId = bc.loader_id,
+        .response = .{
+            .url = url,
+            .status = request.status,
+            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
+        },
+    }, .{ .session_id = session_id });
+}
+
+fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 {
+    var buf: std.ArrayListUnmanaged(u8) = .empty;
+    try url.writeToStream(opts, buf.writer(arena));
+    return buf.items;
+}

src/cdp/domains/page.zig

@@ -21,6 +21,8 @@ const URL = @import("../../url.zig").URL;
 const Page = @import("../../browser/page.zig").Page;
 const Notification = @import("../../notification.zig").Notification;
 
+const Allocator = std.mem.Allocator;
+
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
         enable,
@@ -137,7 +139,7 @@ fn navigate(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
 
     // didn't create?
-    const target_id = bc.target_id orelse return error.TargetIdNotLoaded;
+    // const target_id = bc.target_id orelse return error.TargetIdNotLoaded;
 
     // didn't attach?
     if (bc.session_id == null) {
@@ -148,17 +150,14 @@ fn navigate(cmd: anytype) !void {
 
     var page = bc.session.currentPage() orelse return error.PageNotLoaded;
     bc.loader_id = bc.cdp.loader_id_gen.next();
-    try cmd.sendResult(.{
-        .frameId = target_id,
-        .loaderId = bc.loader_id,
-    }, .{});
 
     try page.navigate(url, .{
         .reason = .address_bar,
+        .cdp_id = cmd.input.id,
     });
 }
 
-pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void {
+pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.PageNavigate) !void {
     // I don't think it's possible that we get these notifications and don't
     // have these things setup.
     std.debug.assert(bc.session.page != null);
@@ -170,7 +169,8 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
 
     bc.reset();
 
-    if (event.reason == .anchor) {
+    const is_anchor = event.opts.reason == .anchor;
+    if (is_anchor) {
         try cdp.sendEvent("Page.frameScheduledNavigation", .{
             .frameId = target_id,
             .delay = 0,
@@ -199,6 +199,22 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
         .frameId = target_id,
     }, .{ .session_id = session_id });
 
+    // Drivers are sensitive to the order of events. Some more than others.
+    // The result for the Page.navigate seems like it _must_ come after
+    // the frameStartedLoading, but before any lifecycleEvent. So we
+    // unfortunately have to put the input_id ito the NavigateOpts which gets
+    // passed back into the notification.
+    if (event.opts.cdp_id) |input_id| {
+        try cdp.sendJSON(.{
+            .id = input_id,
+            .result = .{
+                .frameId = target_id,
+                .loaderId = loader_id,
+            },
+            .sessionId = session_id,
+        });
+    }
+
     if (bc.page_life_cycle_events) {
         try cdp.sendEvent("Page.lifecycleEvent", LifecycleEvent{
             .name = "init",
@@ -208,7 +224,7 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
         }, .{ .session_id = session_id });
     }
 
-    if (event.reason == .anchor) {
+    if (is_anchor) {
         try cdp.sendEvent("Page.frameClearedScheduledNavigation", .{
             .frameId = target_id,
         }, .{ .session_id = session_id });
@@ -219,21 +235,19 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
     // The client will expect us to send new contextCreated events, such that the client has new id's for the active contexts.
     try cdp.sendEvent("Runtime.executionContextsCleared", null, .{ .session_id = session_id });
 
-    var buffer: [512]u8 = undefined;
     {
-        var fba = std.heap.FixedBufferAllocator.init(&buffer);
         const page = bc.session.currentPage() orelse return error.PageNotLoaded;
-        const aux_data = try std.fmt.allocPrint(fba.allocator(), "{{\"isDefault\":true,\"type\":\"default\",\"frameId\":\"{s}\"}}", .{target_id});
+        const aux_data = try std.fmt.allocPrint(arena, "{{\"isDefault\":true,\"type\":\"default\",\"frameId\":\"{s}\"}}", .{target_id});
         bc.inspector.contextCreated(
             page.scope,
             "",
-            try page.origin(fba.allocator()),
+            try page.origin(arena),
             aux_data,
             true,
         );
     }
     if (bc.isolated_world) |*isolated_world| {
-        const aux_json = try std.fmt.bufPrint(&buffer, "{{\"isDefault\":false,\"type\":\"isolated\",\"frameId\":\"{s}\"}}", .{target_id});
+        const aux_json = try std.fmt.allocPrint(arena, "{{\"isDefault\":false,\"type\":\"isolated\",\"frameId\":\"{s}\"}}", .{target_id});
         // Calling contextCreated will assign a new Id to the context and send the contextCreated event
         bc.inspector.contextCreated(
             &isolated_world.executor.scope.?,

src/http/client.zig

@@ -29,6 +29,7 @@ const ArenaAllocator = std.heap.ArenaAllocator;
 const tls = @import("tls");
 const IO = @import("../runtime/loop.zig").IO;
 const Loop = @import("../runtime/loop.zig").Loop;
+const Notification = @import("../notification.zig").Notification;
 
 const log = std.log.scoped(.http_client);
 
@@ -44,6 +45,7 @@ const MAX_HEADER_LINE_LEN = 4096;
 // Thread-safe. Holds our root certificate, connection pool and state pool
 // Used to create Requests.
 pub const Client = struct {
+    req_id: usize,
     allocator: Allocator,
     state_pool: StatePool,
     http_proxy: ?Uri,
@@ -68,6 +70,7 @@ pub const Client = struct {
         errdefer connection_manager.deinit();
 
         return .{
+            .req_id = 0,
             .root_ca = root_ca,
             .allocator = allocator,
             .state_pool = state_pool,
@@ -96,6 +99,25 @@ pub const Client = struct {
 
         return Request.init(self, state, method, uri);
     }
+
+    pub fn requestFactory(self: *Client, notification: ?*Notification) RequestFactory {
+        return .{
+            .client = self,
+            .notification = notification,
+        };
+    }
+};
+
+// A factory for creating requests with a given set of options.
+pub const RequestFactory = struct {
+    client: *Client,
+    notification: ?*Notification,
+
+    pub fn create(self: RequestFactory, method: Request.Method, uri: *const Uri) !Request {
+        var req = try self.client.request(method, uri);
+        req.notification = self.notification;
+        return req;
+    }
 };
 
 // We assume most connections are going to end up in the IdleConnnection pool,
@@ -146,10 +168,12 @@ const Connection = struct {
 // (but request.deinit() should still be called to discard the request
 // before the `sendAsync` is called).
 pub const Request = struct {
+    id: usize,
+
     // The HTTP Method to use
     method: Method,
 
-    // The URI we're requested
+    // The URI we requested
     request_uri: *const Uri,
 
     // The URI that we're connecting to. Can be different than request_uri when
@@ -211,6 +235,16 @@ pub const Request = struct {
     // Whether or not we should verify that the host matches the certificate CN
     _tls_verify_host: bool,
 
+    // We only want to emit a start / complete notifications once per request.
+    // Because of things like redirects and error handling, it is possible for
+    // the notification functions to be called multiple times, so we guard them
+    // with these booleans
+    _notified_start: bool,
+    _notified_complete: bool,
+
+    // The notifier that we emit request notifications to, if any.
+    notification: ?*Notification,
+
     pub const Method = enum {
         GET,
         PUT,
@@ -230,12 +264,18 @@ pub const Request = struct {
 
     fn init(client: *Client, state: *State, method: Method, uri: *const Uri) !Request {
         const decomposed = try decomposeURL(client, uri);
+
+        const id = client.req_id + 1;
+        client.req_id = id;
+
         return .{
+            .id = id,
             .request_uri = uri,
             .connect_uri = decomposed.connect_uri,
             .body = null,
             .headers = .{},
             .method = method,
+            .notification = null,
             .arena = state.arena.allocator(),
             ._secure = decomposed.secure,
             ._connect_host = decomposed.connect_host,
@@ -247,6 +287,8 @@ pub const Request = struct {
             ._keepalive = false,
             ._redirect_count = 0,
             ._has_host_header = false,
+            ._notified_start = false,
+            ._notified_complete = false,
             ._connection_from_keepalive = false,
             ._tls_verify_host = client.tls_verify_host,
         };
@@ -525,6 +567,7 @@ pub const Request = struct {
         }
 
         try self.headers.append(arena, .{ .name = "User-Agent", .value = "Lightpanda/1.0" });
+        self.requestStarting();
     }
 
     // Sets up the request for redirecting.
@@ -641,6 +684,35 @@ pub const Request = struct {
         try writer.writeAll("\r\n");
         return buf[0..fbs.pos];
     }
+
+    fn requestStarting(self: *Request) void {
+        const notification = self.notification orelse return;
+        if (self._notified_start) {
+            return;
+        }
+        self._notified_start = true;
+        notification.dispatch(.http_request_start, &.{
+            .id = self.id,
+            .url = self.request_uri,
+            .method = self.method,
+            .headers = self.headers.items,
+            .has_body = self.body != null,
+        });
+    }
+
+    fn requestCompleted(self: *Request, response: ResponseHeader) void {
+        const notification = self.notification orelse return;
+        if (self._notified_complete) {
+            return;
+        }
+        self._notified_complete = true;
+        notification.dispatch(.http_request_complete, &.{
+            .id = self.id,
+            .url = self.request_uri,
+            .status = response.status,
+            .headers = response.headers.items,
+        });
+    }
 };
 
 // Handles asynchronous requests
@@ -823,6 +895,10 @@ fn AsyncHandler(comptime H: type, comptime L: type) type {
             }
 
             const status = self.conn.received(self.read_buf[0 .. self.read_pos + n]) catch |err| {
+                if (err == error.TlsAlertCloseNotify and self.state == .handshake and self.maybeRetryRequest()) {
+                    return;
+                }
+
                 self.handleError("data processing", err);
                 return;
             };
@@ -832,6 +908,7 @@ fn AsyncHandler(comptime H: type, comptime L: type) type {
                 .need_more => self.receive(),
                 .done => {
                     const redirect = self.redirect orelse {
+                        self.request.requestCompleted(self.reader.response);
                         self.deinit();
                         return;
                     };
@@ -1236,6 +1313,8 @@ const SyncHandler = struct {
                 var decompressor = std.compress.gzip.decompressor(compress_reader.reader());
                 try decompressor.decompress(body.writer(request.arena));
 
+                self.request.requestCompleted(reader.response);
+
                 return .{
                     .header = reader.response,
                     ._done = true,
@@ -1939,7 +2018,7 @@ pub const ResponseHeader = struct {
 // value in-place.
 // The value (and key) are both safe to mutate because they're cloned from
 // the byte stream by our arena.
-const Header = struct {
+pub const Header = struct {
     name: []const u8,
     value: []u8,
 };
@@ -2024,6 +2103,7 @@ pub const Response = struct {
                 return data;
             }
             if (self._done) {
+                self._request.requestCompleted(self.header);
                 return null;
             }
 

src/notification.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 
 const URL = @import("url.zig").URL;
 const page = @import("browser/page.zig");
+const http_client = @import("http/client.zig");
 
 const Allocator = std.mem.Allocator;
 
@@ -59,6 +60,8 @@ pub const Notification = struct {
         page_created: List = .{},
         page_navigate: List = .{},
         page_navigated: List = .{},
+        http_request_start: List = .{},
+        http_request_complete: List = .{},
         notification_created: List = .{},
     };
 
@@ -67,6 +70,8 @@ pub const Notification = struct {
         page_created: *page.Page,
         page_navigate: *const PageNavigate,
         page_navigated: *const PageNavigated,
+        http_request_start: *const RequestStart,
+        http_request_complete: *const RequestComplete,
         notification_created: *Notification,
     };
     const EventType = std.meta.FieldEnum(Events);
@@ -76,7 +81,7 @@ pub const Notification = struct {
     pub const PageNavigate = struct {
         timestamp: u32,
         url: *const URL,
-        reason: page.NavigateReason,
+        opts: page.NavigateOpts,
     };
 
     pub const PageNavigated = struct {
@@ -84,6 +89,21 @@ pub const Notification = struct {
         url: *const URL,
     };
 
+    pub const RequestStart = struct {
+        id: usize,
+        url: *const std.Uri,
+        method: http_client.Request.Method,
+        headers: []std.http.Header,
+        has_body: bool,
+    };
+
+    pub const RequestComplete = struct {
+        id: usize,
+        url: *const std.Uri,
+        status: u16,
+        headers: []http_client.Header,
+    };
+
     pub fn init(allocator: Allocator, parent: ?*Notification) !*Notification {
         // This is put on the heap because we want to raise a .notification_created
         // event, so that, something like Telemetry, can receive the
@@ -128,6 +148,7 @@ pub const Notification = struct {
             .list = list,
             .func = @ptrCast(func),
             .receiver = receiver,
+            .event = event,
             .struct_name = @typeName(@typeInfo(@TypeOf(receiver)).pointer.child),
         };
 
@@ -143,6 +164,30 @@ pub const Notification = struct {
         list.append(node);
     }
 
+    pub fn unregister(self: *Notification, comptime event: EventType, receiver: anytype) void {
+        var nodes = self.listeners.getPtr(@intFromPtr(receiver)) orelse return;
+
+        const node_pool = &self.node_pool;
+
+        var i: usize = 0;
+        while (i < nodes.items.len) {
+            const node = nodes.items[i];
+            if (node.data.event != event) {
+                i += 1;
+                continue;
+            }
+            node.data.list.remove(node);
+            node_pool.destroy(node);
+            _ = nodes.swapRemove(i);
+        }
+
+        if (nodes.items.len == 0) {
+            nodes.deinit(self.allocator);
+            const removed = self.listeners.remove(@intFromPtr(receiver));
+            std.debug.assert(removed == true);
+        }
+    }
+
     pub fn unregisterAll(self: *Notification, receiver: *anyopaque) void {
         const node_pool = &self.node_pool;
 
@@ -184,7 +229,7 @@ fn EventFunc(comptime event: Notification.EventType) type {
     return *const fn (*anyopaque, ArgType(event)) anyerror!void;
 }
 
-// An listener. This is 1 receiver, with its function, and the linked list
+// A listener. This is 1 receiver, with its function, and the linked list
 // node that goes in the appropriate EventListeners list.
 const Listener = struct {
     // the receiver of the event, i.e. the self parameter to `func`
@@ -196,6 +241,8 @@ const Listener = struct {
     // For logging slightly better error
     struct_name: []const u8,
 
+    event: Notification.EventType,
+
     // The event list this listener belongs to.
     // We need this in order to be able to remove the node from the list
     list: *List,
@@ -210,7 +257,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 4,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
 
     var tc = TestClient{};
@@ -219,7 +266,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 4,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     try testing.expectEqual(4, tc.page_navigate);
 
@@ -227,7 +274,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 10,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     try testing.expectEqual(4, tc.page_navigate);
 
@@ -236,7 +283,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 10,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     notifier.dispatch(.page_navigated, &.{ .timestamp = 6, .url = undefined });
     try testing.expectEqual(14, tc.page_navigate);
@@ -246,11 +293,40 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 100,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     notifier.dispatch(.page_navigated, &.{ .timestamp = 100, .url = undefined });
     try testing.expectEqual(14, tc.page_navigate);
     try testing.expectEqual(6, tc.page_navigated);
+
+    {
+        // unregister
+        try notifier.register(.page_navigate, &tc, TestClient.pageNavigate);
+        try notifier.register(.page_navigated, &tc, TestClient.pageNavigated);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(1006, tc.page_navigated);
+
+        notifier.unregister(.page_navigate, &tc);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(2006, tc.page_navigated);
+
+        notifier.unregister(.page_navigated, &tc);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(2006, tc.page_navigated);
+
+        // already unregistered, try anyways
+        notifier.unregister(.page_navigated, &tc);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(2006, tc.page_navigated);
+    }
 }
 
 const TestClient = struct {

src/testing.zig

@@ -418,6 +418,10 @@ pub const JsRunner = struct {
             .url = try self.url.toWebApi(arena),
         });
 
+        self.http_client = try HttpClient.init(arena, 1, .{
+            .tls_verify_host = false,
+        });
+
         self.state = .{
             .arena = arena,
             .loop = &self.loop,
@@ -425,16 +429,12 @@ pub const JsRunner = struct {
             .window = &self.window,
             .renderer = &self.renderer,
             .cookie_jar = &self.cookie_jar,
-            .http_client = &self.http_client,
+            .request_factory = self.http_client.requestFactory(null),
         };
 
         self.storage_shelf = storage.Shelf.init(arena);
         self.window.setStorageShelf(&self.storage_shelf);
 
-        self.http_client = try HttpClient.init(arena, 1, .{
-            .tls_verify_host = false,
-        });
-
         self.executor = try self.env.newExecutionWorld();
         errdefer self.executor.deinit();