HTTP request notification

- Add 2 internal notifications
  1 - http_request_start
  2 - http_request_complete

- When Network.enable CDP message is received, browser context registers for
  these 2 events (when Network.disable is called, it unregisters)

- On http_request_start, CDP will emit a Network.requestWillBeSent message.
  This _does not_ include all the fields, but what we have appears to be enough
  for puppeteer.waitForNetworkIdle.

- On http_request_complete, CDP will emit a Network.responseReceived message.
  This _does not_ include all the fields, bu what we have appears to be enough
  for puppeteer.waitForNetworkIdle.

We currently don't emit any other new events, including any network-specific
lifecycleEvent (i.e. Chrome will emit an networkIdle and networkAlmostIdle).

To support this, the following other things were done:
- CDP now has a `notification_arena` which is re-used between browser contexts.
  Normally, CDP code runs based on a "cmd" which has its own message_arena, but
  these notifications happen out-of-band, so we needed a new arena which is
  valid for handling 1 notification.

- HTTP Client is notification-aware. The SessionState no longer includes the
  *http.Client directly. It instead includes an http.RequestFactory which is
  the combination fo the client + a specific configuration (i.e. *Notification).
  This ensures that all requests made from that factory have the same settings.

- However, despite the above, _some_ requests do not appear to emit CDP events,
  such as loading a <script src="X">. So the page still deals directly with the
  *http.Client.

- Playwright and Puppeteer (but Playwright in particular) are very sensitive to
  event ordering. These new events have introduced additional sensitivity.
  The result sent to Page.navigate had to be moved to inside the navigate event
  handler, which meant passing some cdp-specific data (the input.id) into the
  NavigateOpts. This is the only way I found to keep both happy - the sequence
  of events is closer (but still pretty far) from what Chrome does.

src/browser/env.zig

@@ -7,7 +7,7 @@ const storage = @import("storage/storage.zig");
 const generate = @import("../runtime/generate.zig");
 const Renderer = @import("renderer.zig").Renderer;
 const Loop = @import("../runtime/loop.zig").Loop;
-const HttpClient = @import("../http/client.zig").Client;
+const RequestFactory = @import("../http/client.zig").RequestFactory;
 
 const WebApis = struct {
     // Wrapped like this for debug ergonomics.
@@ -54,8 +54,8 @@ pub const SessionState = struct {
     window: *Window,
     renderer: *Renderer,
     arena: std.mem.Allocator,
-    http_client: *HttpClient,
     cookie_jar: *storage.CookieJar,
+    request_factory: RequestFactory,
 
     // dangerous, but set by the JS framework
     // shorter-lived than the arena above, which

src/browser/page.zig

@@ -98,7 +98,7 @@ pub const Page = struct {
                 .renderer = &self.renderer,
                 .loop = browser.app.loop,
                 .cookie_jar = &session.cookie_jar,
-                .http_client = browser.http_client,
+                .request_factory = browser.http_client.requestFactory(browser.notification),
             },
             .scope = try session.executor.startScope(&self.window, &self.state, self, true),
             .module_map = .empty,
@@ -174,6 +174,7 @@ pub const Page = struct {
     pub fn navigate(self: *Page, request_url: URL, opts: NavigateOpts) !void {
         const arena = self.arena;
         const session = self.session;
+        const notification = session.browser.notification;
 
         log.debug("starting GET {s}", .{request_url});
 
@@ -195,10 +196,11 @@ pub const Page = struct {
         // load the data
         var request = try self.newHTTPRequest(.GET, &self.url, .{ .navigation = true });
         defer request.deinit();
+        request.notification = notification;
 
-        session.browser.notification.dispatch(.page_navigate, &.{
+        notification.dispatch(.page_navigate, &.{
+            .opts = opts,
             .url = &self.url,
-            .reason = opts.reason,
             .timestamp = timestamp(),
         });
 
@@ -238,7 +240,7 @@ pub const Page = struct {
             self.raw_data = arr.items;
         }
 
-        session.browser.notification.dispatch(.page_navigated, &.{
+        notification.dispatch(.page_navigated, &.{
             .url = &self.url,
             .timestamp = timestamp(),
         });
@@ -464,7 +466,9 @@ pub const Page = struct {
     }
 
     fn newHTTPRequest(self: *const Page, method: http.Request.Method, url: *const URL, opts: storage.cookie.LookupOpts) !http.Request {
-        var request = try self.state.http_client.request(method, &url.uri);
+        // Don't use the state's request_factory here, since requests made by the
+        // page (i.e. to load <scripts>) should not generate notifications.
+        var request = try self.session.browser.http_client.request(method, &url.uri);
         errdefer request.deinit();
 
         var arr: std.ArrayListUnmanaged(u8) = .{};
@@ -661,7 +665,8 @@ pub const NavigateReason = enum {
     address_bar,
 };
 
-const NavigateOpts = struct {
+pub const NavigateOpts = struct {
+    cdp_id: ?i64 = null,
     reason: NavigateReason = .address_bar,
 };
 

src/browser/xhr/xhr.zig

@@ -80,7 +80,6 @@ const XMLHttpRequestBodyInit = union(enum) {
 pub const XMLHttpRequest = struct {
     proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
     arena: Allocator,
-    client: *http.Client,
     request: ?http.Request = null,
 
     priv_state: PrivState = .new,
@@ -252,7 +251,6 @@ pub const XMLHttpRequest = struct {
             .state = .unsent,
             .url = null,
             .origin_url = session_state.url,
-            .client = session_state.http_client,
             .cookie_jar = session_state.cookie_jar,
         };
     }
@@ -420,7 +418,7 @@ pub const XMLHttpRequest = struct {
         self.send_flag = true;
         self.priv_state = .open;
 
-        self.request = try self.client.request(self.method, &self.url.?.uri);
+        self.request = try session_state.request_factory.create(self.method, &self.url.?.uri);
         var request = &self.request.?;
         errdefer request.deinit();
 

src/cdp/cdp.zig

@@ -69,6 +69,9 @@ pub fn CDPT(comptime TypeProvider: type) type {
         // 1 message at a time.
         message_arena: std.heap.ArenaAllocator,
 
+        // Used for processing notifications within a browser context.
+        notification_arena: std.heap.ArenaAllocator,
+
         const Self = @This();
 
         pub fn init(app: *App, client: TypeProvider.Client) !Self {
@@ -82,6 +85,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 .allocator = allocator,
                 .browser_context = null,
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
+                .notification_arena = std.heap.ArenaAllocator.init(allocator),
             };
         }
 
@@ -91,6 +95,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
             }
             self.browser.deinit();
             self.message_arena.deinit();
+            self.notification_arena.deinit();
         }
 
         pub fn handleMessage(self: *Self, msg: []const u8) bool {
@@ -259,7 +264,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
             });
         }
 
-        fn sendJSON(self: *Self, message: anytype) !void {
+        pub fn sendJSON(self: *Self, message: anytype) !void {
             return self.client.sendJSON(message, .{
                 .emit_null_optional_fields = false,
             });
@@ -283,6 +288,12 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         // Points to the session arena
         arena: Allocator,
 
+        // From the parent's notification_arena.allocator(). Most of the CDP
+        // code paths deal with a cmd which has its own arena (from the
+        // message_arena). But notifications happen outside of the typical CDP
+        // request->response, and thus don't have a cmd and don't have an arena.
+        notification_arena: Allocator,
+
         // Maps to our Page. (There are other types of targets, but we only
         // deal with "pages" for now). Since we only allow 1 open page at a
         // time, we only have 1 target_id.
@@ -336,6 +347,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
                 .node_search_list = undefined,
                 .isolated_world = null,
                 .inspector = inspector,
+                .notification_arena = cdp.notification_arena.allocator(),
             };
             self.node_search_list = Node.Search.List.init(allocator, &self.node_registry);
             errdefer self.deinit();
@@ -397,6 +409,16 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             return if (raw_url.len == 0) null else raw_url;
         }
 
+        pub fn networkEnable(self: *Self) !void {
+            try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart);
+            try self.cdp.browser.notification.register(.http_request_complete, self, onHttpRequestComplete);
+        }
+
+        pub fn networkDisable(self: *Self) void {
+            self.cdp.browser.notification.unregister(.http_request_start, self);
+            self.cdp.browser.notification.unregister(.http_request_complete, self);
+        }
+
         pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             return @import("domains/page.zig").pageRemove(self);
@@ -409,7 +431,8 @@ pub fn BrowserContext(comptime CDP_T: type) type {
 
         pub fn onPageNavigate(ctx: *anyopaque, data: *const Notification.PageNavigate) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
-            return @import("domains/page.zig").pageNavigate(self, data);
+            defer self.resetNotificationArena();
+            return @import("domains/page.zig").pageNavigate(self.notification_arena, self, data);
         }
 
         pub fn onPageNavigated(ctx: *anyopaque, data: *const Notification.PageNavigated) !void {
@@ -417,6 +440,22 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             return @import("domains/page.zig").pageNavigated(self, data);
         }
 
+        pub fn onHttpRequestStart(ctx: *anyopaque, data: *const Notification.RequestStart) !void {
+            const self: *Self = @alignCast(@ptrCast(ctx));
+            defer self.resetNotificationArena();
+            return @import("domains/network.zig").httpRequestStart(self.notification_arena, self, data);
+        }
+
+        pub fn onHttpRequestComplete(ctx: *anyopaque, data: *const Notification.RequestComplete) !void {
+            const self: *Self = @alignCast(@ptrCast(ctx));
+            defer self.resetNotificationArena();
+            return @import("domains/network.zig").httpRequestComplete(self.notification_arena, self, data);
+        }
+
+        fn resetNotificationArena(self: *Self) void {
+            defer _ = self.cdp.notification_arena.reset(.{ .retain_with_limit = 1024 * 64 });
+        }
+
         pub fn callInspector(self: *const Self, msg: []const u8) void {
             self.inspector.send(msg);
             // force running micro tasks after send input to the inspector.

src/cdp/domains/network.zig

@@ -17,15 +17,130 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 const std = @import("std");
+const Notification = @import("../../notification.zig").Notification;
+
+const Allocator = std.mem.Allocator;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
         enable,
+        disable,
         setCacheDisabled,
     }, cmd.input.action) orelse return error.UnknownMethod;
 
     switch (action) {
-        .enable => return cmd.sendResult(null, .{}),
+        .enable => return enable(cmd),
+        .disable => return disable(cmd),
         .setCacheDisabled => return cmd.sendResult(null, .{}),
     }
 }
+
+fn enable(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    try bc.networkEnable();
+    return cmd.sendResult(null, .{});
+}
+
+fn disable(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    bc.networkDisable();
+    return cmd.sendResult(null, .{});
+}
+
+pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notification.RequestStart) !void {
+    // Isn't possible to do a network request within a Browser (which our
+    // notification is tied to), without a page.
+    std.debug.assert(bc.session.page != null);
+
+    var cdp = bc.cdp;
+
+    // all unreachable because we _have_ to have a page.
+    const session_id = bc.session_id orelse unreachable;
+    const target_id = bc.target_id orelse unreachable;
+    const page = bc.session.currentPage() orelse unreachable;
+
+    const document_url = try urlToString(arena, &page.url.uri, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    const request_url = try urlToString(arena, request.url, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    const request_fragment = try urlToString(arena, request.url, .{
+        .fragment = true,
+    });
+
+    var headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
+    try headers.ensureTotalCapacity(arena, request.headers.len);
+    for (request.headers) |header| {
+        headers.putAssumeCapacity(header.name, header.value);
+    }
+
+    // We're missing a bunch of fields, but, for now, this seems like enough
+    try cdp.sendEvent("Network.requestWillBeSent", .{
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}),
+        .frameId = target_id,
+        .loaderId = bc.loader_id,
+        .documentUrl = document_url,
+        .request = .{
+            .url = request_url,
+            .urlFragment = request_fragment,
+            .method = @tagName(request.method),
+            .hasPostData = request.has_body,
+            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
+        },
+    }, .{ .session_id = session_id });
+}
+
+pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notification.RequestComplete) !void {
+    // Isn't possible to do a network request within a Browser (which our
+    // notification is tied to), without a page.
+    std.debug.assert(bc.session.page != null);
+
+    var cdp = bc.cdp;
+
+    // all unreachable because we _have_ to have a page.
+    const session_id = bc.session_id orelse unreachable;
+    const target_id = bc.target_id orelse unreachable;
+
+    const url = try urlToString(arena, request.url, .{
+        .scheme = true,
+        .authentication = true,
+        .authority = true,
+        .path = true,
+        .query = true,
+    });
+
+    var headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
+    try headers.ensureTotalCapacity(arena, request.headers.len);
+    for (request.headers) |header| {
+        headers.putAssumeCapacity(header.name, header.value);
+    }
+
+    // We're missing a bunch of fields, but, for now, this seems like enough
+    try cdp.sendEvent("Network.responseReceived", .{
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}),
+        .frameId = target_id,
+        .loaderId = bc.loader_id,
+        .response = .{
+            .url = url,
+            .status = request.status,
+            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
+        },
+    }, .{ .session_id = session_id });
+}
+
+fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 {
+    var buf: std.ArrayListUnmanaged(u8) = .empty;
+    try url.writeToStream(opts, buf.writer(arena));
+    return buf.items;
+}

src/cdp/domains/page.zig

@@ -21,6 +21,8 @@ const URL = @import("../../url.zig").URL;
 const Page = @import("../../browser/page.zig").Page;
 const Notification = @import("../../notification.zig").Notification;
 
+const Allocator = std.mem.Allocator;
+
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
         enable,
@@ -137,7 +139,7 @@ fn navigate(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
 
     // didn't create?
-    const target_id = bc.target_id orelse return error.TargetIdNotLoaded;
+    // const target_id = bc.target_id orelse return error.TargetIdNotLoaded;
 
     // didn't attach?
     if (bc.session_id == null) {
@@ -148,17 +150,14 @@ fn navigate(cmd: anytype) !void {
 
     var page = bc.session.currentPage() orelse return error.PageNotLoaded;
     bc.loader_id = bc.cdp.loader_id_gen.next();
-    try cmd.sendResult(.{
-        .frameId = target_id,
-        .loaderId = bc.loader_id,
-    }, .{});
 
     try page.navigate(url, .{
         .reason = .address_bar,
+        .cdp_id = cmd.input.id,
     });
 }
 
-pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void {
+pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.PageNavigate) !void {
     // I don't think it's possible that we get these notifications and don't
     // have these things setup.
     std.debug.assert(bc.session.page != null);
@@ -170,7 +169,8 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
 
     bc.reset();
 
-    if (event.reason == .anchor) {
+    const is_anchor = event.opts.reason == .anchor;
+    if (is_anchor) {
         try cdp.sendEvent("Page.frameScheduledNavigation", .{
             .frameId = target_id,
             .delay = 0,
@@ -199,6 +199,22 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
         .frameId = target_id,
     }, .{ .session_id = session_id });
 
+    // Drivers are sensitive to the order of events. Some more than others.
+    // The result for the Page.navigate seems like it _must_ come after
+    // the frameStartedLoading, but before any lifecycleEvent. So we
+    // unfortunately have to put the input_id ito the NavigateOpts which gets
+    // passed back into the notification.
+    if (event.opts.cdp_id) |input_id| {
+        try cdp.sendJSON(.{
+            .id = input_id,
+            .result = .{
+                .frameId = target_id,
+                .loaderId = loader_id,
+            },
+            .sessionId = session_id,
+        });
+    }
+
     if (bc.page_life_cycle_events) {
         try cdp.sendEvent("Page.lifecycleEvent", LifecycleEvent{
             .name = "init",
@@ -208,7 +224,7 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
         }, .{ .session_id = session_id });
     }
 
-    if (event.reason == .anchor) {
+    if (is_anchor) {
         try cdp.sendEvent("Page.frameClearedScheduledNavigation", .{
             .frameId = target_id,
         }, .{ .session_id = session_id });
@@ -219,21 +235,19 @@ pub fn pageNavigate(bc: anytype, event: *const Notification.PageNavigate) !void
     // The client will expect us to send new contextCreated events, such that the client has new id's for the active contexts.
     try cdp.sendEvent("Runtime.executionContextsCleared", null, .{ .session_id = session_id });
 
-    var buffer: [512]u8 = undefined;
     {
-        var fba = std.heap.FixedBufferAllocator.init(&buffer);
         const page = bc.session.currentPage() orelse return error.PageNotLoaded;
-        const aux_data = try std.fmt.allocPrint(fba.allocator(), "{{\"isDefault\":true,\"type\":\"default\",\"frameId\":\"{s}\"}}", .{target_id});
+        const aux_data = try std.fmt.allocPrint(arena, "{{\"isDefault\":true,\"type\":\"default\",\"frameId\":\"{s}\"}}", .{target_id});
         bc.inspector.contextCreated(
             page.scope,
             "",
-            try page.origin(fba.allocator()),
+            try page.origin(arena),
             aux_data,
             true,
         );
     }
     if (bc.isolated_world) |*isolated_world| {
-        const aux_json = try std.fmt.bufPrint(&buffer, "{{\"isDefault\":false,\"type\":\"isolated\",\"frameId\":\"{s}\"}}", .{target_id});
+        const aux_json = try std.fmt.allocPrint(arena, "{{\"isDefault\":false,\"type\":\"isolated\",\"frameId\":\"{s}\"}}", .{target_id});
         // Calling contextCreated will assign a new Id to the context and send the contextCreated event
         bc.inspector.contextCreated(
             &isolated_world.executor.scope.?,

src/http/client.zig

@@ -29,6 +29,7 @@ const ArenaAllocator = std.heap.ArenaAllocator;
 const tls = @import("tls");
 const IO = @import("../runtime/loop.zig").IO;
 const Loop = @import("../runtime/loop.zig").Loop;
+const Notification = @import("../notification.zig").Notification;
 
 const log = std.log.scoped(.http_client);
 
@@ -44,6 +45,7 @@ const MAX_HEADER_LINE_LEN = 4096;
 // Thread-safe. Holds our root certificate, connection pool and state pool
 // Used to create Requests.
 pub const Client = struct {
+    req_id: usize,
     allocator: Allocator,
     state_pool: StatePool,
     http_proxy: ?Uri,
@@ -68,6 +70,7 @@ pub const Client = struct {
         errdefer connection_manager.deinit();
 
         return .{
+            .req_id = 0,
             .root_ca = root_ca,
             .allocator = allocator,
             .state_pool = state_pool,
@@ -96,6 +99,25 @@ pub const Client = struct {
 
         return Request.init(self, state, method, uri);
     }
+
+    pub fn requestFactory(self: *Client, notification: ?*Notification) RequestFactory {
+        return .{
+            .client = self,
+            .notification = notification,
+        };
+    }
+};
+
+// A factory for creating requests with a given set of options.
+pub const RequestFactory = struct {
+    client: *Client,
+    notification: ?*Notification,
+
+    pub fn create(self: RequestFactory, method: Request.Method, uri: *const Uri) !Request {
+        var req = try self.client.request(method, uri);
+        req.notification = self.notification;
+        return req;
+    }
 };
 
 // We assume most connections are going to end up in the IdleConnnection pool,
@@ -146,10 +168,12 @@ const Connection = struct {
 // (but request.deinit() should still be called to discard the request
 // before the `sendAsync` is called).
 pub const Request = struct {
+    id: usize,
+
     // The HTTP Method to use
     method: Method,
 
-    // The URI we're requested
+    // The URI we requested
     request_uri: *const Uri,
 
     // The URI that we're connecting to. Can be different than request_uri when
@@ -211,6 +235,16 @@ pub const Request = struct {
     // Whether or not we should verify that the host matches the certificate CN
     _tls_verify_host: bool,
 
+    // We only want to emit a start / complete notifications once per request.
+    // Because of things like redirects and error handling, it is possible for
+    // the notification functions to be called multiple times, so we guard them
+    // with these booleans
+    _notified_start: bool,
+    _notified_complete: bool,
+
+    // The notifier that we emit request notifications to, if any.
+    notification: ?*Notification,
+
     pub const Method = enum {
         GET,
         PUT,
@@ -230,12 +264,18 @@ pub const Request = struct {
 
     fn init(client: *Client, state: *State, method: Method, uri: *const Uri) !Request {
         const decomposed = try decomposeURL(client, uri);
+
+        const id = client.req_id + 1;
+        client.req_id = id;
+
         return .{
+            .id = id,
             .request_uri = uri,
             .connect_uri = decomposed.connect_uri,
             .body = null,
             .headers = .{},
             .method = method,
+            .notification = null,
             .arena = state.arena.allocator(),
             ._secure = decomposed.secure,
             ._connect_host = decomposed.connect_host,
@@ -247,6 +287,8 @@ pub const Request = struct {
             ._keepalive = false,
             ._redirect_count = 0,
             ._has_host_header = false,
+            ._notified_start = false,
+            ._notified_complete = false,
             ._connection_from_keepalive = false,
             ._tls_verify_host = client.tls_verify_host,
         };
@@ -525,6 +567,7 @@ pub const Request = struct {
         }
 
         try self.headers.append(arena, .{ .name = "User-Agent", .value = "Lightpanda/1.0" });
+        self.requestStarting();
     }
 
     // Sets up the request for redirecting.
@@ -641,6 +684,35 @@ pub const Request = struct {
         try writer.writeAll("\r\n");
         return buf[0..fbs.pos];
     }
+
+    fn requestStarting(self: *Request) void {
+        const notification = self.notification orelse return;
+        if (self._notified_start) {
+            return;
+        }
+        self._notified_start = true;
+        notification.dispatch(.http_request_start, &.{
+            .id = self.id,
+            .url = self.request_uri,
+            .method = self.method,
+            .headers = self.headers.items,
+            .has_body = self.body != null,
+        });
+    }
+
+    fn requestCompleted(self: *Request, response: ResponseHeader) void {
+        const notification = self.notification orelse return;
+        if (self._notified_complete) {
+            return;
+        }
+        self._notified_complete = true;
+        notification.dispatch(.http_request_complete, &.{
+            .id = self.id,
+            .url = self.request_uri,
+            .status = response.status,
+            .headers = response.headers.items,
+        });
+    }
 };
 
 // Handles asynchronous requests
@@ -832,6 +904,7 @@ fn AsyncHandler(comptime H: type, comptime L: type) type {
                 .need_more => self.receive(),
                 .done => {
                     const redirect = self.redirect orelse {
+                        self.request.requestCompleted(self.reader.response);
                         self.deinit();
                         return;
                     };
@@ -1236,6 +1309,8 @@ const SyncHandler = struct {
                 var decompressor = std.compress.gzip.decompressor(compress_reader.reader());
                 try decompressor.decompress(body.writer(request.arena));
 
+                self.request.requestCompleted(reader.response);
+
                 return .{
                     .header = reader.response,
                     ._done = true,
@@ -1939,7 +2014,7 @@ pub const ResponseHeader = struct {
 // value in-place.
 // The value (and key) are both safe to mutate because they're cloned from
 // the byte stream by our arena.
-const Header = struct {
+pub const Header = struct {
     name: []const u8,
     value: []u8,
 };
@@ -2024,6 +2099,7 @@ pub const Response = struct {
                 return data;
             }
             if (self._done) {
+                self._request.requestCompleted(self.header);
                 return null;
             }
 

src/notification.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 
 const URL = @import("url.zig").URL;
 const page = @import("browser/page.zig");
+const http_client = @import("http/client.zig");
 
 const Allocator = std.mem.Allocator;
 
@@ -59,6 +60,8 @@ pub const Notification = struct {
         page_created: List = .{},
         page_navigate: List = .{},
         page_navigated: List = .{},
+        http_request_start: List = .{},
+        http_request_complete: List = .{},
         notification_created: List = .{},
     };
 
@@ -67,6 +70,8 @@ pub const Notification = struct {
         page_created: *page.Page,
         page_navigate: *const PageNavigate,
         page_navigated: *const PageNavigated,
+        http_request_start: *const RequestStart,
+        http_request_complete: *const RequestComplete,
         notification_created: *Notification,
     };
     const EventType = std.meta.FieldEnum(Events);
@@ -76,7 +81,7 @@ pub const Notification = struct {
     pub const PageNavigate = struct {
         timestamp: u32,
         url: *const URL,
-        reason: page.NavigateReason,
+        opts: page.NavigateOpts,
     };
 
     pub const PageNavigated = struct {
@@ -84,6 +89,21 @@ pub const Notification = struct {
         url: *const URL,
     };
 
+    pub const RequestStart = struct {
+        id: usize,
+        url: *const std.Uri,
+        method: http_client.Request.Method,
+        headers: []std.http.Header,
+        has_body: bool,
+    };
+
+    pub const RequestComplete = struct {
+        id: usize,
+        url: *const std.Uri,
+        status: u16,
+        headers: []http_client.Header,
+    };
+
     pub fn init(allocator: Allocator, parent: ?*Notification) !*Notification {
         // This is put on the heap because we want to raise a .notification_created
         // event, so that, something like Telemetry, can receive the
@@ -128,6 +148,7 @@ pub const Notification = struct {
             .list = list,
             .func = @ptrCast(func),
             .receiver = receiver,
+            .event = event,
             .struct_name = @typeName(@typeInfo(@TypeOf(receiver)).pointer.child),
         };
 
@@ -143,6 +164,30 @@ pub const Notification = struct {
         list.append(node);
     }
 
+    pub fn unregister(self: *Notification, comptime event: EventType, receiver: anytype) void {
+        var nodes = self.listeners.getPtr(@intFromPtr(receiver)) orelse return;
+
+        const node_pool = &self.node_pool;
+
+        var i: usize = 0;
+        while (i < nodes.items.len) {
+            const node = nodes.items[i];
+            if (node.data.event != event) {
+                i += 1;
+                continue;
+            }
+            node.data.list.remove(node);
+            node_pool.destroy(node);
+            _ = nodes.swapRemove(i);
+        }
+
+        if (nodes.items.len == 0) {
+            nodes.deinit(self.allocator);
+            const removed = self.listeners.remove(@intFromPtr(receiver));
+            std.debug.assert(removed == true);
+        }
+    }
+
     pub fn unregisterAll(self: *Notification, receiver: *anyopaque) void {
         const node_pool = &self.node_pool;
 
@@ -184,7 +229,7 @@ fn EventFunc(comptime event: Notification.EventType) type {
     return *const fn (*anyopaque, ArgType(event)) anyerror!void;
 }
 
-// An listener. This is 1 receiver, with its function, and the linked list
+// A listener. This is 1 receiver, with its function, and the linked list
 // node that goes in the appropriate EventListeners list.
 const Listener = struct {
     // the receiver of the event, i.e. the self parameter to `func`
@@ -196,6 +241,8 @@ const Listener = struct {
     // For logging slightly better error
     struct_name: []const u8,
 
+    event: Notification.EventType,
+
     // The event list this listener belongs to.
     // We need this in order to be able to remove the node from the list
     list: *List,
@@ -210,7 +257,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 4,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
 
     var tc = TestClient{};
@@ -219,7 +266,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 4,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     try testing.expectEqual(4, tc.page_navigate);
 
@@ -227,7 +274,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 10,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     try testing.expectEqual(4, tc.page_navigate);
 
@@ -236,7 +283,7 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 10,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     notifier.dispatch(.page_navigated, &.{ .timestamp = 6, .url = undefined });
     try testing.expectEqual(14, tc.page_navigate);
@@ -246,11 +293,40 @@ test "Notification" {
     notifier.dispatch(.page_navigate, &.{
         .timestamp = 100,
         .url = undefined,
-        .reason = undefined,
+        .opts = .{},
     });
     notifier.dispatch(.page_navigated, &.{ .timestamp = 100, .url = undefined });
     try testing.expectEqual(14, tc.page_navigate);
     try testing.expectEqual(6, tc.page_navigated);
+
+    {
+        // unregister
+        try notifier.register(.page_navigate, &tc, TestClient.pageNavigate);
+        try notifier.register(.page_navigated, &tc, TestClient.pageNavigated);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(1006, tc.page_navigated);
+
+        notifier.unregister(.page_navigate, &tc);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(2006, tc.page_navigated);
+
+        notifier.unregister(.page_navigated, &tc);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(2006, tc.page_navigated);
+
+        // already unregistered, try anyways
+        notifier.unregister(.page_navigated, &tc);
+        notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
+        notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
+        try testing.expectEqual(114, tc.page_navigate);
+        try testing.expectEqual(2006, tc.page_navigated);
+    }
 }
 
 const TestClient = struct {

src/testing.zig

@@ -418,6 +418,10 @@ pub const JsRunner = struct {
             .url = try self.url.toWebApi(arena),
         });
 
+        self.http_client = try HttpClient.init(arena, 1, .{
+            .tls_verify_host = false,
+        });
+
         self.state = .{
             .arena = arena,
             .loop = &self.loop,
@@ -425,16 +429,12 @@ pub const JsRunner = struct {
             .window = &self.window,
             .renderer = &self.renderer,
             .cookie_jar = &self.cookie_jar,
-            .http_client = &self.http_client,
+            .request_factory = self.http_client.requestFactory(null),
         };
 
         self.storage_shelf = storage.Shelf.init(arena);
         self.window.setStorageShelf(&self.storage_shelf);
 
-        self.http_client = try HttpClient.init(arena, 1, .{
-            .tls_verify_host = false,
-        });
-
         self.executor = try self.env.newExecutionWorld();
         errdefer self.executor.deinit();