HTTP request notification

- Add 2 internal notifications
  1 - http_request_start
  2 - http_request_complete

- When Network.enable CDP message is received, browser context registers for
  these 2 events (when Network.disable is called, it unregisters)

- On http_request_start, CDP will emit a Network.requestWillBeSent message.
  This _does not_ include all the fields, but what we have appears to be enough
  for puppeteer.waitForNetworkIdle.

- On http_request_complete, CDP will emit a Network.responseReceived message.
  This _does not_ include all the fields, bu what we have appears to be enough
  for puppeteer.waitForNetworkIdle.

We currently don't emit any other new events, including any network-specific
lifecycleEvent (i.e. Chrome will emit an networkIdle and networkAlmostIdle).

To support this, the following other things were done:
- CDP now has a `notification_arena` which is re-used between browser contexts.
  Normally, CDP code runs based on a "cmd" which has its own message_arena, but
  these notifications happen out-of-band, so we needed a new arena which is
  valid for handling 1 notification.

- HTTP Client is notification-aware. The SessionState no longer includes the
  *http.Client directly. It instead includes an http.RequestFactory which is
  the combination fo the client + a specific configuration (i.e. *Notification).
  This ensures that all requests made from that factory have the same settings.

- However, despite the above, _some_ requests do not appear to emit CDP events,
  such as loading a <script src="X">. So the page still deals directly with the
  *http.Client.

- Playwright and Puppeteer (but Playwright in particular) are very sensitive to
  event ordering. These new events have introduced additional sensitivity.
  The result sent to Page.navigate had to be moved to inside the navigate event
  handler, which meant passing some cdp-specific data (the input.id) into the
  NavigateOpts. This is the only way I found to keep both happy - the sequence
  of events is closer (but still pretty far) from what Chrome does.
This commit is contained in:
Karl Seguin
2025-05-21 17:35:53 +08:00
parent d262f017c5
commit 94a30b2167
9 changed files with 364 additions and 41 deletions

View File

@@ -2,6 +2,7 @@ const std = @import("std");
const URL = @import("url.zig").URL;
const page = @import("browser/page.zig");
const http_client = @import("http/client.zig");
const Allocator = std.mem.Allocator;
@@ -59,6 +60,8 @@ pub const Notification = struct {
page_created: List = .{},
page_navigate: List = .{},
page_navigated: List = .{},
http_request_start: List = .{},
http_request_complete: List = .{},
notification_created: List = .{},
};
@@ -67,6 +70,8 @@ pub const Notification = struct {
page_created: *page.Page,
page_navigate: *const PageNavigate,
page_navigated: *const PageNavigated,
http_request_start: *const RequestStart,
http_request_complete: *const RequestComplete,
notification_created: *Notification,
};
const EventType = std.meta.FieldEnum(Events);
@@ -76,7 +81,7 @@ pub const Notification = struct {
pub const PageNavigate = struct {
timestamp: u32,
url: *const URL,
reason: page.NavigateReason,
opts: page.NavigateOpts,
};
pub const PageNavigated = struct {
@@ -84,6 +89,21 @@ pub const Notification = struct {
url: *const URL,
};
pub const RequestStart = struct {
id: usize,
url: *const std.Uri,
method: http_client.Request.Method,
headers: []std.http.Header,
has_body: bool,
};
pub const RequestComplete = struct {
id: usize,
url: *const std.Uri,
status: u16,
headers: []http_client.Header,
};
pub fn init(allocator: Allocator, parent: ?*Notification) !*Notification {
// This is put on the heap because we want to raise a .notification_created
// event, so that, something like Telemetry, can receive the
@@ -128,6 +148,7 @@ pub const Notification = struct {
.list = list,
.func = @ptrCast(func),
.receiver = receiver,
.event = event,
.struct_name = @typeName(@typeInfo(@TypeOf(receiver)).pointer.child),
};
@@ -143,6 +164,30 @@ pub const Notification = struct {
list.append(node);
}
pub fn unregister(self: *Notification, comptime event: EventType, receiver: anytype) void {
var nodes = self.listeners.getPtr(@intFromPtr(receiver)) orelse return;
const node_pool = &self.node_pool;
var i: usize = 0;
while (i < nodes.items.len) {
const node = nodes.items[i];
if (node.data.event != event) {
i += 1;
continue;
}
node.data.list.remove(node);
node_pool.destroy(node);
_ = nodes.swapRemove(i);
}
if (nodes.items.len == 0) {
nodes.deinit(self.allocator);
const removed = self.listeners.remove(@intFromPtr(receiver));
std.debug.assert(removed == true);
}
}
pub fn unregisterAll(self: *Notification, receiver: *anyopaque) void {
const node_pool = &self.node_pool;
@@ -184,7 +229,7 @@ fn EventFunc(comptime event: Notification.EventType) type {
return *const fn (*anyopaque, ArgType(event)) anyerror!void;
}
// An listener. This is 1 receiver, with its function, and the linked list
// A listener. This is 1 receiver, with its function, and the linked list
// node that goes in the appropriate EventListeners list.
const Listener = struct {
// the receiver of the event, i.e. the self parameter to `func`
@@ -196,6 +241,8 @@ const Listener = struct {
// For logging slightly better error
struct_name: []const u8,
event: Notification.EventType,
// The event list this listener belongs to.
// We need this in order to be able to remove the node from the list
list: *List,
@@ -210,7 +257,7 @@ test "Notification" {
notifier.dispatch(.page_navigate, &.{
.timestamp = 4,
.url = undefined,
.reason = undefined,
.opts = .{},
});
var tc = TestClient{};
@@ -219,7 +266,7 @@ test "Notification" {
notifier.dispatch(.page_navigate, &.{
.timestamp = 4,
.url = undefined,
.reason = undefined,
.opts = .{},
});
try testing.expectEqual(4, tc.page_navigate);
@@ -227,7 +274,7 @@ test "Notification" {
notifier.dispatch(.page_navigate, &.{
.timestamp = 10,
.url = undefined,
.reason = undefined,
.opts = .{},
});
try testing.expectEqual(4, tc.page_navigate);
@@ -236,7 +283,7 @@ test "Notification" {
notifier.dispatch(.page_navigate, &.{
.timestamp = 10,
.url = undefined,
.reason = undefined,
.opts = .{},
});
notifier.dispatch(.page_navigated, &.{ .timestamp = 6, .url = undefined });
try testing.expectEqual(14, tc.page_navigate);
@@ -246,11 +293,40 @@ test "Notification" {
notifier.dispatch(.page_navigate, &.{
.timestamp = 100,
.url = undefined,
.reason = undefined,
.opts = .{},
});
notifier.dispatch(.page_navigated, &.{ .timestamp = 100, .url = undefined });
try testing.expectEqual(14, tc.page_navigate);
try testing.expectEqual(6, tc.page_navigated);
{
// unregister
try notifier.register(.page_navigate, &tc, TestClient.pageNavigate);
try notifier.register(.page_navigated, &tc, TestClient.pageNavigated);
notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
try testing.expectEqual(114, tc.page_navigate);
try testing.expectEqual(1006, tc.page_navigated);
notifier.unregister(.page_navigate, &tc);
notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
try testing.expectEqual(114, tc.page_navigate);
try testing.expectEqual(2006, tc.page_navigated);
notifier.unregister(.page_navigated, &tc);
notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
try testing.expectEqual(114, tc.page_navigate);
try testing.expectEqual(2006, tc.page_navigated);
// already unregistered, try anyways
notifier.unregister(.page_navigated, &tc);
notifier.dispatch(.page_navigate, &.{ .timestamp = 100, .url = undefined, .opts = .{} });
notifier.dispatch(.page_navigated, &.{ .timestamp = 1000, .url = undefined });
try testing.expectEqual(114, tc.page_navigate);
try testing.expectEqual(2006, tc.page_navigated);
}
}
const TestClient = struct {