http_headers_done_receiving

src/cdp/cdp.zig

@@ -29,6 +29,7 @@ const Page = @import("../browser/page.zig").Page;
 const Inspector = @import("../browser/env.zig").Env.Inspector;
 const Incrementing = @import("../id.zig").Incrementing;
 const Notification = @import("../notification.zig").Notification;
+const NetworkState = @import("domains/network.zig").NetworkState;
 const InterceptState = @import("domains/fetch.zig").InterceptState;
 
 const polyfill = @import("../browser/polyfill/polyfill.zig");
@@ -76,6 +77,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
         // Extra headers to add to all requests. TBD under which conditions this should be reset.
         extra_headers: std.ArrayListUnmanaged(std.http.Header) = .empty,
 
+        network_state: NetworkState,
         intercept_state: InterceptState,
 
         const Self = @This();
@@ -92,6 +94,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 .browser_context = null,
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
                 .notification_arena = std.heap.ArenaAllocator.init(allocator),
+                .network_state = try NetworkState.init(allocator),
                 .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena?
             };
         }
@@ -101,6 +104,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 bc.deinit();
             }
             self.intercept_state.deinit(); // TBD Should this live in BC?
+            self.network_state.deinit();
             self.browser.deinit();
             self.message_arena.deinit();
             self.notification_arena.deinit();
@@ -447,13 +451,15 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         pub fn networkEnable(self: *Self) !void {
             try self.cdp.browser.notification.register(.http_request_fail, self, onHttpRequestFail);
             try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart);
-            try self.cdp.browser.notification.register(.http_request_complete, self, onHttpRequestComplete);
+            try self.cdp.browser.notification.register(.http_header_received, self, onHttpHeaderReceived);
+            try self.cdp.browser.notification.register(.http_headers_done_receiving, self, onHttpHeadersDoneReceiving);
         }
 
         pub fn networkDisable(self: *Self) void {
             self.cdp.browser.notification.unregister(.http_request_fail, self);
             self.cdp.browser.notification.unregister(.http_request_start, self);
-            self.cdp.browser.notification.unregister(.http_request_complete, self);
+            self.cdp.browser.notification.unregister(.http_header_received, self);
+            self.cdp.browser.notification.unregister(.http_headers_done_receiving, self);
         }
 
         pub fn fetchEnable(self: *Self) !void {
@@ -466,7 +472,8 @@ pub fn BrowserContext(comptime CDP_T: type) type {
 
         pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
-            return @import("domains/page.zig").pageRemove(self);
+            try @import("domains/page.zig").pageRemove(self);
+            try @import("domains/network.zig").pageRemove(self);
         }
 
         pub fn onPageCreated(ctx: *anyopaque, page: *Page) !void {
@@ -497,16 +504,22 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data);
         }
 
+        pub fn onHttpHeaderReceived(ctx: *anyopaque, data: *const Notification.ResponseHeader) !void {
+            const self: *Self = @alignCast(@ptrCast(ctx));
+            defer self.resetNotificationArena();
+            try self.cdp.network_state.putOrAppendReceivedHeader(data.request_id, data.status, data.header);
+        }
+
         pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             defer self.resetNotificationArena();
             return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, data);
         }
 
-        pub fn onHttpRequestComplete(ctx: *anyopaque, data: *const Notification.RequestComplete) !void {
+        pub fn onHttpHeadersDoneReceiving(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             defer self.resetNotificationArena();
-            return @import("domains/network.zig").httpRequestComplete(self.notification_arena, self, data);
+            return @import("domains/network.zig").httpHeadersDoneReceiving(self.notification_arena, self, data);
         }
 
         fn resetNotificationArena(self: *Self) void {

src/cdp/domains/network.zig

@@ -51,6 +51,49 @@ pub fn processMessage(cmd: anytype) !void {
     }
 }
 
+const Response = struct {
+    status: u16,
+    headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty, // These may not be complete yet, but we only tell the client Network.responseReceived when all the headers are in
+    // Later should store body as well to support getResponseBody which should only work once Network.loadingFinished is sent
+    // but the body itself would be loaded with each chunks as Network.dataReceiveds are coming in.
+};
+
+// Stored in CDP
+pub const NetworkState = struct {
+    const Self = @This();
+    received: std.AutoArrayHashMap(u64, Response),
+    arena: std.heap.ArenaAllocator,
+
+    pub fn init(allocator: Allocator) !NetworkState {
+        return .{
+            .received = std.AutoArrayHashMap(u64, Response).init(allocator),
+            .arena = std.heap.ArenaAllocator.init(allocator),
+        };
+    }
+
+    pub fn deinit(self: *Self) void {
+        self.received.deinit();
+        self.arena.deinit();
+    }
+
+    pub fn putOrAppendReceivedHeader(self: *NetworkState, request_id: u64, status: u16, header: std.http.Header) !void {
+        const kv = try self.received.getOrPut(request_id);
+        if (!kv.found_existing) kv.value_ptr.* = .{ .status = status };
+
+        const a = self.arena.allocator();
+        const name = try a.dupe(u8, header.name);
+        const value = try a.dupe(u8, header.value);
+        try kv.value_ptr.headers.put(a, name, value);
+    }
+};
+
+pub fn pageRemove(bc: anytype) !void {
+    // The main page is going to be removed
+    const state = &bc.cdp.network_state;
+    state.received.clearRetainingCapacity(); // May need to be in pageRemoved
+    _ = state.arena.reset(.{ .retain_with_limit = 1024 });
+}
+
 fn enable(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
     try bc.networkEnable();
@@ -282,7 +325,7 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification
     }, .{ .session_id = session_id });
 }
 
-pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notification.RequestComplete) !void {
+pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void {
     // Isn't possible to do a network request within a Browser (which our
     // notification is tied to), without a page.
     std.debug.assert(bc.session.page != null);
@@ -293,7 +336,7 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi
     const session_id = bc.session_id orelse unreachable;
     const target_id = bc.target_id orelse unreachable;
 
-    const url = try urlToString(arena, request.url, .{
+    const url = try urlToString(arena, &request.transfer.uri, .{
         .scheme = true,
         .authentication = true,
         .authority = true,
@@ -301,22 +344,17 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi
         .query = true,
     });
 
-    // @newhttp
+    const response = bc.cdp.network_state.received.get(request.transfer.id) orelse return error.ResponseNotFound;
-    const headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
-    // try headers.ensureTotalCapacity(arena, request.headers.len);
-    // for (request.headers) |header| {
-    //     headers.putAssumeCapacity(header.name, header.value);
-    // }
 
     // We're missing a bunch of fields, but, for now, this seems like enough
     try cdp.sendEvent("Network.responseReceived", .{
-        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}),
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.transfer.id}),
         .loaderId = bc.loader_id,
         .response = .{
             .url = url,
-            .status = request.status,
+            .status = response.status,
-            .statusText = @as(std.http.Status, @enumFromInt(request.status)).phrase() orelse "Unknown",
+            .statusText = @as(std.http.Status, @enumFromInt(response.status)).phrase() orelse "Unknown",
-            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
+            .headers = std.json.ArrayHashMap([]const u8){ .map = response.headers },
         },
         .frameId = target_id,
     }, .{ .session_id = session_id });

src/http/Client.zig

@@ -247,6 +247,7 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer {
         .req = req,
         .ctx = req.ctx,
         .client = self,
+        .notification = &self.notification,
     };
     return transfer;
 }
@@ -549,6 +550,8 @@ pub const Transfer = struct {
 
     _redirecting: bool = false,
 
+    notification: *?*Notification, // Points to the Client's notification. TBD if a Browser can remove the notification before all Transfers are gone.
+
     fn deinit(self: *Transfer) void {
         self.req.headers.deinit();
         if (self._handle) |handle| {
@@ -673,12 +676,26 @@ pub const Transfer = struct {
                 // returning < buf_len terminates the request
                 return 0;
             };
+            if (transfer.notification.*) |notification| { // TBD before or after callback?
+                notification.dispatch(.http_headers_done_receiving, &.{
+                    .transfer = transfer,
+                });
+            }
         } else {
             if (transfer.req.header_callback) |cb| {
                 cb(transfer, header) catch |err| {
                     log.err(.http, "header_callback", .{ .err = err, .req = transfer });
                     return 0;
                 };
+                if (transfer.notification.*) |notification| { // TBD before or after callback?
+                    if (Http.Headers.parseHeader(header)) |hdr_name_value| {
+                        notification.dispatch(.http_header_received, &.{
+                            .request_id = transfer.id,
+                            .status = hdr.status,
+                            .header = hdr_name_value,
+                        });
+                    } else log.err(.http, "invalid header", .{ .line = header });
+                }
             }
         }
         return buf_len;

src/http/Http.zig

@@ -244,8 +244,8 @@ pub const Headers = struct {
         return list;
     }
 
-    fn parseHeader(header_str: []const u8) ?struct { name: []const u8, value: []const u8 } {
+    pub fn parseHeader(header_str: []const u8) ?std.http.Header {
-        const colon_pos = std.mem.indexOf(u8, header_str, ":") orelse return null;
+        const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
 
         const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
         const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");

src/notification.zig

@@ -63,7 +63,8 @@ pub const Notification = struct {
         http_request_fail: List = .{},
         http_request_start: List = .{},
         http_request_intercept: List = .{},
-        http_request_complete: List = .{},
+        http_header_received: List = .{},
+        http_headers_done_receiving: List = .{},
         notification_created: List = .{},
     };
 
@@ -75,7 +76,8 @@ pub const Notification = struct {
         http_request_fail: *const RequestFail,
         http_request_start: *const RequestStart,
         http_request_intercept: *const RequestIntercept,
-        http_request_complete: *const RequestComplete,
+        http_header_received: *const ResponseHeader,
+        http_headers_done_receiving: *const ResponseHeadersDone,
         notification_created: *Notification,
     };
     const EventType = std.meta.FieldEnum(Events);
@@ -102,17 +104,21 @@ pub const Notification = struct {
         wait_for_interception: *bool,
     };
 
+    pub const ResponseHeader = struct {
+        request_id: u64,
+        status: u16,
+        header: std.http.Header,
+    };
+
+    pub const ResponseHeadersDone = struct {
+        transfer: *Transfer,
+    };
+
     pub const RequestFail = struct {
         transfer: *Transfer,
         err: anyerror,
     };
 
-    pub const RequestComplete = struct {
-        id: usize,
-        url: *const std.Uri,
-        status: u16,
-    };
-
     pub fn init(allocator: Allocator, parent: ?*Notification) !*Notification {
         // This is put on the heap because we want to raise a .notification_created
         // event, so that, something like Telemetry, can receive the