Optimize Network.responseReceived

Add a header iterator to the transfer. This removes the need for NetworkState,
duping header name/values, and the http_header_received event.

src/cdp/cdp.zig

@@ -77,7 +77,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
         // Extra headers to add to all requests. TBD under which conditions this should be reset.
         extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty,
 
-        network_state: NetworkState,
         intercept_state: InterceptState,
 
         const Self = @This();
@@ -94,7 +93,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 .browser_context = null,
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
                 .notification_arena = std.heap.ArenaAllocator.init(allocator),
-                .network_state = try NetworkState.init(allocator),
                 .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena?
             };
         }
@@ -104,7 +102,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 bc.deinit();
             }
             self.intercept_state.deinit(); // TBD Should this live in BC?
-            self.network_state.deinit();
             self.browser.deinit();
             self.message_arena.deinit();
             self.notification_arena.deinit();
@@ -451,15 +448,13 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         pub fn networkEnable(self: *Self) !void {
             try self.cdp.browser.notification.register(.http_request_fail, self, onHttpRequestFail);
             try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart);
-            try self.cdp.browser.notification.register(.http_header_received, self, onHttpHeaderReceived);
+            try self.cdp.browser.notification.register(.http_headers_done, self, onHttpHeadersDone);
-            try self.cdp.browser.notification.register(.http_headers_done_receiving, self, onHttpHeadersDoneReceiving);
         }
 
         pub fn networkDisable(self: *Self) void {
             self.cdp.browser.notification.unregister(.http_request_fail, self);
             self.cdp.browser.notification.unregister(.http_request_start, self);
-            self.cdp.browser.notification.unregister(.http_header_received, self);
+            self.cdp.browser.notification.unregister(.http_headers_done, self);
-            self.cdp.browser.notification.unregister(.http_headers_done_receiving, self);
         }
 
         pub fn fetchEnable(self: *Self) !void {
@@ -473,7 +468,6 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             try @import("domains/page.zig").pageRemove(self);
-            try @import("domains/network.zig").pageRemove(self);
         }
 
         pub fn onPageCreated(ctx: *anyopaque, page: *Page) !void {
@@ -504,22 +498,16 @@ pub fn BrowserContext(comptime CDP_T: type) type {
             try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data);
         }
 
-        pub fn onHttpHeaderReceived(ctx: *anyopaque, data: *const Notification.ResponseHeader) !void {
-            const self: *Self = @alignCast(@ptrCast(ctx));
-            defer self.resetNotificationArena();
-            try self.cdp.network_state.putOrAppendReceivedHeader(data.request_id, data.status, data.header);
-        }
-
         pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             defer self.resetNotificationArena();
             return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, data);
         }
 
-        pub fn onHttpHeadersDoneReceiving(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void {
+        pub fn onHttpHeadersDone(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             defer self.resetNotificationArena();
-            return @import("domains/network.zig").httpHeadersDoneReceiving(self.notification_arena, self, data);
+            return @import("domains/network.zig").httpHeadersDone(self.notification_arena, self, data);
         }
 
         fn resetNotificationArena(self: *Self) void {

src/cdp/domains/network.zig

@@ -22,6 +22,7 @@ const Allocator = std.mem.Allocator;
 const Notification = @import("../../notification.zig").Notification;
 const log = @import("../../log.zig");
 const CdpStorage = @import("storage.zig");
+const Transfer = @import("../../http/Client.zig").Transfer;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
@@ -61,43 +62,6 @@ const Response = struct {
     // be loaded with each chunks as Network.dataReceiveds are coming in.
 };
 
-// Stored in CDP
-pub const NetworkState = struct {
-    arena: std.heap.ArenaAllocator,
-    received: std.AutoArrayHashMap(u64, Response),
-
-    pub fn init(allocator: Allocator) !NetworkState {
-        return .{
-            .arena = std.heap.ArenaAllocator.init(allocator),
-            .received = std.AutoArrayHashMap(u64, Response).init(allocator),
-        };
-    }
-
-    pub fn deinit(self: *NetworkState) void {
-        self.received.deinit();
-        self.arena.deinit();
-    }
-
-    pub fn putOrAppendReceivedHeader(self: *NetworkState, request_id: u64, status: u16, header: std.http.Header) !void {
-        const kv = try self.received.getOrPut(request_id);
-        if (!kv.found_existing) {
-            kv.value_ptr.* = .{ .status = status };
-        }
-
-        const a = self.arena.allocator();
-        const name = try a.dupe(u8, header.name);
-        const value = try a.dupe(u8, header.value);
-        try kv.value_ptr.headers.put(a, name, value);
-    }
-};
-
-pub fn pageRemove(bc: anytype) !void {
-    // The main page is going to be removed
-    const state = &bc.cdp.network_state;
-    state.received.clearRetainingCapacity(); // May need to be in pageRemoved
-    _ = state.arena.reset(.{ .retain_with_limit = 1024 });
-}
-
 fn enable(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
     try bc.networkEnable();
@@ -314,7 +278,7 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification
     }, .{ .session_id = session_id });
 }
 
-pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void {
+pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void {
     // Isn't possible to do a network request within a Browser (which our
     // notification is tied to), without a page.
     std.debug.assert(bc.session.page != null);
@@ -333,7 +297,7 @@ pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const N
         .query = true,
     });
 
-    const response = bc.cdp.network_state.received.get(request.transfer.id) orelse return error.ResponseNotFound;
+    const status = request.transfer.response_header.?.status;
 
     // We're missing a bunch of fields, but, for now, this seems like enough
     try cdp.sendEvent("Network.responseReceived", .{
@@ -341,9 +305,9 @@ pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const N
         .loaderId = bc.loader_id,
         .response = .{
             .url = url,
-            .status = response.status,
+            .status = status,
-            .statusText = @as(std.http.Status, @enumFromInt(response.status)).phrase() orelse "Unknown",
+            .statusText = @as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown",
-            .headers = std.json.ArrayHashMap([]const u8){ .map = response.headers },
+            .headers = ResponseHeaderWriter.init(request.transfer),
         },
         .frameId = target_id,
     }, .{ .session_id = session_id });
@@ -355,6 +319,26 @@ pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToS
     return buf.items;
 }
 
+const ResponseHeaderWriter = struct {
+    transfer: *Transfer,
+
+    fn init(transfer: *Transfer) ResponseHeaderWriter {
+        return .{
+            .transfer = transfer,
+        };
+    }
+
+    pub fn jsonStringify(self: *const ResponseHeaderWriter, writer: anytype) !void {
+        try writer.beginObject();
+        var it = self.transfer.responseHeaderIterator();
+        while (it.next()) |hdr| {
+            try writer.objectField(hdr.name);
+            try writer.write(hdr.value);
+        }
+        try writer.endObject();
+    }
+};
+
 const testing = @import("../testing.zig");
 test "cdp.network setExtraHTTPHeaders" {
     var ctx = testing.context();

src/http/Client.zig

@@ -668,6 +668,12 @@ pub const Transfer = struct {
         }
 
         if (buf_len == 2) {
+            if (getResponseHeader(easy, "content-type")) |value| {
+                const len = @min(value.len, hdr._content_type.len);
+                hdr._content_type_len = len;
+                @memcpy(hdr._content_type[0..len], value[0..len]);
+            }
+
             transfer.req.header_done_callback(transfer) catch |err| {
                 log.err(.http, "header_done_callback", .{ .err = err, .req = transfer });
                 // returning < buf_len terminates the request
@@ -675,7 +681,7 @@ pub const Transfer = struct {
             };
 
             if (transfer.client.notification) |notification| {
-                notification.dispatch(.http_headers_done_receiving, &.{
+                notification.dispatch(.http_headers_done, &.{
                     .transfer = transfer,
                 });
             }
@@ -686,16 +692,6 @@ pub const Transfer = struct {
                     return 0;
                 };
             }
-
-            if (transfer.client.notification) |notification| {
-                if (Http.Headers.parseHeader(header)) |hdr_name_value| {
-                    notification.dispatch(.http_header_received, &.{
-                        .request_id = transfer.id,
-                        .status = hdr.status,
-                        .header = hdr_name_value,
-                    });
-                } else log.err(.http, "invalid header", .{ .line = header });
-            }
         }
         return buf_len;
     }
@@ -721,6 +717,12 @@ pub const Transfer = struct {
         return chunk_len;
     }
 
+    // we assume that the caller is smart and only calling this after being
+    // told that the header was ready.
+    pub fn responseHeaderIterator(self: *Transfer) HeaderIterator {
+        return .{ .easy = self._handle.?.conn.easy };
+    }
+
     // pub because Page.printWaitAnalysis uses it
     pub fn fromEasy(easy: *c.CURL) !*Transfer {
         var private: *anyopaque = undefined;
@@ -742,3 +744,37 @@ pub const Header = struct {
         return self._content_type[0..self._content_type_len];
     }
 };
+
+const HeaderIterator = struct {
+    easy: *c.CURL,
+    prev: ?*c.curl_header = null,
+
+    pub fn next(self: *HeaderIterator) ?struct { name: []const u8, value: []const u8 } {
+        const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null;
+        self.prev = h;
+
+        const header = h.*;
+        return .{
+            .name = std.mem.span(header.name),
+            .value = std.mem.span(header.value),
+        };
+    }
+};
+
+fn getResponseHeader(easy: *c.CURL, name: [:0]const u8) ?[]const u8 {
+    var hdr: [*c]c.curl_header = null;
+    const result = c.curl_easy_header(easy, name, 0, c.CURLH_HEADER, -1, &hdr);
+    if (result == c.CURLE_OK) {
+        return std.mem.span(hdr.*.value);
+    }
+
+    if (result == c.CURLE_FAILED_INIT) {
+        // seems to be what it returns if the header isn't found
+        return null;
+    }
+    log.err(.http, "get response header", .{
+        .name = name,
+        .err = @import("errors.zig").fromCode(result),
+    });
+    return null;
+}

src/http/Http.zig

@@ -21,8 +21,9 @@ const std = @import("std");
 pub const c = @cImport({
     @cInclude("curl/curl.h");
 });
-const errors = @import("errors.zig");
+
 const Client = @import("Client.zig");
+const errors = @import("errors.zig");
 
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;

src/notification.zig

@@ -63,8 +63,7 @@ pub const Notification = struct {
         http_request_fail: List = .{},
         http_request_start: List = .{},
         http_request_intercept: List = .{},
-        http_header_received: List = .{},
+        http_headers_done: List = .{},
-        http_headers_done_receiving: List = .{},
         notification_created: List = .{},
     };
 
@@ -76,8 +75,7 @@ pub const Notification = struct {
         http_request_fail: *const RequestFail,
         http_request_start: *const RequestStart,
         http_request_intercept: *const RequestIntercept,
-        http_header_received: *const ResponseHeader,
+        http_headers_done: *const ResponseHeadersDone,
-        http_headers_done_receiving: *const ResponseHeadersDone,
         notification_created: *Notification,
     };
     const EventType = std.meta.FieldEnum(Events);
@@ -104,12 +102,6 @@ pub const Notification = struct {
         wait_for_interception: *bool,
     };
 
-    pub const ResponseHeader = struct {
-        request_id: u64,
-        status: u16,
-        header: std.http.Header,
-    };
-
     pub const ResponseHeadersDone = struct {
         transfer: *Transfer,
     };