From 96b10f4b8517cb4675dce5bc666e996942ad2d3a Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Thu, 14 Aug 2025 15:50:56 +0800 Subject: [PATCH] Optimize Network.responseReceived Add a header iterator to the transfer. This removes the need for NetworkState, duping header name/values, and the http_header_received event. --- src/cdp/cdp.zig | 20 +++-------- src/cdp/domains/network.zig | 68 ++++++++++++++----------------------- src/http/Client.zig | 58 +++++++++++++++++++++++++------ src/http/Http.zig | 3 +- src/notification.zig | 12 ++----- 5 files changed, 81 insertions(+), 80 deletions(-) diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index a01ddce7..2435f7fd 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -77,7 +77,6 @@ pub fn CDPT(comptime TypeProvider: type) type { // Extra headers to add to all requests. TBD under which conditions this should be reset. extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty, - network_state: NetworkState, intercept_state: InterceptState, const Self = @This(); @@ -94,7 +93,6 @@ pub fn CDPT(comptime TypeProvider: type) type { .browser_context = null, .message_arena = std.heap.ArenaAllocator.init(allocator), .notification_arena = std.heap.ArenaAllocator.init(allocator), - .network_state = try NetworkState.init(allocator), .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena? }; } @@ -104,7 +102,6 @@ pub fn CDPT(comptime TypeProvider: type) type { bc.deinit(); } self.intercept_state.deinit(); // TBD Should this live in BC? - self.network_state.deinit(); self.browser.deinit(); self.message_arena.deinit(); self.notification_arena.deinit(); @@ -451,15 +448,13 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn networkEnable(self: *Self) !void { try self.cdp.browser.notification.register(.http_request_fail, self, onHttpRequestFail); try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart); - try self.cdp.browser.notification.register(.http_header_received, self, onHttpHeaderReceived); - try self.cdp.browser.notification.register(.http_headers_done_receiving, self, onHttpHeadersDoneReceiving); + try self.cdp.browser.notification.register(.http_headers_done, self, onHttpHeadersDone); } pub fn networkDisable(self: *Self) void { self.cdp.browser.notification.unregister(.http_request_fail, self); self.cdp.browser.notification.unregister(.http_request_start, self); - self.cdp.browser.notification.unregister(.http_header_received, self); - self.cdp.browser.notification.unregister(.http_headers_done_receiving, self); + self.cdp.browser.notification.unregister(.http_headers_done, self); } pub fn fetchEnable(self: *Self) !void { @@ -473,7 +468,6 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void { const self: *Self = @alignCast(@ptrCast(ctx)); try @import("domains/page.zig").pageRemove(self); - try @import("domains/network.zig").pageRemove(self); } pub fn onPageCreated(ctx: *anyopaque, page: *Page) !void { @@ -504,22 +498,16 @@ pub fn BrowserContext(comptime CDP_T: type) type { try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data); } - pub fn onHttpHeaderReceived(ctx: *anyopaque, data: *const Notification.ResponseHeader) !void { - const self: *Self = @alignCast(@ptrCast(ctx)); - defer self.resetNotificationArena(); - try self.cdp.network_state.putOrAppendReceivedHeader(data.request_id, data.status, data.header); - } - pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, data); } - pub fn onHttpHeadersDoneReceiving(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void { + pub fn onHttpHeadersDone(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); - return @import("domains/network.zig").httpHeadersDoneReceiving(self.notification_arena, self, data); + return @import("domains/network.zig").httpHeadersDone(self.notification_arena, self, data); } fn resetNotificationArena(self: *Self) void { diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index aa88f6f8..d3f30be2 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -22,6 +22,7 @@ const Allocator = std.mem.Allocator; const Notification = @import("../../notification.zig").Notification; const log = @import("../../log.zig"); const CdpStorage = @import("storage.zig"); +const Transfer = @import("../../http/Client.zig").Transfer; pub fn processMessage(cmd: anytype) !void { const action = std.meta.stringToEnum(enum { @@ -61,43 +62,6 @@ const Response = struct { // be loaded with each chunks as Network.dataReceiveds are coming in. }; -// Stored in CDP -pub const NetworkState = struct { - arena: std.heap.ArenaAllocator, - received: std.AutoArrayHashMap(u64, Response), - - pub fn init(allocator: Allocator) !NetworkState { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .received = std.AutoArrayHashMap(u64, Response).init(allocator), - }; - } - - pub fn deinit(self: *NetworkState) void { - self.received.deinit(); - self.arena.deinit(); - } - - pub fn putOrAppendReceivedHeader(self: *NetworkState, request_id: u64, status: u16, header: std.http.Header) !void { - const kv = try self.received.getOrPut(request_id); - if (!kv.found_existing) { - kv.value_ptr.* = .{ .status = status }; - } - - const a = self.arena.allocator(); - const name = try a.dupe(u8, header.name); - const value = try a.dupe(u8, header.value); - try kv.value_ptr.headers.put(a, name, value); - } -}; - -pub fn pageRemove(bc: anytype) !void { - // The main page is going to be removed - const state = &bc.cdp.network_state; - state.received.clearRetainingCapacity(); // May need to be in pageRemoved - _ = state.arena.reset(.{ .retain_with_limit = 1024 }); -} - fn enable(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; try bc.networkEnable(); @@ -314,7 +278,7 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification }, .{ .session_id = session_id }); } -pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void { +pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. std.debug.assert(bc.session.page != null); @@ -333,7 +297,7 @@ pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const N .query = true, }); - const response = bc.cdp.network_state.received.get(request.transfer.id) orelse return error.ResponseNotFound; + const status = request.transfer.response_header.?.status; // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.responseReceived", .{ @@ -341,9 +305,9 @@ pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const N .loaderId = bc.loader_id, .response = .{ .url = url, - .status = response.status, - .statusText = @as(std.http.Status, @enumFromInt(response.status)).phrase() orelse "Unknown", - .headers = std.json.ArrayHashMap([]const u8){ .map = response.headers }, + .status = status, + .statusText = @as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown", + .headers = ResponseHeaderWriter.init(request.transfer), }, .frameId = target_id, }, .{ .session_id = session_id }); @@ -355,6 +319,26 @@ pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToS return buf.items; } +const ResponseHeaderWriter = struct { + transfer: *Transfer, + + fn init(transfer: *Transfer) ResponseHeaderWriter { + return .{ + .transfer = transfer, + }; + } + + pub fn jsonStringify(self: *const ResponseHeaderWriter, writer: anytype) !void { + try writer.beginObject(); + var it = self.transfer.responseHeaderIterator(); + while (it.next()) |hdr| { + try writer.objectField(hdr.name); + try writer.write(hdr.value); + } + try writer.endObject(); + } +}; + const testing = @import("../testing.zig"); test "cdp.network setExtraHTTPHeaders" { var ctx = testing.context(); diff --git a/src/http/Client.zig b/src/http/Client.zig index e66ba9ec..1e426f6f 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -668,6 +668,12 @@ pub const Transfer = struct { } if (buf_len == 2) { + if (getResponseHeader(easy, "content-type")) |value| { + const len = @min(value.len, hdr._content_type.len); + hdr._content_type_len = len; + @memcpy(hdr._content_type[0..len], value[0..len]); + } + transfer.req.header_done_callback(transfer) catch |err| { log.err(.http, "header_done_callback", .{ .err = err, .req = transfer }); // returning < buf_len terminates the request @@ -675,7 +681,7 @@ pub const Transfer = struct { }; if (transfer.client.notification) |notification| { - notification.dispatch(.http_headers_done_receiving, &.{ + notification.dispatch(.http_headers_done, &.{ .transfer = transfer, }); } @@ -686,16 +692,6 @@ pub const Transfer = struct { return 0; }; } - - if (transfer.client.notification) |notification| { - if (Http.Headers.parseHeader(header)) |hdr_name_value| { - notification.dispatch(.http_header_received, &.{ - .request_id = transfer.id, - .status = hdr.status, - .header = hdr_name_value, - }); - } else log.err(.http, "invalid header", .{ .line = header }); - } } return buf_len; } @@ -721,6 +717,12 @@ pub const Transfer = struct { return chunk_len; } + // we assume that the caller is smart and only calling this after being + // told that the header was ready. + pub fn responseHeaderIterator(self: *Transfer) HeaderIterator { + return .{ .easy = self._handle.?.conn.easy }; + } + // pub because Page.printWaitAnalysis uses it pub fn fromEasy(easy: *c.CURL) !*Transfer { var private: *anyopaque = undefined; @@ -742,3 +744,37 @@ pub const Header = struct { return self._content_type[0..self._content_type_len]; } }; + +const HeaderIterator = struct { + easy: *c.CURL, + prev: ?*c.curl_header = null, + + pub fn next(self: *HeaderIterator) ?struct { name: []const u8, value: []const u8 } { + const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null; + self.prev = h; + + const header = h.*; + return .{ + .name = std.mem.span(header.name), + .value = std.mem.span(header.value), + }; + } +}; + +fn getResponseHeader(easy: *c.CURL, name: [:0]const u8) ?[]const u8 { + var hdr: [*c]c.curl_header = null; + const result = c.curl_easy_header(easy, name, 0, c.CURLH_HEADER, -1, &hdr); + if (result == c.CURLE_OK) { + return std.mem.span(hdr.*.value); + } + + if (result == c.CURLE_FAILED_INIT) { + // seems to be what it returns if the header isn't found + return null; + } + log.err(.http, "get response header", .{ + .name = name, + .err = @import("errors.zig").fromCode(result), + }); + return null; +} diff --git a/src/http/Http.zig b/src/http/Http.zig index 506a0f13..3195a24c 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -21,8 +21,9 @@ const std = @import("std"); pub const c = @cImport({ @cInclude("curl/curl.h"); }); -const errors = @import("errors.zig"); + const Client = @import("Client.zig"); +const errors = @import("errors.zig"); const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; diff --git a/src/notification.zig b/src/notification.zig index 0e8860de..c499c584 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -63,8 +63,7 @@ pub const Notification = struct { http_request_fail: List = .{}, http_request_start: List = .{}, http_request_intercept: List = .{}, - http_header_received: List = .{}, - http_headers_done_receiving: List = .{}, + http_headers_done: List = .{}, notification_created: List = .{}, }; @@ -76,8 +75,7 @@ pub const Notification = struct { http_request_fail: *const RequestFail, http_request_start: *const RequestStart, http_request_intercept: *const RequestIntercept, - http_header_received: *const ResponseHeader, - http_headers_done_receiving: *const ResponseHeadersDone, + http_headers_done: *const ResponseHeadersDone, notification_created: *Notification, }; const EventType = std.meta.FieldEnum(Events); @@ -104,12 +102,6 @@ pub const Notification = struct { wait_for_interception: *bool, }; - pub const ResponseHeader = struct { - request_id: u64, - status: u16, - header: std.http.Header, - }; - pub const ResponseHeadersDone = struct { transfer: *Transfer, };