From f1672dd6d2cb7d1245c23555331d76d0efbf0e13 Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Thu, 5 Jun 2025 16:03:06 +0200 Subject: [PATCH 1/2] setExtraHTTPHeaders --- src/cdp/cdp.zig | 3 ++ src/cdp/domains/network.zig | 68 +++++++++++++++++++++++++++++++++++-- src/http/client.zig | 3 +- src/notification.zig | 3 +- 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 19ba3c71..8a474c37 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -71,6 +71,9 @@ pub fn CDPT(comptime TypeProvider: type) type { // Used for processing notifications within a browser context. notification_arena: std.heap.ArenaAllocator, + // Extra headers to add to all requests. TBD under which conditions this should be reset. + extra_headers: std.ArrayListUnmanaged(std.http.Header) = .empty, + const Self = @This(); pub fn init(app: *App, client: TypeProvider.Client) !Self { diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index f461dd3e..3d20237d 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -18,6 +18,7 @@ const std = @import("std"); const Notification = @import("../../notification.zig").Notification; +const log = @import("../../log.zig"); const Allocator = std.mem.Allocator; @@ -26,12 +27,14 @@ pub fn processMessage(cmd: anytype) !void { enable, disable, setCacheDisabled, + setExtraHTTPHeaders, }, cmd.input.action) orelse return error.UnknownMethod; switch (action) { .enable => return enable(cmd), .disable => return disable(cmd), .setCacheDisabled => return cmd.sendResult(null, .{}), + .setExtraHTTPHeaders => return setExtraHTTPHeaders(cmd), } } @@ -47,6 +50,27 @@ fn disable(cmd: anytype) !void { return cmd.sendResult(null, .{}); } +fn setExtraHTTPHeaders(cmd: anytype) !void { + const params = (try cmd.params(struct { + headers: std.json.ArrayHashMap([]const u8), + })) orelse return error.InvalidParams; + + const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; + + // Copy the headers onto the browser context arena + const arena = bc.arena; + const extra_headers = &bc.cdp.extra_headers; + + extra_headers.clearRetainingCapacity(); + try extra_headers.ensureTotalCapacity(arena, params.headers.map.count()); + var it = params.headers.map.iterator(); + while (it.next()) |header| { + extra_headers.appendAssumeCapacity(.{ .name = try arena.dupe(u8, header.key_ptr.*), .value = try arena.dupe(u8, header.value_ptr.*) }); + } + + return cmd.sendResult(null, .{}); +} + pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notification.RequestStart) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. @@ -59,6 +83,21 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notificat const target_id = bc.target_id orelse unreachable; const page = bc.session.currentPage() orelse unreachable; + // Modify request with extra CDP headers + const original_len = request.headers.items.len; + try request.headers.ensureTotalCapacity(arena, original_len + cdp.extra_headers.items.len); + outer: for (cdp.extra_headers.items) |extra| { + for (request.headers.items[0..original_len]) |*existing_header| { + if (std.mem.eql(u8, existing_header.name, extra.name)) { + // If the header already exists, we overwrite it + log.debug(.cdp, "request header overwritten", .{ .name = extra.name }); + existing_header.value = extra.value; + continue :outer; + } + } + request.headers.appendAssumeCapacity(extra); + } + const document_url = try urlToString(arena, &page.url.uri, .{ .scheme = true, .authentication = true, @@ -80,8 +119,8 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notificat }); var headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - try headers.ensureTotalCapacity(arena, request.headers.len); - for (request.headers) |header| { + try headers.ensureTotalCapacity(arena, request.headers.items.len); + for (request.headers.items) |header| { headers.putAssumeCapacity(header.name, header.value); } @@ -129,13 +168,13 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.responseReceived", .{ .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}), - .frameId = target_id, .loaderId = bc.loader_id, .response = .{ .url = url, .status = request.status, .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, }, + .frameId = target_id, }, .{ .session_id = session_id }); } @@ -144,3 +183,26 @@ fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStrea try url.writeToStream(opts, buf.writer(arena)); return buf.items; } + +const testing = @import("../testing.zig"); +test "cdp.network setExtraHTTPHeaders" { + var ctx = testing.context(); + defer ctx.deinit(); + + // _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" }); + try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } }); + + try ctx.processMessage(.{ + .id = 3, + .method = "Network.setExtraHTTPHeaders", + .params = .{ .headers = .{ .foo = "bar" } }, + }); + + try ctx.processMessage(.{ + .id = 4, + .method = "Network.setExtraHTTPHeaders", + .params = .{ .headers = .{ .food = "bars" } }, + }); + + try testing.expectEqual(ctx.cdp_.?.browser_context.?.cdp.extra_headers.items.len, 1); +} diff --git a/src/http/client.zig b/src/http/client.zig index 81b11cd7..de367020 100644 --- a/src/http/client.zig +++ b/src/http/client.zig @@ -736,10 +736,11 @@ pub const Request = struct { } self._notified_start = true; notification.dispatch(.http_request_start, &.{ + .arena = self.arena, .id = self.id, .url = self.request_uri, .method = self.method, - .headers = self.headers.items, + .headers = &self.headers, .has_body = self.body != null, }); } diff --git a/src/notification.zig b/src/notification.zig index 5cb198f7..5a32f559 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -89,10 +89,11 @@ pub const Notification = struct { }; pub const RequestStart = struct { + arena: Allocator, id: usize, url: *const std.Uri, method: http_client.Request.Method, - headers: []std.http.Header, + headers: *std.ArrayListUnmanaged(std.http.Header), has_body: bool, }; From bacef41a3b37192fea33327daa61cfec6b585ccb Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Fri, 6 Jun 2025 10:33:15 +0200 Subject: [PATCH 2/2] extra header feedback --- src/cdp/domains/network.zig | 35 ++++++++++++++++++++++------------- src/cdp/domains/target.zig | 3 +++ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 3d20237d..203131e8 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -71,6 +71,19 @@ fn setExtraHTTPHeaders(cmd: anytype) !void { return cmd.sendResult(null, .{}); } +// Upsert a header into the headers array. +// returns true if the header was added, false if it was updated +fn putAssumeCapacity(headers: *std.ArrayListUnmanaged(std.http.Header), extra: std.http.Header) bool { + for (headers.items) |*header| { + if (std.mem.eql(u8, header.name, extra.name)) { + header.value = extra.value; + return false; + } + } + headers.appendAssumeCapacity(extra); + return true; +} + pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notification.RequestStart) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. @@ -84,18 +97,10 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notificat const page = bc.session.currentPage() orelse unreachable; // Modify request with extra CDP headers - const original_len = request.headers.items.len; - try request.headers.ensureTotalCapacity(arena, original_len + cdp.extra_headers.items.len); - outer: for (cdp.extra_headers.items) |extra| { - for (request.headers.items[0..original_len]) |*existing_header| { - if (std.mem.eql(u8, existing_header.name, extra.name)) { - // If the header already exists, we overwrite it - log.debug(.cdp, "request header overwritten", .{ .name = extra.name }); - existing_header.value = extra.value; - continue :outer; - } - } - request.headers.appendAssumeCapacity(extra); + try request.headers.ensureTotalCapacity(request.arena, request.headers.items.len + cdp.extra_headers.items.len); + for (cdp.extra_headers.items) |extra| { + const new = putAssumeCapacity(request.headers, extra); + if (!new) log.debug(.cdp, "request header overwritten", .{ .name = extra.name }); } const document_url = try urlToString(arena, &page.url.uri, .{ @@ -204,5 +209,9 @@ test "cdp.network setExtraHTTPHeaders" { .params = .{ .headers = .{ .food = "bars" } }, }); - try testing.expectEqual(ctx.cdp_.?.browser_context.?.cdp.extra_headers.items.len, 1); + const bc = ctx.cdp().browser_context.?; + try testing.expectEqual(bc.cdp.extra_headers.items.len, 1); + + try ctx.processMessage(.{ .id = 5, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } }); + try testing.expectEqual(bc.cdp.extra_headers.items.len, 0); } diff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zig index f6c197ce..3461bd44 100644 --- a/src/cdp/domains/target.zig +++ b/src/cdp/domains/target.zig @@ -389,6 +389,9 @@ fn doAttachtoTarget(cmd: anytype, target_id: []const u8) !void { std.debug.assert(bc.session_id == null); const session_id = cmd.cdp.session_id_gen.next(); + // extra_headers should not be kept on a new page or tab, currently we have only 1 page, we clear it just in case + bc.cdp.extra_headers.clearRetainingCapacity(); + try cmd.sendEvent("Target.attachedToTarget", AttachToTarget{ .sessionId = session_id, .targetInfo = TargetInfo{