Merge pull request #946 from lightpanda-io/request_interception

Request Interception

src/browser/ScriptManager.zig

@@ -24,8 +24,8 @@ const parser = @import("netsurf.zig");
 const Env = @import("env.zig").Env;
 const Page = @import("page.zig").Page;
 const DataURI = @import("DataURI.zig");
+const Http = @import("../http/Http.zig");
 const Browser = @import("browser.zig").Browser;
-const HttpClient = @import("../http/Client.zig");
 const URL = @import("../url.zig").URL;
 
 const Allocator = std.mem.Allocator;
@@ -57,7 +57,7 @@ deferreds: OrderList,
 
 shutdown: bool = false,
 
-client: *HttpClient,
+client: *Http.Client,
 allocator: Allocator,
 buffer_pool: BufferPool,
 script_pool: std.heap.MemoryPool(PendingScript),
@@ -229,7 +229,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
 
     errdefer pending_script.deinit();
 
-    var headers = try HttpClient.Headers.init();
+    var headers = try Http.Headers.init();
     try page.requestCookie(.{}).headersForRequest(page.arena, remote_url.?, &headers);
 
     try self.client.request(.{
@@ -238,6 +238,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
         .method = .GET,
         .headers = headers,
         .cookie_jar = page.cookie_jar,
+        .resource_type = .script,
         .start_callback = if (log.enabled(.http, .debug)) startCallback else null,
         .header_done_callback = headerCallback,
         .data_callback = dataCallback,
@@ -296,7 +297,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
         .buffer_pool = &self.buffer_pool,
     };
 
-    var headers = try HttpClient.Headers.init();
+    var headers = try Http.Headers.init();
     try self.page.requestCookie(.{}).headersForRequest(self.page.arena, url, &headers);
 
     var client = self.client;
@@ -306,6 +307,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
         .headers = headers,
         .cookie_jar = self.page.cookie_jar,
         .ctx = &blocking,
+        .resource_type = .script,
         .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null,
         .header_done_callback = Blocking.headerCallback,
         .data_callback = Blocking.dataCallback,
@@ -423,7 +425,7 @@ fn getList(self: *ScriptManager, script: *const Script) *OrderList {
     return &self.scripts;
 }
 
-fn startCallback(transfer: *HttpClient.Transfer) !void {
+fn startCallback(transfer: *Http.Transfer) !void {
     const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
     script.startCallback(transfer) catch |err| {
         log.err(.http, "SM.startCallback", .{ .err = err, .transfer = transfer });
@@ -431,7 +433,7 @@ fn startCallback(transfer: *HttpClient.Transfer) !void {
     };
 }
 
-fn headerCallback(transfer: *HttpClient.Transfer) !void {
+fn headerCallback(transfer: *Http.Transfer) !void {
     const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
     script.headerCallback(transfer) catch |err| {
         log.err(.http, "SM.headerCallback", .{
@@ -443,7 +445,7 @@ fn headerCallback(transfer: *HttpClient.Transfer) !void {
     };
 }
 
-fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
     const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
     script.dataCallback(transfer, data) catch |err| {
         log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len });
@@ -488,12 +490,12 @@ const PendingScript = struct {
         }
     }
 
-    fn startCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void {
+    fn startCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
         _ = self;
         log.debug(.http, "script fetch start", .{ .req = transfer });
     }
 
-    fn headerCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void {
+    fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
         const header = &transfer.response_header.?;
         log.debug(.http, "script header", .{
             .req = transfer,
@@ -513,7 +515,7 @@ const PendingScript = struct {
         self.script.source = .{ .remote = self.manager.buffer_pool.get() };
     }
 
-    fn dataCallback(self: *PendingScript, transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn dataCallback(self: *PendingScript, transfer: *Http.Transfer, data: []const u8) !void {
         _ = transfer;
         // too verbose
         // log.debug(.http, "script data chunk", .{
@@ -766,11 +768,11 @@ const Blocking = struct {
         done: BlockingResult,
     };
 
-    fn startCallback(transfer: *HttpClient.Transfer) !void {
+    fn startCallback(transfer: *Http.Transfer) !void {
         log.debug(.http, "script fetch start", .{ .req = transfer, .blocking = true });
     }
 
-    fn headerCallback(transfer: *HttpClient.Transfer) !void {
+    fn headerCallback(transfer: *Http.Transfer) !void {
         const header = &transfer.response_header.?;
         log.debug(.http, "script header", .{
             .req = transfer,
@@ -787,7 +789,7 @@ const Blocking = struct {
         self.buffer = self.buffer_pool.get();
     }
 
-    fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
         // too verbose
         // log.debug(.http, "script data chunk", .{
         //     .req = transfer,

src/browser/page.zig

@@ -30,7 +30,7 @@ const Renderer = @import("renderer.zig").Renderer;
 const Window = @import("html/window.zig").Window;
 const Walker = @import("dom/walker.zig").WalkerDepthFirst;
 const Scheduler = @import("Scheduler.zig");
-const HttpClient = @import("../http/Client.zig");
+const Http = @import("../http/Http.zig");
 const ScriptManager = @import("ScriptManager.zig");
 const HTMLDocument = @import("html/document.zig").HTMLDocument;
 
@@ -87,13 +87,23 @@ pub const Page = struct {
     polyfill_loader: polyfill.Loader = .{},
 
     scheduler: Scheduler,
-    http_client: *HttpClient,
+    http_client: *Http.Client,
     script_manager: ScriptManager,
 
     mode: Mode,
 
     load_state: LoadState = .parsing,
 
+    // Page.wait balances waiting for resources / tasks and producing an output.
+    // Up until a timeout, Page.wait will always wait for inflight or pending
+    // HTTP requests, via the Http.Client.active counter. However, intercepted
+    // requests (via CDP, but it could be anything), aren't considered "active"
+    // connection. So it's possible that we have intercepted requests (which are
+    // pending on some driver to continue/abort) while Http.Client.active == 0.
+    // This boolean exists to supplment Http.Client.active and inform Page.wait
+    // of pending connections.
+    request_intercepted: bool = false,
+
     const Mode = union(enum) {
         pre: void,
         err: anyerror,
@@ -275,16 +285,26 @@ pub const Page = struct {
         while (true) {
             SW: switch (self.mode) {
                 .pre, .raw => {
+                    if (self.request_intercepted) {
+                        // the page request was intercepted.
+
+                        // there shouldn't be any active requests;
+                        std.debug.assert(http_client.active == 0);
+
+                        // nothing we can do for this, need to kick the can up
+                        // the chain and wait for activity (e.g. a CDP message)
+                        // to unblock this.
+                        return;
+                    }
+
                     // The main page hasn't started/finished navigating.
                     // There's no JS to run, and no reason to run the scheduler.
-
                     if (http_client.active == 0) {
                         // haven't started navigating, I guess.
                         return;
                     }
 
                     // There should only be 1 active http transfer, the main page
-                    std.debug.assert(http_client.active == 1);
                     try http_client.tick(ms_remaining);
                 },
                 .html, .parsed => {
@@ -330,20 +350,35 @@ pub const Page = struct {
 
                     _ = try scheduler.runLowPriority();
 
-                    // We'll block here, waiting for network IO. We know
+                    const request_intercepted = self.request_intercepted;
-                    // when the next timeout is scheduled, and we know how long
+
-                    // the caller wants to wait for, so we can pick a good wait
+                    // We want to prioritize processing intercepted requests
-                    // duration
+                    // because, the sooner they get unblocked, the sooner we
-                    const ms_to_wait = @min(ms_remaining, ms_to_next_task orelse 1000);
+                    // can start the HTTP request. But we still want to advanced
+                    // existing HTTP requests, if possible. So, if we have
+                    // intercepted requests, we'll still look at existing HTTP
+                    // requests, but we won't block waiting for more data.
+                    const ms_to_wait =
+                        if (request_intercepted) 0
+
+                        // But if we have no intercepted requests, we'll wait
+                        // for as long as we can for data to our existing
+                        // inflight requests
+                        else @min(ms_remaining, ms_to_next_task orelse 1000);
+
                     try http_client.tick(ms_to_wait);
 
-                    if (try_catch.hasCaught()) {
+                    if (request_intercepted) {
-                        const msg = (try try_catch.err(self.arena)) orelse "unknown";
+                        // Again, proritizing intercepted requests. Exit this
-                        log.warn(.user_script, "page wait", .{ .err = msg, .src = "data" });
+                        // loop so that our caller can hopefully resolve them
-                        return error.JsError;
+                        // (i.e. continue or abort them);
+                        return;
                     }
                 },
-                .err => |err| return err,
+                .err => |err| {
+                    self.mode = .{ .raw_done = @errorName(err) };
+                    return err;
+                },
                 .raw_done => return,
             }
 
@@ -362,7 +397,7 @@ pub const Page = struct {
             std.debug.print("\nactive requests: {d}\n", .{self.http_client.active});
             var n_ = self.http_client.handles.in_use.first;
             while (n_) |n| {
-                const transfer = HttpClient.Transfer.fromEasy(n.data.conn.easy) catch |err| {
+                const transfer = Http.Transfer.fromEasy(n.data.conn.easy) catch |err| {
                     std.debug.print(" - failed to load transfer: {any}\n", .{err});
                     break;
                 };
@@ -435,7 +470,7 @@ pub const Page = struct {
         is_http: bool = true,
         is_navigation: bool = false,
     };
-    pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie {
+    pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) Http.Client.RequestCookie {
         return .{
             .jar = self.cookie_jar,
             .origin = &self.url.uri,
@@ -473,7 +508,7 @@ pub const Page = struct {
         const owned_url = try self.arena.dupeZ(u8, request_url);
         self.url = try URL.parse(owned_url, null);
 
-        var headers = try HttpClient.Headers.init();
+        var headers = try Http.Headers.init();
         if (opts.header) |hdr| try headers.add(hdr);
         try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, owned_url, &headers);
 
@@ -484,6 +519,7 @@ pub const Page = struct {
             .headers = headers,
             .body = opts.body,
             .cookie_jar = self.cookie_jar,
+            .resource_type = .document,
             .header_done_callback = pageHeaderDoneCallback,
             .data_callback = pageDataCallback,
             .done_callback = pageDoneCallback,
@@ -563,7 +599,7 @@ pub const Page = struct {
         );
     }
 
-    fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !void {
+    fn pageHeaderDoneCallback(transfer: *Http.Transfer) !void {
         var self: *Page = @alignCast(@ptrCast(transfer.ctx));
 
         // would be different than self.url in the case of a redirect
@@ -578,7 +614,7 @@ pub const Page = struct {
         });
     }
 
-    fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn pageDataCallback(transfer: *Http.Transfer, data: []const u8) !void {
         var self: *Page = @alignCast(@ptrCast(transfer.ctx));
 
         if (self.mode == .pre) {
@@ -1002,7 +1038,7 @@ pub const NavigateReason = enum {
 pub const NavigateOpts = struct {
     cdp_id: ?i64 = null,
     reason: NavigateReason = .address_bar,
-    method: HttpClient.Method = .GET,
+    method: Http.Method = .GET,
     body: ?[]const u8 = null,
     header: ?[:0]const u8 = null,
 };

src/browser/xhr/xhr.zig

@@ -30,7 +30,7 @@ const URL = @import("../../url.zig").URL;
 const Mime = @import("../mime.zig").Mime;
 const parser = @import("../netsurf.zig");
 const Page = @import("../page.zig").Page;
-const HttpClient = @import("../../http/Client.zig");
+const Http = @import("../../http/Http.zig");
 const CookieJar = @import("../storage/storage.zig").CookieJar;
 
 // XHR interfaces
@@ -80,12 +80,12 @@ const XMLHttpRequestBodyInit = union(enum) {
 pub const XMLHttpRequest = struct {
     proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
     arena: Allocator,
-    transfer: ?*HttpClient.Transfer = null,
+    transfer: ?*Http.Transfer = null,
     err: ?anyerror = null,
     last_dispatch: i64 = 0,
     send_flag: bool = false,
 
-    method: HttpClient.Method,
+    method: Http.Method,
     state: State,
     url: ?[:0]const u8 = null,
 
@@ -320,7 +320,7 @@ pub const XMLHttpRequest = struct {
     }
 
     const methods = [_]struct {
-        tag: HttpClient.Method,
+        tag: Http.Method,
         name: []const u8,
     }{
         .{ .tag = .DELETE, .name = "DELETE" },
@@ -330,7 +330,7 @@ pub const XMLHttpRequest = struct {
         .{ .tag = .POST, .name = "POST" },
         .{ .tag = .PUT, .name = "PUT" },
     };
-    pub fn validMethod(m: []const u8) DOMError!HttpClient.Method {
+    pub fn validMethod(m: []const u8) DOMError!Http.Method {
         for (methods) |method| {
             if (std.ascii.eqlIgnoreCase(method.name, m)) {
                 return method.tag;
@@ -370,7 +370,7 @@ pub const XMLHttpRequest = struct {
             }
         }
 
-        var headers = try HttpClient.Headers.init();
+        var headers = try Http.Headers.init();
         for (self.headers.items) |hdr| {
             try headers.add(hdr);
         }
@@ -383,6 +383,7 @@ pub const XMLHttpRequest = struct {
             .headers = headers,
             .body = self.request_body,
             .cookie_jar = page.cookie_jar,
+            .resource_type = .xhr,
             .start_callback = httpStartCallback,
             .header_callback = httpHeaderCallback,
             .header_done_callback = httpHeaderDoneCallback,
@@ -392,18 +393,19 @@ pub const XMLHttpRequest = struct {
         });
     }
 
-    fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
+    fn httpStartCallback(transfer: *Http.Transfer) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
         log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" });
         self.transfer = transfer;
     }
 
-    fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: []const u8) !void {
+    fn httpHeaderCallback(transfer: *Http.Transfer, header: Http.Header) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
-        try self.response_headers.append(self.arena, try self.arena.dupe(u8, header));
+        const joined = try std.fmt.allocPrint(self.arena, "{s}: {s}", .{ header.name, header.value });
+        try self.response_headers.append(self.arena, joined);
     }
 
-    fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !void {
+    fn httpHeaderDoneCallback(transfer: *Http.Transfer) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
 
         const header = &transfer.response_header.?;
@@ -433,7 +435,7 @@ pub const XMLHttpRequest = struct {
         self.dispatchEvt("readystatechange");
     }
 
-    fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
+    fn httpDataCallback(transfer: *Http.Transfer, data: []const u8) !void {
         const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
         try self.response_bytes.appendSlice(self.arena, data);
 

src/cdp/cdp.zig

@@ -74,11 +74,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
         // Used for processing notifications within a browser context.
         notification_arena: std.heap.ArenaAllocator,
 
-        // Extra headers to add to all requests. TBD under which conditions this should be reset.
-        extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty,
-
-        intercept_state: InterceptState,
-
         const Self = @This();
 
         pub fn init(app: *App, client: TypeProvider.Client) !Self {
@@ -93,7 +88,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
                 .browser_context = null,
                 .message_arena = std.heap.ArenaAllocator.init(allocator),
                 .notification_arena = std.heap.ArenaAllocator.init(allocator),
-                .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena?
             };
         }
 
@@ -101,7 +95,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
             if (self.browser_context) |*bc| {
                 bc.deinit();
             }
-            self.intercept_state.deinit(); // TBD Should this live in BC?
             self.browser.deinit();
             self.message_arena.deinit();
             self.notification_arena.deinit();
@@ -346,6 +339,11 @@ pub fn BrowserContext(comptime CDP_T: type) type {
 
         http_proxy_changed: bool = false,
 
+        // Extra headers to add to all requests.
+        extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty,
+
+        intercept_state: InterceptState,
+
         const Self = @This();
 
         fn init(self: *Self, id: []const u8, cdp: *CDP_T) !void {
@@ -375,6 +373,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
                 .isolated_world = null,
                 .inspector = inspector,
                 .notification_arena = cdp.notification_arena.allocator(),
+                .intercept_state = try InterceptState.init(allocator),
             };
             self.node_search_list = Node.Search.List.init(allocator, &self.node_registry);
             errdefer self.deinit();
@@ -388,6 +387,12 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         pub fn deinit(self: *Self) void {
             self.inspector.deinit();
 
+            // abort all intercepted requests before closing the sesion/page
+            // since some of these might callback into the page/scriptmanager
+            for (self.intercept_state.pendingTransfers()) |transfer| {
+                transfer.abort();
+            }
+
             // If the session has a page, we need to clear it first. The page
             // context is always nested inside of the isolated world context,
             // so we need to shutdown the page one first.
@@ -407,6 +412,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
                     log.warn(.http, "restoreOriginalProxy", .{ .err = err });
                 };
             }
+            self.intercept_state.deinit();
         }
 
         pub fn reset(self: *Self) void {
@@ -495,7 +501,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
         pub fn onHttpRequestIntercept(ctx: *anyopaque, data: *const Notification.RequestIntercept) !void {
             const self: *Self = @alignCast(@ptrCast(ctx));
             defer self.resetNotificationArena();
-            try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data);
+            try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, data);
         }
 
         pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void {

src/cdp/domains/dom.zig

@@ -304,7 +304,7 @@ fn describeNode(cmd: anytype) !void {
         pierce: bool = false,
     })) orelse return error.InvalidParams;
 
-    if (params.depth != 1 or params.pierce) return error.NotYetImplementedParams;
+    if (params.depth != 1 or params.pierce) return error.NotImplemented;
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
 
     const node = try getNode(cmd.arena, bc, params.nodeId, params.backendNodeId, params.objectId);

src/cdp/domains/fetch.zig

@@ -18,10 +18,12 @@
 
 const std = @import("std");
 const Allocator = std.mem.Allocator;
-const Notification = @import("../../notification.zig").Notification;
+
 const log = @import("../../log.zig");
-const Method = @import("../../http/Client.zig").Method;
+const network = @import("network.zig");
-const Transfer = @import("../../http/Client.zig").Transfer;
+
+const Http = @import("../../http/Http.zig");
+const Notification = @import("../../notification.zig").Notification;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
@@ -29,6 +31,7 @@ pub fn processMessage(cmd: anytype) !void {
         enable,
         continueRequest,
         failRequest,
+        fulfillRequest,
     }, cmd.input.action) orelse return error.UnknownMethod;
 
     switch (action) {
@@ -36,27 +39,48 @@ pub fn processMessage(cmd: anytype) !void {
         .enable => return enable(cmd),
         .continueRequest => return continueRequest(cmd),
         .failRequest => return failRequest(cmd),
+        .fulfillRequest => return fulfillRequest(cmd),
     }
 }
 
 // Stored in CDP
 pub const InterceptState = struct {
-    const Self = @This();
+    allocator: Allocator,
-    waiting: std.AutoArrayHashMap(u64, *Transfer),
+    waiting: std.AutoArrayHashMapUnmanaged(u64, *Http.Transfer),
 
     pub fn init(allocator: Allocator) !InterceptState {
         return .{
-            .waiting = std.AutoArrayHashMap(u64, *Transfer).init(allocator),
+            .waiting = .empty,
+            .allocator = allocator,
         };
     }
 
-    pub fn deinit(self: *Self) void {
+    pub fn empty(self: *const InterceptState) bool {
-        self.waiting.deinit();
+        return self.waiting.count() == 0;
+    }
+
+    pub fn put(self: *InterceptState, transfer: *Http.Transfer) !void {
+        return self.waiting.put(self.allocator, transfer.id, transfer);
+    }
+
+    pub fn remove(self: *InterceptState, id: u64) ?*Http.Transfer {
+        const entry = self.waiting.fetchSwapRemove(id) orelse return null;
+        return entry.value;
+    }
+
+    pub fn deinit(self: *InterceptState) void {
+        self.waiting.deinit(self.allocator);
+    }
+
+    pub fn pendingTransfers(self: *const InterceptState) []*Http.Transfer {
+        return self.waiting.values();
     }
 };
 
 const RequestPattern = struct {
-    urlPattern: []const u8 = "*", // Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed. Escape character is backslash. Omitting is equivalent to "*".
+    // Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed.
+    // Escape character is backslash. Omitting is equivalent to "*".
+    urlPattern: []const u8 = "*",
     resourceType: ?ResourceType = null,
     requestStage: RequestStage = .Request,
 };
@@ -115,8 +139,14 @@ fn disable(cmd: anytype) !void {
 
 fn enable(cmd: anytype) !void {
     const params = (try cmd.params(EnableParam)) orelse EnableParam{};
-    if (params.patterns.len != 0) log.warn(.cdp, "Fetch.enable No patterns yet", .{});
+    if (!arePatternsSupported(params.patterns)) {
-    if (params.handleAuthRequests) log.warn(.cdp, "Fetch.enable No auth yet", .{});
+        log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable advanced patterns are not" });
+        return cmd.sendResult(null, .{});
+    }
+
+    if (params.handleAuthRequests) {
+        log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable handleAuthRequests is not supported yet" });
+    }
 
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
     try bc.fetchEnable();
@@ -124,57 +154,67 @@ fn enable(cmd: anytype) !void {
     return cmd.sendResult(null, .{});
 }
 
-pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void {
+fn arePatternsSupported(patterns: []RequestPattern) bool {
-    var cdp = bc.cdp;
+    if (patterns.len == 0) {
+        return true;
+    }
+    if (patterns.len > 1) {
+        return false;
+    }
 
+    // While we don't support patterns, yet, both Playwright and Puppeteer send
+    // a default pattern which happens to be what we support:
+    // [{"urlPattern":"*","requestStage":"Request"}]
+    // So, rather than erroring on this case because we don't support patterns,
+    // we'll allow it, because this pattern is how it works as-is.
+    const pattern = patterns[0];
+    if (!std.mem.eql(u8, pattern.urlPattern, "*")) {
+        return false;
+    }
+    if (pattern.resourceType != null) {
+        return false;
+    }
+    if (pattern.requestStage != .Request) {
+        return false;
+    }
+    return true;
+}
+
+pub fn requestIntercept(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void {
     // unreachable because we _have_ to have a page.
     const session_id = bc.session_id orelse unreachable;
     const target_id = bc.target_id orelse unreachable;
+    const page = bc.session.currentPage() orelse unreachable;
 
     // We keep it around to wait for modifications to the request.
     // NOTE: we assume whomever created the request created it with a lifetime of the Page.
     // TODO: What to do when receiving replies for a previous page's requests?
 
     const transfer = intercept.transfer;
-    try cdp.intercept_state.waiting.put(transfer.id, transfer);
+    try bc.intercept_state.put(transfer);
 
-    // NOTE: .request data preparation is duped from network.zig
+    try bc.cdp.sendEvent("Fetch.requestPaused", .{
-    const full_request_url = transfer.uri;
-    const request_url = try @import("network.zig").urlToString(arena, &full_request_url, .{
-        .scheme = true,
-        .authentication = true,
-        .authority = true,
-        .path = true,
-        .query = true,
-    });
-    const request_fragment = try @import("network.zig").urlToString(arena, &full_request_url, .{
-        .fragment = true,
-    });
-    const headers = try transfer.req.headers.asHashMap(arena);
-    // End of duped code
-
-    try cdp.sendEvent("Fetch.requestPaused", .{
         .requestId = try std.fmt.allocPrint(arena, "INTERCEPT-{d}", .{transfer.id}),
-        .request = .{
+        .request = network.TransferAsRequestWriter.init(transfer),
-            .url = request_url,
-            .urlFragment = request_fragment,
-            .method = @tagName(transfer.req.method),
-            .hasPostData = transfer.req.body != null,
-            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
-        },
         .frameId = target_id,
-        .resourceType = ResourceType.Document, //  TODO!
+        .resourceType = switch (transfer.req.resource_type) {
+            .script => "Script",
+            .xhr => "XHR",
+            .document => "Document",
+        },
         .networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}),
     }, .{ .session_id = session_id });
 
+    log.debug(.cdp, "request intercept", .{
+        .state = "paused",
+        .id = transfer.id,
+        .url = transfer.uri,
+    });
     // Await either continueRequest, failRequest or fulfillRequest
-    intercept.wait_for_interception.* = true;
-}
 
-const HeaderEntry = struct {
+    intercept.wait_for_interception.* = true;
-    name: []const u8,
+    page.request_intercepted = true;
-    value: []const u8,
+}
-};
 
 fn continueRequest(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
@@ -183,49 +223,129 @@ fn continueRequest(cmd: anytype) !void {
         url: ?[]const u8 = null,
         method: ?[]const u8 = null,
         postData: ?[]const u8 = null,
-        headers: ?[]const HeaderEntry = null,
+        headers: ?[]const Http.Header = null,
         interceptResponse: bool = false,
     })) orelse return error.InvalidParams;
-    if (params.postData != null or params.headers != null or params.interceptResponse) return error.NotYetImplementedParams;
 
+    if (params.interceptResponse) {
+        return error.NotImplemented;
+    }
+
+    const page = bc.session.currentPage() orelse return error.PageNotLoaded;
+
+    var intercept_state = &bc.intercept_state;
     const request_id = try idFromRequestId(params.requestId);
-    const entry = bc.cdp.intercept_state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound;
+    const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
-    const transfer = entry.value;
+
+    log.debug(.cdp, "request intercept", .{
+        .state = "contiune",
+        .id = transfer.id,
+        .url = transfer.uri,
+        .new_url = params.url,
+    });
 
     // Update the request with the new parameters
     if (params.url) |url| {
-        // The request url must be modified in a way that's not observable by page. So page.url is not updated.
+        try transfer.updateURL(try page.arena.dupeZ(u8, url));
-        try transfer.updateURL(try bc.cdp.browser.page_arena.allocator().dupeZ(u8, url));
     }
     if (params.method) |method| {
-        transfer.req.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams;
+        transfer.req.method = std.meta.stringToEnum(Http.Method, method) orelse return error.InvalidParams;
+    }
+
+    if (params.headers) |headers| {
+        try transfer.replaceRequestHeaders(cmd.arena, headers);
+    }
+
+    if (params.postData) |b| {
+        const decoder = std.base64.standard.Decoder;
+        const body = try bc.arena.alloc(u8, try decoder.calcSizeForSlice(b));
+        try decoder.decode(body, b);
+        transfer.req.body = body;
     }
 
-    log.info(.cdp, "Request continued by intercept", .{ .id = params.requestId });
     try bc.cdp.browser.http_client.process(transfer);
 
+    if (intercept_state.empty()) {
+        page.request_intercepted = false;
+    }
+
+    return cmd.sendResult(null, .{});
+}
+
+fn fulfillRequest(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+
+    const params = (try cmd.params(struct {
+        requestId: []const u8, // "INTERCEPT-{d}"
+        responseCode: u16,
+        responseHeaders: ?[]const Http.Header = null,
+        binaryResponseHeaders: ?[]const u8 = null,
+        body: ?[]const u8 = null,
+        responsePhrase: ?[]const u8 = null,
+    })) orelse return error.InvalidParams;
+
+    if (params.binaryResponseHeaders != null) {
+        log.warn(.cdp, "not implemented", .{ .feature = "Fetch.fulfillRequest binaryResponseHeade" });
+        return error.NotImplemented;
+    }
+
+    var intercept_state = &bc.intercept_state;
+    const request_id = try idFromRequestId(params.requestId);
+    const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
+
+    log.debug(.cdp, "request intercept", .{
+        .state = "fulfilled",
+        .id = transfer.id,
+        .url = transfer.uri,
+        .status = params.responseCode,
+        .body = params.body != null,
+    });
+
+    var body: ?[]const u8 = null;
+    if (params.body) |b| {
+        const decoder = std.base64.standard.Decoder;
+        const buf = try cmd.arena.alloc(u8, try decoder.calcSizeForSlice(b));
+        try decoder.decode(buf, b);
+        body = buf;
+    }
+
+    try transfer.fulfill(params.responseCode, params.responseHeaders orelse &.{}, body);
+
     return cmd.sendResult(null, .{});
 }
 
 fn failRequest(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
-    var state = &bc.cdp.intercept_state;
     const params = (try cmd.params(struct {
         requestId: []const u8, // "INTERCEPT-{d}"
         errorReason: ErrorReason,
     })) orelse return error.InvalidParams;
 
-    const request_id = try idFromRequestId(params.requestId);
+    const page = bc.session.currentPage() orelse return error.PageNotLoaded;
-    const entry = state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound;
-    // entry.value is the transfer
-    entry.value.abort();
 
-    log.info(.cdp, "Request aborted by intercept", .{ .reason = params.errorReason });
+    var intercept_state = &bc.intercept_state;
+    const request_id = try idFromRequestId(params.requestId);
+
+    const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
+    defer transfer.abort();
+
+    log.info(.cdp, "request intercept", .{
+        .state = "fail",
+        .id = request_id,
+        .url = transfer.uri,
+        .reason = params.errorReason,
+    });
+
+    if (intercept_state.empty()) {
+        page.request_intercepted = false;
+    }
     return cmd.sendResult(null, .{});
 }
 
 // Get u64 from requestId which is formatted as: "INTERCEPT-{d}"
 fn idFromRequestId(request_id: []const u8) !u64 {
-    if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) return error.InvalidParams;
+    if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) {
+        return error.InvalidParams;
+    }
     return std.fmt.parseInt(u64, request_id[10..], 10) catch return error.InvalidParams;
 }

src/cdp/domains/network.zig

@@ -19,10 +19,10 @@
 const std = @import("std");
 const Allocator = std.mem.Allocator;
 
-const Notification = @import("../../notification.zig").Notification;
 const log = @import("../../log.zig");
 const CdpStorage = @import("storage.zig");
 const Transfer = @import("../../http/Client.zig").Transfer;
+const Notification = @import("../../notification.zig").Notification;
 
 pub fn processMessage(cmd: anytype) !void {
     const action = std.meta.stringToEnum(enum {
@@ -83,7 +83,7 @@ fn setExtraHTTPHeaders(cmd: anytype) !void {
 
     // Copy the headers onto the browser context arena
     const arena = bc.arena;
-    const extra_headers = &bc.cdp.extra_headers;
+    const extra_headers = &bc.extra_headers;
 
     extra_headers.clearRetainingCapacity();
     try extra_headers.ensureTotalCapacity(arena, params.headers.map.count());
@@ -121,7 +121,7 @@ fn deleteCookies(cmd: anytype) !void {
         path: ?[]const u8 = null,
         partitionKey: ?CdpStorage.CookiePartitionKey = null,
     })) orelse return error.InvalidParams;
-    if (params.partitionKey != null) return error.NotYetImplementedParams;
+    if (params.partitionKey != null) return error.NotImplemented;
 
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
     const cookies = &bc.session.cookie_jar.cookies;
@@ -235,50 +235,16 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification
     const page = bc.session.currentPage() orelse unreachable;
 
     // Modify request with extra CDP headers
-    for (cdp.extra_headers.items) |extra| {
+    for (bc.extra_headers.items) |extra| {
         try data.transfer.req.headers.add(extra);
     }
 
-    const document_url = try urlToString(arena, &page.url.uri, .{
-        .scheme = true,
-        .authentication = true,
-        .authority = true,
-        .path = true,
-        .query = true,
-    });
-
     const transfer = data.transfer;
-    const full_request_url = transfer.uri;
-    const request_url = try urlToString(arena, &full_request_url, .{
-        .scheme = true,
-        .authentication = true,
-        .authority = true,
-        .path = true,
-        .query = true,
-    });
-    const request_fragment = try urlToString(arena, &full_request_url, .{
-        .fragment = true, // TODO since path is false, this likely does not work as intended
-    });
-
-    const headers = try transfer.req.headers.asHashMap(arena);
-
     // We're missing a bunch of fields, but, for now, this seems like enough
-    try cdp.sendEvent("Network.requestWillBeSent", .{
+    try cdp.sendEvent("Network.requestWillBeSent", .{ .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), .frameId = target_id, .loaderId = bc.loader_id, .documentUrl = DocumentUrlWriter.init(&page.url.uri), .request = TransferAsRequestWriter.init(transfer) }, .{ .session_id = session_id });
-        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}),
-        .frameId = target_id,
-        .loaderId = bc.loader_id,
-        .documentUrl = document_url,
-        .request = .{
-            .url = request_url,
-            .urlFragment = request_fragment,
-            .method = @tagName(transfer.req.method),
-            .hasPostData = transfer.req.body != null,
-            .headers = std.json.ArrayHashMap([]const u8){ .map = headers },
-        },
-    }, .{ .session_id = session_id });
 }
 
-pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void {
+pub fn httpHeadersDone(arena: Allocator, bc: anytype, data: *const Notification.ResponseHeadersDone) !void {
     // Isn't possible to do a network request within a Browser (which our
     // notification is tied to), without a page.
     std.debug.assert(bc.session.page != null);
@@ -289,54 +255,157 @@ pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notificati
     const session_id = bc.session_id orelse unreachable;
     const target_id = bc.target_id orelse unreachable;
 
-    const url = try urlToString(arena, &request.transfer.uri, .{
-        .scheme = true,
-        .authentication = true,
-        .authority = true,
-        .path = true,
-        .query = true,
-    });
-
-    const status = request.transfer.response_header.?.status;
-
     // We're missing a bunch of fields, but, for now, this seems like enough
     try cdp.sendEvent("Network.responseReceived", .{
-        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.transfer.id}),
+        .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}),
         .loaderId = bc.loader_id,
-        .response = .{
-            .url = url,
-            .status = status,
-            .statusText = @as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown",
-            .headers = ResponseHeaderWriter.init(request.transfer),
-        },
         .frameId = target_id,
+        .response = TransferAsResponseWriter.init(data.transfer),
     }, .{ .session_id = session_id });
 }
 
-pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 {
+pub const TransferAsRequestWriter = struct {
-    var buf: std.ArrayListUnmanaged(u8) = .empty;
-    try url.writeToStream(opts, buf.writer(arena));
-    return buf.items;
-}
-
-const ResponseHeaderWriter = struct {
     transfer: *Transfer,
 
-    fn init(transfer: *Transfer) ResponseHeaderWriter {
+    pub fn init(transfer: *Transfer) TransferAsRequestWriter {
         return .{
             .transfer = transfer,
         };
     }
 
-    pub fn jsonStringify(self: *const ResponseHeaderWriter, writer: anytype) !void {
+    pub fn jsonStringify(self: *const TransferAsRequestWriter, writer: anytype) !void {
+        const stream = writer.stream;
+        const transfer = self.transfer;
+
         try writer.beginObject();
-        var it = self.transfer.responseHeaderIterator();
+        {
+            try writer.objectField("url");
+            try writer.beginWriteRaw();
+            try stream.writeByte('\"');
+            try transfer.uri.writeToStream(.{
+                .scheme = true,
+                .authentication = true,
+                .authority = true,
+                .path = true,
+                .query = true,
+            }, stream);
+            try stream.writeByte('\"');
+            writer.endWriteRaw();
+        }
+
+        {
+            if (transfer.uri.fragment) |frag| {
+                try writer.objectField("urlFragment");
+                try writer.beginWriteRaw();
+                try stream.writeAll("\"#");
+                try stream.writeAll(frag.percent_encoded);
+                try stream.writeByte('\"');
+                writer.endWriteRaw();
+            }
+        }
+
+        {
+            try writer.objectField("method");
+            try writer.write(@tagName(transfer.req.method));
+        }
+
+        {
+            try writer.objectField("hasPostData");
+            try writer.write(transfer.req.body != null);
+        }
+
+        {
+            try writer.objectField("headers");
+            try writer.beginObject();
+            var it = transfer.req.headers.iterator();
             while (it.next()) |hdr| {
                 try writer.objectField(hdr.name);
                 try writer.write(hdr.value);
             }
             try writer.endObject();
         }
+        try writer.endObject();
+    }
+};
+
+const TransferAsResponseWriter = struct {
+    transfer: *Transfer,
+
+    fn init(transfer: *Transfer) TransferAsResponseWriter {
+        return .{
+            .transfer = transfer,
+        };
+    }
+
+    pub fn jsonStringify(self: *const TransferAsResponseWriter, writer: anytype) !void {
+        const stream = writer.stream;
+        const transfer = self.transfer;
+
+        try writer.beginObject();
+        {
+            try writer.objectField("url");
+            try writer.beginWriteRaw();
+            try stream.writeByte('\"');
+            try transfer.uri.writeToStream(.{
+                .scheme = true,
+                .authentication = true,
+                .authority = true,
+                .path = true,
+                .query = true,
+            }, stream);
+            try stream.writeByte('\"');
+            writer.endWriteRaw();
+        }
+
+        if (transfer.response_header) |*rh| {
+            // it should not be possible for this to be false, but I'm not
+            // feeling brave today.
+            const status = rh.status;
+            try writer.objectField("status");
+            try writer.write(status);
+
+            try writer.objectField("statusText");
+            try writer.write(@as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown");
+        }
+
+        {
+            try writer.objectField("headers");
+            try writer.beginObject();
+            var it = transfer.responseHeaderIterator();
+            while (it.next()) |hdr| {
+                try writer.objectField(hdr.name);
+                try writer.write(hdr.value);
+            }
+            try writer.endObject();
+        }
+        try writer.endObject();
+    }
+};
+
+const DocumentUrlWriter = struct {
+    uri: *std.Uri,
+
+    fn init(uri: *std.Uri) DocumentUrlWriter {
+        return .{
+            .uri = uri,
+        };
+    }
+
+    pub fn jsonStringify(self: *const DocumentUrlWriter, writer: anytype) !void {
+        const stream = writer.stream;
+
+        try writer.beginWriteRaw();
+        try stream.writeByte('\"');
+        try self.uri.writeToStream(.{
+            .scheme = true,
+            .authentication = true,
+            .authority = true,
+            .path = true,
+            .query = true,
+        }, stream);
+        try stream.writeByte('\"');
+        writer.endWriteRaw();
+    }
 };
 
 const testing = @import("../testing.zig");
@@ -344,8 +413,8 @@ test "cdp.network setExtraHTTPHeaders" {
     var ctx = testing.context();
     defer ctx.deinit();
 
-    // _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" });
+    _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" });
-    try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } });
+    // try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } });
 
     try ctx.processMessage(.{
         .id = 3,
@@ -360,10 +429,7 @@ test "cdp.network setExtraHTTPHeaders" {
     });
 
     const bc = ctx.cdp().browser_context.?;
-    try testing.expectEqual(bc.cdp.extra_headers.items.len, 1);
+    try testing.expectEqual(bc.extra_headers.items.len, 1);
-
-    try ctx.processMessage(.{ .id = 5, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
-    try testing.expectEqual(bc.cdp.extra_headers.items.len, 0);
 }
 
 test "cdp.Network: cookies" {

src/cdp/domains/storage.zig

@@ -129,7 +129,7 @@ pub const CdpCookie = struct {
 
 pub fn setCdpCookie(cookie_jar: *CookieJar, param: CdpCookie) !void {
     if (param.priority != .Medium or param.sameParty != null or param.sourceScheme != null or param.partitionKey != null) {
-        return error.NotYetImplementedParams;
+        return error.NotImplemented;
     }
 
     var arena = std.heap.ArenaAllocator.init(cookie_jar.allocator);

src/cdp/domains/target.zig

@@ -73,7 +73,9 @@ fn createBrowserContext(cmd: anytype) !void {
         originsWithUniversalNetworkAccess: ?[]const []const u8 = null,
     });
     if (params) |p| {
-        if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) std.debug.print("Target.createBrowserContext: Not implemented param set\n", .{});
+        if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) {
+            log.warn(.cdp, "not implemented", .{ .feature = "Target.createBrowserContext: Not implemented param set" });
+        }
     }
 
     const bc = cmd.createBrowserContext() catch |err| switch (err) {
@@ -407,8 +409,9 @@ fn doAttachtoTarget(cmd: anytype, target_id: []const u8) !void {
     std.debug.assert(bc.session_id == null);
     const session_id = cmd.cdp.session_id_gen.next();
 
-    // extra_headers should not be kept on a new page or tab, currently we have only 1 page, we clear it just in case
+    // extra_headers should not be kept on a new page or tab,
-    bc.cdp.extra_headers.clearRetainingCapacity();
+    // currently we have only 1 page, we clear it just in case
+    bc.extra_headers.clearRetainingCapacity();
 
     try cmd.sendEvent("Target.attachedToTarget", AttachToTarget{
         .sessionId = session_id,

src/http/Client.zig

@@ -19,10 +19,10 @@
 const std = @import("std");
 const log = @import("../log.zig");
 const builtin = @import("builtin");
+
 const Http = @import("Http.zig");
-pub const Headers = Http.Headers;
 const Notification = @import("../notification.zig").Notification;
-const storage = @import("../browser/storage/storage.zig");
+const CookieJar = @import("../browser/storage/storage.zig").CookieJar;
 
 const urlStitch = @import("../url.zig").stitch;
 
@@ -34,7 +34,7 @@ const ArenaAllocator = std.heap.ArenaAllocator;
 const errorCheck = Http.errorCheck;
 const errorMCheck = Http.errorMCheck;
 
-pub const Method = Http.Method;
+const Method = Http.Method;
 
 // This is loosely tied to a browser Page. Loading all the <scripts>, doing
 // XHR requests, and loading imports all happens through here. Sine the app
@@ -309,6 +309,10 @@ fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) !void {
         try conn.setMethod(req.method);
         if (req.body) |b| {
             try conn.setBody(b);
+        } else if (req.method == .POST) {
+            // libcurl will crash if the method is POST but there's no body
+            // TODO: is there a setting for that..seems weird.
+            try conn.setBody("");
         }
 
         var header_list = req.headers;
@@ -496,7 +500,7 @@ pub const RequestCookie = struct {
     origin: *const std.Uri,
     jar: *@import("../browser/storage/cookie.zig").Jar,
 
-    pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Headers) !void {
+    pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Http.Headers) !void {
         const uri = std.Uri.parse(url) catch |err| {
             log.warn(.http, "invalid url", .{ .err = err, .url = url });
             return error.InvalidUrl;
@@ -519,19 +523,26 @@ pub const RequestCookie = struct {
 pub const Request = struct {
     method: Method,
     url: [:0]const u8,
-    headers: Headers,
+    headers: Http.Headers,
     body: ?[]const u8 = null,
-    cookie_jar: *storage.CookieJar,
+    cookie_jar: *CookieJar,
+    resource_type: ResourceType,
 
     // arbitrary data that can be associated with this request
     ctx: *anyopaque = undefined,
 
     start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null,
-    header_callback: ?*const fn (transfer: *Transfer, header: []const u8) anyerror!void = null,
+    header_callback: ?*const fn (transfer: *Transfer, header: Http.Header) anyerror!void = null,
     header_done_callback: *const fn (transfer: *Transfer) anyerror!void,
     data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void,
     done_callback: *const fn (ctx: *anyopaque) anyerror!void,
     error_callback: *const fn (ctx: *anyopaque, err: anyerror) void,
+
+    const ResourceType = enum {
+        document,
+        xhr,
+        script,
+    };
 };
 
 pub const Transfer = struct {
@@ -544,7 +555,7 @@ pub const Transfer = struct {
     _notified_fail: bool = false,
 
     // We'll store the response header here
-    response_header: ?Header = null,
+    response_header: ?ResponseHeader = null,
 
     _handle: ?*Handle = null,
 
@@ -582,6 +593,22 @@ pub const Transfer = struct {
         self.req.url = url;
     }
 
+    pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const Http.Header) !void {
+        self.req.headers.deinit();
+
+        var buf: std.ArrayListUnmanaged(u8) = .empty;
+        var new_headers = try Http.Headers.init();
+        for (headers) |hdr| {
+            // safe to re-use this buffer, because Headers.add because curl copies
+            // the value we pass into curl_slist_append.
+            defer buf.clearRetainingCapacity();
+            try std.fmt.format(buf.writer(allocator), "{s}: {s}", .{ hdr.name, hdr.value });
+            try buf.append(allocator, 0); // null terminated
+            try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);
+        }
+        self.req.headers = new_headers;
+    }
+
     pub fn abort(self: *Transfer) void {
         self.client.requestFailed(self, error.Abort);
         if (self._handle != null) {
@@ -697,7 +724,7 @@ pub const Transfer = struct {
             if (getResponseHeader(easy, "content-type", 0)) |ct| {
                 var hdr = &transfer.response_header.?;
                 const value = ct.value;
-                const len = @min(value.len, hdr._content_type.len);
+                const len = @min(value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN);
                 hdr._content_type_len = len;
                 @memcpy(hdr._content_type[0..len], value[0..len]);
             }
@@ -726,12 +753,14 @@ pub const Transfer = struct {
             }
         } else {
             if (transfer.req.header_callback) |cb| {
-                cb(transfer, header) catch |err| {
+                if (Http.Headers.parseHeader(header)) |hdr| {
+                    cb(transfer, hdr) catch |err| {
                         log.err(.http, "header_callback", .{ .err = err, .req = transfer });
                         return 0;
                     };
                 }
             }
+        }
         return buf_len;
     }
 
@@ -768,15 +797,64 @@ pub const Transfer = struct {
         try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_PRIVATE, &private));
         return @alignCast(@ptrCast(private));
     }
+
+    pub fn fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void {
+        if (transfer._handle != null) {
+            // should never happen, should have been intercepted/paused, and then
+            // either continued, aborted and fulfilled once.
+            @branchHint(.unlikely);
+            return error.RequestInProgress;
+        }
+
+        transfer._fulfill(status, headers, body) catch |err| {
+            transfer.req.error_callback(transfer.req.ctx, err);
+            return err;
+        };
+    }
+
+    fn _fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void {
+        const req = &transfer.req;
+        if (req.start_callback) |cb| {
+            try cb(transfer);
+        }
+
+        if (req.header_callback) |cb| {
+            for (headers) |hdr| {
+                try cb(transfer, hdr);
+            }
+        }
+
+        transfer.response_header = .{
+            .status = status,
+            .url = req.url,
+        };
+        for (headers) |hdr| {
+            if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) {
+                const len = @min(hdr.value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN);
+                @memcpy(transfer.response_header.?._content_type[0..len], hdr.value[0..len]);
+                break;
+            }
+        }
+
+        try req.header_done_callback(transfer);
+
+        if (body) |b| {
+            try req.data_callback(transfer, b);
+        }
+
+        try req.done_callback(req.ctx);
+    }
 };
 
-pub const Header = struct {
+pub const ResponseHeader = struct {
+    const MAX_CONTENT_TYPE_LEN = 64;
+
     status: u16,
     url: [*c]const u8,
     _content_type_len: usize = 0,
-    _content_type: [64]u8 = undefined,
+    _content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined,
 
-    pub fn contentType(self: *Header) ?[]u8 {
+    pub fn contentType(self: *ResponseHeader) ?[]u8 {
         if (self._content_type_len == 0) {
             return null;
         }
@@ -788,7 +866,7 @@ const HeaderIterator = struct {
     easy: *c.CURL,
     prev: ?*c.curl_header = null,
 
-    pub fn next(self: *HeaderIterator) ?struct { name: []const u8, value: []const u8 } {
+    pub fn next(self: *HeaderIterator) ?Http.Header {
         const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null;
         self.prev = h;
 
@@ -800,12 +878,12 @@ const HeaderIterator = struct {
     }
 };
 
-const ResponseHeader = struct {
+const CurlHeaderValue = struct {
     value: []const u8,
     amount: usize,
 };
 
-fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?ResponseHeader {
+fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?CurlHeaderValue {
     var hdr: [*c]c.curl_header = null;
     const result = c.curl_easy_header(easy, name, index, c.CURLH_HEADER, -1, &hdr);
     if (result == c.CURLE_OK) {

src/http/Http.zig

@@ -22,14 +22,15 @@ pub const c = @cImport({
     @cInclude("curl/curl.h");
 });
 
-const Client = @import("Client.zig");
+pub const ENABLE_DEBUG = false;
+pub const Client = @import("Client.zig");
+pub const Transfer = Client.Transfer;
+
 const errors = @import("errors.zig");
 
 const Allocator = std.mem.Allocator;
 const ArenaAllocator = std.heap.ArenaAllocator;
 
-pub const ENABLE_DEBUG = false;
-
 // Client.zig does the bulk of the work and is loosely tied to a browser Page.
 // But we still need something above Client.zig for the "utility" http stuff
 // we need to do, like telemetry. The most important thing we want from this
@@ -216,6 +217,11 @@ pub const Connection = struct {
     }
 };
 
+pub const Header = struct {
+    name: []const u8,
+    value: []const u8,
+};
+
 pub const Headers = struct {
     headers: *c.curl_slist,
     cookies: ?[*c]const u8,
@@ -237,25 +243,7 @@ pub const Headers = struct {
         self.headers = updated_headers;
     }
 
-    pub fn asHashMap(self: *const Headers, allocator: Allocator) !std.StringArrayHashMapUnmanaged([]const u8) {
+    pub fn parseHeader(header_str: []const u8) ?Header {
-        var list: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
-        try list.ensureTotalCapacity(allocator, self.count());
-
-        var current: [*c]c.curl_slist = self.headers;
-        while (current) |node| {
-            const str = std.mem.span(@as([*:0]const u8, @ptrCast(node.*.data)));
-            const header = parseHeader(str) orelse return error.InvalidHeader;
-            list.putAssumeCapacity(header.name, header.value);
-            current = node.*.next;
-        }
-        // special case for cookies
-        if (self.cookies) |v| {
-            list.putAssumeCapacity("Cookie", std.mem.span(@as([*:0]const u8, @ptrCast(v))));
-        }
-        return list;
-    }
-
-    pub fn parseHeader(header_str: []const u8) ?std.http.Header {
         const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
 
         const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
@@ -264,19 +252,28 @@ pub const Headers = struct {
         return .{ .name = name, .value = value };
     }
 
-    pub fn count(self: *const Headers) usize {
+    pub fn iterator(self: *Headers) Iterator {
-        var current: [*c]c.curl_slist = self.headers;
+        return .{
-        var num: usize = 0;
+            .header = self.headers,
-        while (current) |node| {
+            .cookies = self.cookies,
-            num += 1;
+        };
-            current = node.*.next;
     }
-        // special case for cookies
+
-        if (self.cookies != null) {
+    const Iterator = struct {
-            num += 1;
+        header: [*c]c.curl_slist,
-        }
+        cookies: ?[*c]const u8,
-        return num;
+
+        pub fn next(self: *Iterator) ?Header {
+            const h = self.header orelse {
+                const cookies = self.cookies orelse return null;
+                self.cookies = null;
+                return .{ .name = "Cookie", .value = std.mem.span(@as([*:0]const u8, cookies)) };
+            };
+
+            self.header = h.*.next;
+            return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data))));
         }
+    };
 };
 
 pub fn errorCheck(code: c.CURLcode) errors.Error!void {