diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 696448c2..1c0690f7 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -29,6 +29,7 @@ const Page = @import("../browser/page.zig").Page; const Inspector = @import("../browser/env.zig").Env.Inspector; const Incrementing = @import("../id.zig").Incrementing; const Notification = @import("../notification.zig").Notification; +const NetworkState = @import("domains/network.zig").NetworkState; const InterceptState = @import("domains/fetch.zig").InterceptState; const polyfill = @import("../browser/polyfill/polyfill.zig"); @@ -76,6 +77,7 @@ pub fn CDPT(comptime TypeProvider: type) type { // Extra headers to add to all requests. TBD under which conditions this should be reset. extra_headers: std.ArrayListUnmanaged(std.http.Header) = .empty, + network_state: NetworkState, intercept_state: InterceptState, const Self = @This(); @@ -92,6 +94,7 @@ pub fn CDPT(comptime TypeProvider: type) type { .browser_context = null, .message_arena = std.heap.ArenaAllocator.init(allocator), .notification_arena = std.heap.ArenaAllocator.init(allocator), + .network_state = try NetworkState.init(allocator), .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena? }; } @@ -101,6 +104,7 @@ pub fn CDPT(comptime TypeProvider: type) type { bc.deinit(); } self.intercept_state.deinit(); // TBD Should this live in BC? + self.network_state.deinit(); self.browser.deinit(); self.message_arena.deinit(); self.notification_arena.deinit(); @@ -447,13 +451,15 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn networkEnable(self: *Self) !void { try self.cdp.browser.notification.register(.http_request_fail, self, onHttpRequestFail); try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart); - try self.cdp.browser.notification.register(.http_request_complete, self, onHttpRequestComplete); + try self.cdp.browser.notification.register(.http_header_received, self, onHttpHeaderReceived); + try self.cdp.browser.notification.register(.http_headers_done_receiving, self, onHttpHeadersDoneReceiving); } pub fn networkDisable(self: *Self) void { self.cdp.browser.notification.unregister(.http_request_fail, self); self.cdp.browser.notification.unregister(.http_request_start, self); - self.cdp.browser.notification.unregister(.http_request_complete, self); + self.cdp.browser.notification.unregister(.http_header_received, self); + self.cdp.browser.notification.unregister(.http_headers_done_receiving, self); } pub fn fetchEnable(self: *Self) !void { @@ -466,7 +472,8 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void { const self: *Self = @alignCast(@ptrCast(ctx)); - return @import("domains/page.zig").pageRemove(self); + try @import("domains/page.zig").pageRemove(self); + try @import("domains/network.zig").pageRemove(self); } pub fn onPageCreated(ctx: *anyopaque, page: *Page) !void { @@ -497,16 +504,22 @@ pub fn BrowserContext(comptime CDP_T: type) type { try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data); } + pub fn onHttpHeaderReceived(ctx: *anyopaque, data: *const Notification.ResponseHeader) !void { + const self: *Self = @alignCast(@ptrCast(ctx)); + defer self.resetNotificationArena(); + try self.cdp.network_state.putOrAppendReceivedHeader(data.request_id, data.status, data.header); + } + pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, data); } - pub fn onHttpRequestComplete(ctx: *anyopaque, data: *const Notification.RequestComplete) !void { + pub fn onHttpHeadersDoneReceiving(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); - return @import("domains/network.zig").httpRequestComplete(self.notification_arena, self, data); + return @import("domains/network.zig").httpHeadersDoneReceiving(self.notification_arena, self, data); } fn resetNotificationArena(self: *Self) void { diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index befa824c..fe5ce21e 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -51,6 +51,49 @@ pub fn processMessage(cmd: anytype) !void { } } +const Response = struct { + status: u16, + headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty, // These may not be complete yet, but we only tell the client Network.responseReceived when all the headers are in + // Later should store body as well to support getResponseBody which should only work once Network.loadingFinished is sent + // but the body itself would be loaded with each chunks as Network.dataReceiveds are coming in. +}; + +// Stored in CDP +pub const NetworkState = struct { + const Self = @This(); + received: std.AutoArrayHashMap(u64, Response), + arena: std.heap.ArenaAllocator, + + pub fn init(allocator: Allocator) !NetworkState { + return .{ + .received = std.AutoArrayHashMap(u64, Response).init(allocator), + .arena = std.heap.ArenaAllocator.init(allocator), + }; + } + + pub fn deinit(self: *Self) void { + self.received.deinit(); + self.arena.deinit(); + } + + pub fn putOrAppendReceivedHeader(self: *NetworkState, request_id: u64, status: u16, header: std.http.Header) !void { + const kv = try self.received.getOrPut(request_id); + if (!kv.found_existing) kv.value_ptr.* = .{ .status = status }; + + const a = self.arena.allocator(); + const name = try a.dupe(u8, header.name); + const value = try a.dupe(u8, header.value); + try kv.value_ptr.headers.put(a, name, value); + } +}; + +pub fn pageRemove(bc: anytype) !void { + // The main page is going to be removed + const state = &bc.cdp.network_state; + state.received.clearRetainingCapacity(); // May need to be in pageRemoved + _ = state.arena.reset(.{ .retain_with_limit = 1024 }); +} + fn enable(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; try bc.networkEnable(); @@ -282,7 +325,7 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification }, .{ .session_id = session_id }); } -pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notification.RequestComplete) !void { +pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. std.debug.assert(bc.session.page != null); @@ -293,7 +336,7 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi const session_id = bc.session_id orelse unreachable; const target_id = bc.target_id orelse unreachable; - const url = try urlToString(arena, request.url, .{ + const url = try urlToString(arena, &request.transfer.uri, .{ .scheme = true, .authentication = true, .authority = true, @@ -301,22 +344,17 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi .query = true, }); - // @newhttp - const headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - // try headers.ensureTotalCapacity(arena, request.headers.len); - // for (request.headers) |header| { - // headers.putAssumeCapacity(header.name, header.value); - // } + const response = bc.cdp.network_state.received.get(request.transfer.id) orelse return error.ResponseNotFound; // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.responseReceived", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}), + .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.transfer.id}), .loaderId = bc.loader_id, .response = .{ .url = url, - .status = request.status, - .statusText = @as(std.http.Status, @enumFromInt(request.status)).phrase() orelse "Unknown", - .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, + .status = response.status, + .statusText = @as(std.http.Status, @enumFromInt(response.status)).phrase() orelse "Unknown", + .headers = std.json.ArrayHashMap([]const u8){ .map = response.headers }, }, .frameId = target_id, }, .{ .session_id = session_id }); diff --git a/src/http/Client.zig b/src/http/Client.zig index f27533f5..58e99112 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -247,6 +247,7 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer { .req = req, .ctx = req.ctx, .client = self, + .notification = &self.notification, }; return transfer; } @@ -549,6 +550,8 @@ pub const Transfer = struct { _redirecting: bool = false, + notification: *?*Notification, // Points to the Client's notification. TBD if a Browser can remove the notification before all Transfers are gone. + fn deinit(self: *Transfer) void { self.req.headers.deinit(); if (self._handle) |handle| { @@ -673,12 +676,26 @@ pub const Transfer = struct { // returning < buf_len terminates the request return 0; }; + if (transfer.notification.*) |notification| { // TBD before or after callback? + notification.dispatch(.http_headers_done_receiving, &.{ + .transfer = transfer, + }); + } } else { if (transfer.req.header_callback) |cb| { cb(transfer, header) catch |err| { log.err(.http, "header_callback", .{ .err = err, .req = transfer }); return 0; }; + if (transfer.notification.*) |notification| { // TBD before or after callback? + if (Http.Headers.parseHeader(header)) |hdr_name_value| { + notification.dispatch(.http_header_received, &.{ + .request_id = transfer.id, + .status = hdr.status, + .header = hdr_name_value, + }); + } else log.err(.http, "invalid header", .{ .line = header }); + } } } return buf_len; diff --git a/src/http/Http.zig b/src/http/Http.zig index b83f3477..506a0f13 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -244,8 +244,8 @@ pub const Headers = struct { return list; } - fn parseHeader(header_str: []const u8) ?struct { name: []const u8, value: []const u8 } { - const colon_pos = std.mem.indexOf(u8, header_str, ":") orelse return null; + pub fn parseHeader(header_str: []const u8) ?std.http.Header { + const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null; const name = std.mem.trim(u8, header_str[0..colon_pos], " \t"); const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t"); diff --git a/src/notification.zig b/src/notification.zig index 450d8fb0..0e8860de 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -63,7 +63,8 @@ pub const Notification = struct { http_request_fail: List = .{}, http_request_start: List = .{}, http_request_intercept: List = .{}, - http_request_complete: List = .{}, + http_header_received: List = .{}, + http_headers_done_receiving: List = .{}, notification_created: List = .{}, }; @@ -75,7 +76,8 @@ pub const Notification = struct { http_request_fail: *const RequestFail, http_request_start: *const RequestStart, http_request_intercept: *const RequestIntercept, - http_request_complete: *const RequestComplete, + http_header_received: *const ResponseHeader, + http_headers_done_receiving: *const ResponseHeadersDone, notification_created: *Notification, }; const EventType = std.meta.FieldEnum(Events); @@ -102,17 +104,21 @@ pub const Notification = struct { wait_for_interception: *bool, }; + pub const ResponseHeader = struct { + request_id: u64, + status: u16, + header: std.http.Header, + }; + + pub const ResponseHeadersDone = struct { + transfer: *Transfer, + }; + pub const RequestFail = struct { transfer: *Transfer, err: anyerror, }; - pub const RequestComplete = struct { - id: usize, - url: *const std.Uri, - status: u16, - }; - pub fn init(allocator: Allocator, parent: ?*Notification) !*Notification { // This is put on the heap because we want to raise a .notification_created // event, so that, something like Telemetry, can receive the