public/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2025-10-29 15:13:28 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Implement Network.getResponseBody

Browse Source 
Add response_data event, CDP now captures the full body so that it can respond
to the Network.getResponseBody. This isn't memory efficient, but I don't see
another way to do it. At least this way, it's only capturing/storing every
response body when (a) CDP is used and (b) Network.enabled is called. That is,
as opposed to baking this into Http/Client.zig, which would force the memory
consumption for all use-cases.

There's arguably some optimizations we could make for XHR requests, which also
dupe/own the response. As of now, the response is dupe'd separately for CDP
and XHR.
This commit is contained in:
Karl Seguin
2025-08-21 10:32:03 +08:00
parent 557f8444b2
commit cd33e9ad0e
4 changed files with 84 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/cdp/cdp.zig
View File

@@ -344,6 +344,15 @@ pub fn BrowserContext(comptime CDP_T: type) type {
intercept_state: InterceptState,
// When network is enabled, we'll capture the transfer.id -> body
// This is awfully memory intensive, but our underlying http client and
// its users (script manager and page) correctly do not hold the body
// memory longer than they have to. In fact, the main request is only
// ever streamed. So if CDP is the only thing that needs bodies in
// memory for an arbitrary amount of time, then that's where we're going
// to store the,
captured_responses: std.AutoHashMapUnmanaged(usize, std.ArrayListUnmanaged(u8)),
const Self = @This();
fn init(self: *Self, id: []const u8, cdp: *CDP_T) !void {
@@ -374,6 +383,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
.inspector = inspector,
.notification_arena = cdp.notification_arena.allocator(),
.intercept_state = try InterceptState.init(allocator),
.captured_responses = .empty,
};
self.node_search_list = Node.Search.List.init(allocator, &self.node_registry);
errdefer self.deinit();
@@ -454,15 +464,17 @@ pub fn BrowserContext(comptime CDP_T: type) type {
pub fn networkEnable(self: *Self) !void {
try self.cdp.browser.notification.register(.http_request_fail, self, onHttpRequestFail);
try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart);
try self.cdp.browser.notification.register(.http_headers_done, self, onHttpHeadersDone);
try self.cdp.browser.notification.register(.http_request_done, self, onHttpRequestDone);
try self.cdp.browser.notification.register(.http_response_data, self, onHttpResponseData);
try self.cdp.browser.notification.register(.http_response_header_done, self, onHttpResponseHeadersDone);
}
pub fn networkDisable(self: *Self) void {
self.cdp.browser.notification.unregister(.http_request_fail, self);
self.cdp.browser.notification.unregister(.http_request_start, self);
self.cdp.browser.notification.unregister(.http_headers_done, self);
self.cdp.browser.notification.unregister(.http_request_done, self);
self.cdp.browser.notification.unregister(.http_response_data, self);
self.cdp.browser.notification.unregister(.http_response_header_done, self);
}
pub fn fetchEnable(self: *Self) !void {
@@ -483,45 +495,57 @@ pub fn BrowserContext(comptime CDP_T: type) type {
return @import("domains/page.zig").pageCreated(self, page);
}
pub fn onPageNavigate(ctx: *anyopaque, data: *const Notification.PageNavigate) !void {
pub fn onPageNavigate(ctx: *anyopaque, msg: *const Notification.PageNavigate) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
return @import("domains/page.zig").pageNavigate(self.notification_arena, self, data);
return @import("domains/page.zig").pageNavigate(self.notification_arena, self, msg);
}
pub fn onPageNavigated(ctx: *anyopaque, data: *const Notification.PageNavigated) !void {
pub fn onPageNavigated(ctx: *anyopaque, msg: *const Notification.PageNavigated) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
return @import("domains/page.zig").pageNavigated(self, data);
return @import("domains/page.zig").pageNavigated(self, msg);
}
pub fn onHttpRequestStart(ctx: *anyopaque, data: *const Notification.RequestStart) !void {
pub fn onHttpRequestStart(ctx: *anyopaque, msg: *const Notification.RequestStart) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, data);
try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, msg);
}
pub fn onHttpRequestIntercept(ctx: *anyopaque, data: *const Notification.RequestIntercept) !void {
pub fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, data);
try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, msg);
}
pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void {
pub fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, data);
return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, msg);
}
pub fn onHttpHeadersDone(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void {
pub fn onHttpResponseHeadersDone(ctx: *anyopaque, msg: *const Notification.ResponseHeaderDone) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
return @import("domains/network.zig").httpHeadersDone(self.notification_arena, self, data);
return @import("domains/network.zig").httpResponseHeaderDone(self.notification_arena, self, msg);
}
pub fn onHttpRequestDone(ctx: *anyopaque, data: *const Notification.RequestDone) !void {
pub fn onHttpRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
return @import("domains/network.zig").httpRequestDone(self.notification_arena, self, data);
return @import("domains/network.zig").httpRequestDone(self.notification_arena, self, msg);
}
pub fn onHttpResponseData(ctx: *anyopaque, msg: *const Notification.ResponseData) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
const arena = self.arena;
const id = msg.transfer.id;
const gop = try self.captured_responses.getOrPut(arena, id);
if (!gop.found_existing) {
gop.value_ptr.* = .{};
}
try gop.value_ptr.appendSlice(arena, try arena.dupe(u8, msg.data));
}
fn resetNotificationArena(self: *Self) void {

47
src/cdp/domains/network.zig
View File

@@ -54,16 +54,6 @@ pub fn processMessage(cmd: anytype) !void {
}
}
const Response = struct {
status: u16,
headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty,
// These may not be complete yet, but we only tell the client
// Network.responseReceived when all the headers are in.
// Later should store body as well to support getResponseBody which should
// only work once Network.loadingFinished is sent but the body itself would
// be loaded with each chunks as Network.dataReceiveds are coming in.
};
fn enable(cmd: anytype) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
try bc.networkEnable();
@@ -209,15 +199,17 @@ fn getResponseBody(cmd: anytype) !void {
requestId: []const u8, // "REQ-{d}"
})) orelse return error.InvalidParams;
_ = params;
const request_id = try idFromRequestId(params.requestId);
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const buf = bc.captured_responses.getPtr(request_id) orelse return error.RequestNotFound;
try cmd.sendResult(.{
.body = "TODO",
.body = buf.items,
.base64Encoded = false,
}, .{});
}
pub fn httpRequestFail(arena: Allocator, bc: anytype, data: *const Notification.RequestFail) !void {
pub fn httpRequestFail(arena: Allocator, bc: anytype, msg: *const Notification.RequestFail) !void {
// It's possible that the request failed because we aborted when the client
// sent Target.closeTarget. In that case, bc.session_id will be cleared
// already, and we can skip sending these messages to the client.
@@ -229,15 +221,15 @@ pub fn httpRequestFail(arena: Allocator, bc: anytype, data: *const Notification.
// We're missing a bunch of fields, but, for now, this seems like enough
try bc.cdp.sendEvent("Network.loadingFailed", .{
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}),
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{msg.transfer.id}),
// Seems to be what chrome answers with. I assume it depends on the type of error?
.type = "Ping",
.errorText = data.err,
.errorText = msg.err,
.canceled = false,
}, .{ .session_id = session_id });
}
pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification.RequestStart) !void {
pub fn httpRequestStart(arena: Allocator, bc: anytype, msg: *const Notification.RequestStart) !void {
// Isn't possible to do a network request within a Browser (which our
// notification is tied to), without a page.
std.debug.assert(bc.session.page != null);
@@ -251,15 +243,15 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification
// Modify request with extra CDP headers
for (bc.extra_headers.items) |extra| {
try data.transfer.req.headers.add(extra);
try msg.transfer.req.headers.add(extra);
}
const transfer = data.transfer;
const transfer = msg.transfer;
// We're missing a bunch of fields, but, for now, this seems like enough
try cdp.sendEvent("Network.requestWillBeSent", .{ .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), .frameId = target_id, .loaderId = bc.loader_id, .documentUrl = DocumentUrlWriter.init(&page.url.uri), .request = TransferAsRequestWriter.init(transfer) }, .{ .session_id = session_id });
}
pub fn httpHeadersDone(arena: Allocator, bc: anytype, data: *const Notification.ResponseHeadersDone) !void {
pub fn httpResponseHeaderDone(arena: Allocator, bc: anytype, msg: *const Notification.ResponseHeaderDone) !void {
// Isn't possible to do a network request within a Browser (which our
// notification is tied to), without a page.
std.debug.assert(bc.session.page != null);
@@ -272,14 +264,14 @@ pub fn httpHeadersDone(arena: Allocator, bc: anytype, data: *const Notification.
// We're missing a bunch of fields, but, for now, this seems like enough
try cdp.sendEvent("Network.responseReceived", .{
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}),
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{msg.transfer.id}),
.loaderId = bc.loader_id,
.frameId = target_id,
.response = TransferAsResponseWriter.init(data.transfer),
.response = TransferAsResponseWriter.init(msg.transfer),
}, .{ .session_id = session_id });
}
pub fn httpRequestDone(arena: Allocator, bc: anytype, data: *const Notification.RequestDone) !void {
pub fn httpRequestDone(arena: Allocator, bc: anytype, msg: *const Notification.RequestDone) !void {
// Isn't possible to do a network request within a Browser (which our
// notification is tied to), without a page.
std.debug.assert(bc.session.page != null);
@@ -290,8 +282,8 @@ pub fn httpRequestDone(arena: Allocator, bc: anytype, data: *const Notification.
const session_id = bc.session_id orelse unreachable;
try cdp.sendEvent("Network.loadingFinished", .{
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}),
.encodedDataLength = data.transfer.bytes_received,
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{msg.transfer.id}),
.encodedDataLength = msg.transfer.bytes_received,
}, .{ .session_id = session_id });
}
@@ -439,6 +431,13 @@ const DocumentUrlWriter = struct {
}
};
fn idFromRequestId(request_id: []const u8) !u64 {
if (!std.mem.startsWith(u8, request_id, "REQ-")) {
return error.InvalidParams;
}
return std.fmt.parseInt(u64, request_id[4..], 10) catch return error.InvalidParams;
}
const testing = @import("../testing.zig");
test "cdp.network setExtraHTTPHeaders" {
var ctx = testing.context();

13
src/http/Client.zig
View File

@@ -757,7 +757,7 @@ pub const Transfer = struct {
};
if (transfer.client.notification) |notification| {
notification.dispatch(.http_headers_done, &.{
notification.dispatch(.http_response_header_done, &.{
.transfer = transfer,
});
}
@@ -780,10 +780,19 @@ pub const Transfer = struct {
}
transfer.bytes_received += chunk_len;
transfer.req.data_callback(transfer, buffer[0..chunk_len]) catch |err| {
const chunk = buffer[0..chunk_len];
transfer.req.data_callback(transfer, chunk) catch |err| {
log.err(.http, "data_callback", .{ .err = err, .req = transfer });
return c.CURL_WRITEFUNC_ERROR;
};
if (transfer.client.notification) |notification| {
notification.dispatch(.http_response_data, &.{
.data = chunk,
.transfer = transfer,
});
}
return chunk_len;
}

13
src/notification.zig
View File

@@ -63,8 +63,9 @@ pub const Notification = struct {
http_request_fail: List = .{},
http_request_start: List = .{},
http_request_intercept: List = .{},
http_headers_done: List = .{},
http_request_done: List = .{},
http_response_data: List = .{},
http_response_header_done: List = .{},
notification_created: List = .{},
};
@@ -76,8 +77,9 @@ pub const Notification = struct {
http_request_fail: *const RequestFail,
http_request_start: *const RequestStart,
http_request_intercept: *const RequestIntercept,
http_headers_done: *const ResponseHeadersDone,
http_request_done: *const RequestDone,
http_response_data: *const ResponseData,
http_response_header_done: *const ResponseHeaderDone,
notification_created: *Notification,
};
const EventType = std.meta.FieldEnum(Events);
@@ -104,7 +106,12 @@ pub const Notification = struct {
wait_for_interception: *bool,
};
pub const ResponseHeadersDone = struct {
pub const ResponseData = struct {
data: []const u8,
transfer: *Transfer,
};
pub const ResponseHeaderDone = struct {
transfer: *Transfer,
};