public/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2025-10-30 07:31:47 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Reduce allocations made during request interception

Browse Source 
Stream (to json) the Transfer as a request and response object in the various
network interception-related events (e.g. Network.responseReceived).

Add a page.request_intercepted boolean flag for CDP to signal the page that
requests have been intercepted, allowing Page.wait to prioritize intercept
handling (or, at least, not block it).
This commit is contained in:
Karl Seguin
2025-08-15 14:01:57 +08:00
parent d9ed4cfca8
commit 01223601f2
6 changed files with 280 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/cdp/cdp.zig
View File

@@ -495,7 +495,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
pub fn onHttpRequestIntercept(ctx: *anyopaque, data: *const Notification.RequestIntercept) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data);
try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, data);
}
pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void {

114
src/cdp/domains/fetch.zig
View File

@@ -18,10 +18,13 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Notification = @import("../../notification.zig").Notification;
const log = @import("../../log.zig");
const network = @import("network.zig");
const Method = @import("../../http/Client.zig").Method;
const Transfer = @import("../../http/Client.zig").Transfer;
const Notification = @import("../../notification.zig").Notification;
pub fn processMessage(cmd: anytype) !void {
const action = std.meta.stringToEnum(enum {
@@ -41,22 +44,38 @@ pub fn processMessage(cmd: anytype) !void {
// Stored in CDP
pub const InterceptState = struct {
const Self = @This();
waiting: std.AutoArrayHashMap(u64, *Transfer),
allocator: Allocator,
waiting: std.AutoArrayHashMapUnmanaged(u64, *Transfer),
pub fn init(allocator: Allocator) !InterceptState {
return .{
.waiting = std.AutoArrayHashMap(u64, *Transfer).init(allocator),
.waiting = .empty,
.allocator = allocator,
};
}
pub fn deinit(self: *Self) void {
self.waiting.deinit();
pub fn empty(self: *const InterceptState) bool {
return self.waiting.count() == 0;
}
pub fn put(self: *InterceptState, transfer: *Transfer) !void {
return self.waiting.put(self.allocator, transfer.id, transfer);
}
pub fn remove(self: *InterceptState, id: u64) ?*Transfer {
const entry = self.waiting.fetchSwapRemove(id) orelse return null;
return entry.value;
}
pub fn deinit(self: *InterceptState) void {
self.waiting.deinit(self.allocator);
}
};
const RequestPattern = struct {
urlPattern: []const u8 = "*", // Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed. Escape character is backslash. Omitting is equivalent to "*".
// Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed.
// Escape character is backslash. Omitting is equivalent to "*".
urlPattern: []const u8 = "*",
resourceType: ?ResourceType = null,
requestStage: RequestStage = .Request,
};
@@ -115,8 +134,12 @@ fn disable(cmd: anytype) !void {
fn enable(cmd: anytype) !void {
const params = (try cmd.params(EnableParam)) orelse EnableParam{};
if (params.patterns.len != 0) log.warn(.cdp, "Fetch.enable No patterns yet", .{});
if (params.handleAuthRequests) log.warn(.cdp, "Fetch.enable No auth yet", .{});
if (params.patterns.len != 0) {
log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable No patterns yet" });
}
if (params.handleAuthRequests) {
log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable No auth yet" });
}
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
try bc.fetchEnable();
@@ -124,51 +147,33 @@ fn enable(cmd: anytype) !void {
return cmd.sendResult(null, .{});
}
pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void {
pub fn requestIntercept(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void {
var cdp = bc.cdp;
// unreachable because we _have_ to have a page.
const session_id = bc.session_id orelse unreachable;
const target_id = bc.target_id orelse unreachable;
const page = bc.session.currentPage() orelse unreachable;
// We keep it around to wait for modifications to the request.
// NOTE: we assume whomever created the request created it with a lifetime of the Page.
// TODO: What to do when receiving replies for a previous page's requests?
const transfer = intercept.transfer;
try cdp.intercept_state.waiting.put(transfer.id, transfer);
// NOTE: .request data preparation is duped from network.zig
const full_request_url = transfer.uri;
const request_url = try @import("network.zig").urlToString(arena, &full_request_url, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const request_fragment = try @import("network.zig").urlToString(arena, &full_request_url, .{
.fragment = true,
});
const headers = try transfer.req.headers.asHashMap(arena);
// End of duped code
try cdp.intercept_state.put(transfer);
try cdp.sendEvent("Fetch.requestPaused", .{
.requestId = try std.fmt.allocPrint(arena, "INTERCEPT-{d}", .{transfer.id}),
.request = .{
.url = request_url,
.urlFragment = request_fragment,
.method = @tagName(transfer.req.method),
.hasPostData = transfer.req.body != null,
.headers = std.json.ArrayHashMap([]const u8){ .map = headers },
},
.request = network.TransferAsRequestWriter.init(transfer),
.frameId = target_id,
.resourceType = ResourceType.Document, // TODO!
.networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}),
}, .{ .session_id = session_id });
// Await either continueRequest, failRequest or fulfillRequest
intercept.wait_for_interception.* = true;
page.request_intercepted = true;
}
const HeaderEntry = struct {
@@ -186,39 +191,58 @@ fn continueRequest(cmd: anytype) !void {
headers: ?[]const HeaderEntry = null,
interceptResponse: bool = false,
})) orelse return error.InvalidParams;
if (params.postData != null or params.headers != null or params.interceptResponse) return error.NotYetImplementedParams;
if (params.postData != null or params.headers != null or params.interceptResponse) {
return error.NotYetImplementedParams;
}
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
var intercept_state = &bc.cdp.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const entry = bc.cdp.intercept_state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound;
const transfer = entry.value;
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
// Update the request with the new parameters
if (params.url) |url| {
// The request url must be modified in a way that's not observable by page. So page.url is not updated.
try transfer.updateURL(try bc.cdp.browser.page_arena.allocator().dupeZ(u8, url));
// The request url must be modified in a way that's not observable by page.
// So page.url is not updated.
try transfer.updateURL(try page.arena.dupeZ(u8, url));
}
if (params.method) |method| {
transfer.req.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams;
}
log.info(.cdp, "Request continued by intercept", .{ .id = params.requestId });
log.info(.cdp, "Request continued by intercept", .{
.id = params.requestId,
.url = transfer.uri,
});
try bc.cdp.browser.http_client.process(transfer);
if (intercept_state.empty()) {
page.request_intercepted = false;
}
return cmd.sendResult(null, .{});
}
fn failRequest(cmd: anytype) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
var state = &bc.cdp.intercept_state;
const params = (try cmd.params(struct {
requestId: []const u8, // "INTERCEPT-{d}"
errorReason: ErrorReason,
})) orelse return error.InvalidParams;
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
var intercept_state = &bc.cdp.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const entry = state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound;
// entry.value is the transfer
entry.value.abort();
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
transfer.abort();
if (intercept_state.empty()) {
page.request_intercepted = false;
}
log.info(.cdp, "Request aborted by intercept", .{ .reason = params.errorReason });
return cmd.sendResult(null, .{});
@@ -226,6 +250,8 @@ fn failRequest(cmd: anytype) !void {
// Get u64 from requestId which is formatted as: "INTERCEPT-{d}"
fn idFromRequestId(request_id: []const u8) !u64 {
if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) return error.InvalidParams;
if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) {
return error.InvalidParams;
}
return std.fmt.parseInt(u64, request_id[10..], 10) catch return error.InvalidParams;
}

203
src/cdp/domains/network.zig
View File

@@ -19,10 +19,10 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Notification = @import("../../notification.zig").Notification;
const log = @import("../../log.zig");
const CdpStorage = @import("storage.zig");
const Transfer = @import("../../http/Client.zig").Transfer;
const Notification = @import("../../notification.zig").Notification;
pub fn processMessage(cmd: anytype) !void {
const action = std.meta.stringToEnum(enum {
@@ -239,46 +239,12 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification
try data.transfer.req.headers.add(extra);
}
const document_url = try urlToString(arena, &page.url.uri, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const transfer = data.transfer;
const full_request_url = transfer.uri;
const request_url = try urlToString(arena, &full_request_url, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const request_fragment = try urlToString(arena, &full_request_url, .{
.fragment = true, // TODO since path is false, this likely does not work as intended
});
const headers = try transfer.req.headers.asHashMap(arena);
// We're missing a bunch of fields, but, for now, this seems like enough
try cdp.sendEvent("Network.requestWillBeSent", .{
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}),
.frameId = target_id,
.loaderId = bc.loader_id,
.documentUrl = document_url,
.request = .{
.url = request_url,
.urlFragment = request_fragment,
.method = @tagName(transfer.req.method),
.hasPostData = transfer.req.body != null,
.headers = std.json.ArrayHashMap([]const u8){ .map = headers },
},
}, .{ .session_id = session_id });
try cdp.sendEvent("Network.requestWillBeSent", .{ .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), .frameId = target_id, .loaderId = bc.loader_id, .documentUrl = DocumentUrlWriter.init(&page.url.uri), .request = TransferAsRequestWriter.init(transfer) }, .{ .session_id = session_id });
}
pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void {
pub fn httpHeadersDone(arena: Allocator, bc: anytype, data: *const Notification.ResponseHeadersDone) !void {
// Isn't possible to do a network request within a Browser (which our
// notification is tied to), without a page.
std.debug.assert(bc.session.page != null);
@@ -289,56 +255,159 @@ pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notificati
const session_id = bc.session_id orelse unreachable;
const target_id = bc.target_id orelse unreachable;
const url = try urlToString(arena, &request.transfer.uri, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const status = request.transfer.response_header.?.status;
// We're missing a bunch of fields, but, for now, this seems like enough
try cdp.sendEvent("Network.responseReceived", .{
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.transfer.id}),
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}),
.loaderId = bc.loader_id,
.response = .{
.url = url,
.status = status,
.statusText = @as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown",
.headers = ResponseHeaderWriter.init(request.transfer),
},
.frameId = target_id,
.response = TransferAsResponseWriter.init(data.transfer),
}, .{ .session_id = session_id });
}
pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 {
var buf: std.ArrayListUnmanaged(u8) = .empty;
try url.writeToStream(opts, buf.writer(arena));
return buf.items;
}
const ResponseHeaderWriter = struct {
pub const TransferAsRequestWriter = struct {
transfer: *Transfer,
fn init(transfer: *Transfer) ResponseHeaderWriter {
pub fn init(transfer: *Transfer) TransferAsRequestWriter {
return .{
.transfer = transfer,
};
}
pub fn jsonStringify(self: *const ResponseHeaderWriter, writer: anytype) !void {
pub fn jsonStringify(self: *const TransferAsRequestWriter, writer: anytype) !void {
const stream = writer.stream;
const transfer = self.transfer;
try writer.beginObject();
var it = self.transfer.responseHeaderIterator();
while (it.next()) |hdr| {
try writer.objectField(hdr.name);
try writer.write(hdr.value);
{
try writer.objectField("url");
try writer.beginWriteRaw();
try stream.writeByte('\"');
try transfer.uri.writeToStream(.{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
}, stream);
try stream.writeByte('\"');
writer.endWriteRaw();
}
{
if (transfer.uri.fragment) |frag| {
try writer.objectField("urlFragment");
try writer.beginWriteRaw();
try stream.writeAll("\"#");
try stream.writeAll(frag.percent_encoded);
try stream.writeByte('\"');
writer.endWriteRaw();
}
}
{
try writer.objectField("method");
try writer.write(@tagName(transfer.req.method));
}
{
try writer.objectField("hasPostData");
try writer.write(transfer.req.body != null);
}
{
try writer.objectField("headers");
try writer.beginObject();
var it = transfer.req.headers.iterator();
while (it.next()) |hdr| {
try writer.objectField(hdr.name);
try writer.write(hdr.value);
}
try writer.endObject();
}
try writer.endObject();
}
};
const TransferAsResponseWriter = struct {
transfer: *Transfer,
fn init(transfer: *Transfer) TransferAsResponseWriter {
return .{
.transfer = transfer,
};
}
pub fn jsonStringify(self: *const TransferAsResponseWriter, writer: anytype) !void {
const stream = writer.stream;
const transfer = self.transfer;
try writer.beginObject();
{
try writer.objectField("url");
try writer.beginWriteRaw();
try stream.writeByte('\"');
try transfer.uri.writeToStream(.{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
}, stream);
try stream.writeByte('\"');
writer.endWriteRaw();
}
if (transfer.response_header) |*rh| {
// it should not be possible for this to be false, but I'm not
// feeling brave today.
const status = rh.status;
try writer.objectField("status");
try writer.write(status);
try writer.objectField("statusText");
try writer.write(@as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown");
}
{
try writer.objectField("headers");
try writer.beginObject();
var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
try writer.objectField(hdr.name);
try writer.write(hdr.value);
}
try writer.endObject();
}
try writer.endObject();
}
};
const DocumentUrlWriter = struct {
uri: *std.Uri,
fn init(uri: *std.Uri) DocumentUrlWriter {
return .{
.uri = uri,
};
}
pub fn jsonStringify(self: *const DocumentUrlWriter, writer: anytype) !void {
const stream = writer.stream;
try writer.beginWriteRaw();
try stream.writeByte('\"');
try self.uri.writeToStream(.{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
}, stream);
try stream.writeByte('\"');
writer.endWriteRaw();
}
};
const testing = @import("../testing.zig");
test "cdp.network setExtraHTTPHeaders" {
var ctx = testing.context();

4
src/cdp/domains/target.zig
View File

@@ -73,7 +73,9 @@ fn createBrowserContext(cmd: anytype) !void {
originsWithUniversalNetworkAccess: ?[]const []const u8 = null,
});
if (params) |p| {
if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) std.debug.print("Target.createBrowserContext: Not implemented param set\n", .{});
if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) {
log.warn(.cdp, "not implemented", .{ .feature = "Target.createBrowserContext: Not implemented param set" });
}
}
const bc = cmd.createBrowserContext() catch |err| switch (err) {