public/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2025-10-29 23:23:28 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #946 from lightpanda-io/request_interception

Browse Source 
Request Interception
This commit is contained in:
Karl Seguin
2025-08-20 07:53:08 +08:00
committed by GitHub
parent 4de4e7504d f5ec74252d
commit af4066da87
11 changed files with 555 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/browser/ScriptManager.zig
View File

@@ -24,8 +24,8 @@ const parser = @import("netsurf.zig");
const Env = @import("env.zig").Env;
const Page = @import("page.zig").Page;
const DataURI = @import("DataURI.zig");
const Http = @import("../http/Http.zig");
const Browser = @import("browser.zig").Browser;
const HttpClient = @import("../http/Client.zig");
const URL = @import("../url.zig").URL;
const Allocator = std.mem.Allocator;
@@ -57,7 +57,7 @@ deferreds: OrderList,
shutdown: bool = false,
client: *HttpClient,
client: *Http.Client,
allocator: Allocator,
buffer_pool: BufferPool,
script_pool: std.heap.MemoryPool(PendingScript),
@@ -229,7 +229,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
errdefer pending_script.deinit();
var headers = try HttpClient.Headers.init();
var headers = try Http.Headers.init();
try page.requestCookie(.{}).headersForRequest(page.arena, remote_url.?, &headers);
try self.client.request(.{
@@ -238,6 +238,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
.method = .GET,
.headers = headers,
.cookie_jar = page.cookie_jar,
.resource_type = .script,
.start_callback = if (log.enabled(.http, .debug)) startCallback else null,
.header_done_callback = headerCallback,
.data_callback = dataCallback,
@@ -296,7 +297,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
.buffer_pool = &self.buffer_pool,
};
var headers = try HttpClient.Headers.init();
var headers = try Http.Headers.init();
try self.page.requestCookie(.{}).headersForRequest(self.page.arena, url, &headers);
var client = self.client;
@@ -306,6 +307,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult {
.headers = headers,
.cookie_jar = self.page.cookie_jar,
.ctx = &blocking,
.resource_type = .script,
.start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null,
.header_done_callback = Blocking.headerCallback,
.data_callback = Blocking.dataCallback,
@@ -423,7 +425,7 @@ fn getList(self: *ScriptManager, script: *const Script) *OrderList {
return &self.scripts;
}
fn startCallback(transfer: *HttpClient.Transfer) !void {
fn startCallback(transfer: *Http.Transfer) !void {
const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
script.startCallback(transfer) catch |err| {
log.err(.http, "SM.startCallback", .{ .err = err, .transfer = transfer });
@@ -431,7 +433,7 @@ fn startCallback(transfer: *HttpClient.Transfer) !void {
};
}
fn headerCallback(transfer: *HttpClient.Transfer) !void {
fn headerCallback(transfer: *Http.Transfer) !void {
const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
script.headerCallback(transfer) catch |err| {
log.err(.http, "SM.headerCallback", .{
@@ -443,7 +445,7 @@ fn headerCallback(transfer: *HttpClient.Transfer) !void {
};
}
fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx));
script.dataCallback(transfer, data) catch |err| {
log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len });
@@ -488,12 +490,12 @@ const PendingScript = struct {
}
}
fn startCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void {
fn startCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
_ = self;
log.debug(.http, "script fetch start", .{ .req = transfer });
}
fn headerCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void {
fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) !void {
const header = &transfer.response_header.?;
log.debug(.http, "script header", .{
.req = transfer,
@@ -513,7 +515,7 @@ const PendingScript = struct {
self.script.source = .{ .remote = self.manager.buffer_pool.get() };
}
fn dataCallback(self: *PendingScript, transfer: *HttpClient.Transfer, data: []const u8) !void {
fn dataCallback(self: *PendingScript, transfer: *Http.Transfer, data: []const u8) !void {
_ = transfer;
// too verbose
// log.debug(.http, "script data chunk", .{
@@ -766,11 +768,11 @@ const Blocking = struct {
done: BlockingResult,
};
fn startCallback(transfer: *HttpClient.Transfer) !void {
fn startCallback(transfer: *Http.Transfer) !void {
log.debug(.http, "script fetch start", .{ .req = transfer, .blocking = true });
}
fn headerCallback(transfer: *HttpClient.Transfer) !void {
fn headerCallback(transfer: *Http.Transfer) !void {
const header = &transfer.response_header.?;
log.debug(.http, "script header", .{
.req = transfer,
@@ -787,7 +789,7 @@ const Blocking = struct {
self.buffer = self.buffer_pool.get();
}
fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void {
// too verbose
// log.debug(.http, "script data chunk", .{
// .req = transfer,

76
src/browser/page.zig
View File

@@ -30,7 +30,7 @@ const Renderer = @import("renderer.zig").Renderer;
const Window = @import("html/window.zig").Window;
const Walker = @import("dom/walker.zig").WalkerDepthFirst;
const Scheduler = @import("Scheduler.zig");
const HttpClient = @import("../http/Client.zig");
const Http = @import("../http/Http.zig");
const ScriptManager = @import("ScriptManager.zig");
const HTMLDocument = @import("html/document.zig").HTMLDocument;
@@ -87,13 +87,23 @@ pub const Page = struct {
polyfill_loader: polyfill.Loader = .{},
scheduler: Scheduler,
http_client: *HttpClient,
http_client: *Http.Client,
script_manager: ScriptManager,
mode: Mode,
load_state: LoadState = .parsing,
// Page.wait balances waiting for resources / tasks and producing an output.
// Up until a timeout, Page.wait will always wait for inflight or pending
// HTTP requests, via the Http.Client.active counter. However, intercepted
// requests (via CDP, but it could be anything), aren't considered "active"
// connection. So it's possible that we have intercepted requests (which are
// pending on some driver to continue/abort) while Http.Client.active == 0.
// This boolean exists to supplment Http.Client.active and inform Page.wait
// of pending connections.
request_intercepted: bool = false,
const Mode = union(enum) {
pre: void,
err: anyerror,
@@ -275,16 +285,26 @@ pub const Page = struct {
while (true) {
SW: switch (self.mode) {
.pre, .raw => {
if (self.request_intercepted) {
// the page request was intercepted.
// there shouldn't be any active requests;
std.debug.assert(http_client.active == 0);
// nothing we can do for this, need to kick the can up
// the chain and wait for activity (e.g. a CDP message)
// to unblock this.
return;
}
// The main page hasn't started/finished navigating.
// There's no JS to run, and no reason to run the scheduler.
if (http_client.active == 0) {
// haven't started navigating, I guess.
return;
}
// There should only be 1 active http transfer, the main page
std.debug.assert(http_client.active == 1);
try http_client.tick(ms_remaining);
},
.html, .parsed => {
@@ -330,20 +350,35 @@ pub const Page = struct {
_ = try scheduler.runLowPriority();
// We'll block here, waiting for network IO. We know
// when the next timeout is scheduled, and we know how long
// the caller wants to wait for, so we can pick a good wait
// duration
const ms_to_wait = @min(ms_remaining, ms_to_next_task orelse 1000);
const request_intercepted = self.request_intercepted;
// We want to prioritize processing intercepted requests
// because, the sooner they get unblocked, the sooner we
// can start the HTTP request. But we still want to advanced
// existing HTTP requests, if possible. So, if we have
// intercepted requests, we'll still look at existing HTTP
// requests, but we won't block waiting for more data.
const ms_to_wait =
if (request_intercepted) 0
// But if we have no intercepted requests, we'll wait
// for as long as we can for data to our existing
// inflight requests
else @min(ms_remaining, ms_to_next_task orelse 1000);
try http_client.tick(ms_to_wait);
if (try_catch.hasCaught()) {
const msg = (try try_catch.err(self.arena)) orelse "unknown";
log.warn(.user_script, "page wait", .{ .err = msg, .src = "data" });
return error.JsError;
if (request_intercepted) {
// Again, proritizing intercepted requests. Exit this
// loop so that our caller can hopefully resolve them
// (i.e. continue or abort them);
return;
}
},
.err => |err| return err,
.err => |err| {
self.mode = .{ .raw_done = @errorName(err) };
return err;
},
.raw_done => return,
}
@@ -362,7 +397,7 @@ pub const Page = struct {
std.debug.print("\nactive requests: {d}\n", .{self.http_client.active});
var n_ = self.http_client.handles.in_use.first;
while (n_) |n| {
const transfer = HttpClient.Transfer.fromEasy(n.data.conn.easy) catch |err| {
const transfer = Http.Transfer.fromEasy(n.data.conn.easy) catch |err| {
std.debug.print(" - failed to load transfer: {any}\n", .{err});
break;
};
@@ -435,7 +470,7 @@ pub const Page = struct {
is_http: bool = true,
is_navigation: bool = false,
};
pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie {
pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) Http.Client.RequestCookie {
return .{
.jar = self.cookie_jar,
.origin = &self.url.uri,
@@ -473,7 +508,7 @@ pub const Page = struct {
const owned_url = try self.arena.dupeZ(u8, request_url);
self.url = try URL.parse(owned_url, null);
var headers = try HttpClient.Headers.init();
var headers = try Http.Headers.init();
if (opts.header) |hdr| try headers.add(hdr);
try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, owned_url, &headers);
@@ -484,6 +519,7 @@ pub const Page = struct {
.headers = headers,
.body = opts.body,
.cookie_jar = self.cookie_jar,
.resource_type = .document,
.header_done_callback = pageHeaderDoneCallback,
.data_callback = pageDataCallback,
.done_callback = pageDoneCallback,
@@ -563,7 +599,7 @@ pub const Page = struct {
);
}
fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !void {
fn pageHeaderDoneCallback(transfer: *Http.Transfer) !void {
var self: *Page = @alignCast(@ptrCast(transfer.ctx));
// would be different than self.url in the case of a redirect
@@ -578,7 +614,7 @@ pub const Page = struct {
});
}
fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
fn pageDataCallback(transfer: *Http.Transfer, data: []const u8) !void {
var self: *Page = @alignCast(@ptrCast(transfer.ctx));
if (self.mode == .pre) {
@@ -1002,7 +1038,7 @@ pub const NavigateReason = enum {
pub const NavigateOpts = struct {
cdp_id: ?i64 = null,
reason: NavigateReason = .address_bar,
method: HttpClient.Method = .GET,
method: Http.Method = .GET,
body: ?[]const u8 = null,
header: ?[:0]const u8 = null,
};

24
src/browser/xhr/xhr.zig
View File

@@ -30,7 +30,7 @@ const URL = @import("../../url.zig").URL;
const Mime = @import("../mime.zig").Mime;
const parser = @import("../netsurf.zig");
const Page = @import("../page.zig").Page;
const HttpClient = @import("../../http/Client.zig");
const Http = @import("../../http/Http.zig");
const CookieJar = @import("../storage/storage.zig").CookieJar;
// XHR interfaces
@@ -80,12 +80,12 @@ const XMLHttpRequestBodyInit = union(enum) {
pub const XMLHttpRequest = struct {
proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
arena: Allocator,
transfer: ?*HttpClient.Transfer = null,
transfer: ?*Http.Transfer = null,
err: ?anyerror = null,
last_dispatch: i64 = 0,
send_flag: bool = false,
method: HttpClient.Method,
method: Http.Method,
state: State,
url: ?[:0]const u8 = null,
@@ -320,7 +320,7 @@ pub const XMLHttpRequest = struct {
}
const methods = [_]struct {
tag: HttpClient.Method,
tag: Http.Method,
name: []const u8,
}{
.{ .tag = .DELETE, .name = "DELETE" },
@@ -330,7 +330,7 @@ pub const XMLHttpRequest = struct {
.{ .tag = .POST, .name = "POST" },
.{ .tag = .PUT, .name = "PUT" },
};
pub fn validMethod(m: []const u8) DOMError!HttpClient.Method {
pub fn validMethod(m: []const u8) DOMError!Http.Method {
for (methods) |method| {
if (std.ascii.eqlIgnoreCase(method.name, m)) {
return method.tag;
@@ -370,7 +370,7 @@ pub const XMLHttpRequest = struct {
}
}
var headers = try HttpClient.Headers.init();
var headers = try Http.Headers.init();
for (self.headers.items) |hdr| {
try headers.add(hdr);
}
@@ -383,6 +383,7 @@ pub const XMLHttpRequest = struct {
.headers = headers,
.body = self.request_body,
.cookie_jar = page.cookie_jar,
.resource_type = .xhr,
.start_callback = httpStartCallback,
.header_callback = httpHeaderCallback,
.header_done_callback = httpHeaderDoneCallback,
@@ -392,18 +393,19 @@ pub const XMLHttpRequest = struct {
});
}
fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
fn httpStartCallback(transfer: *Http.Transfer) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" });
self.transfer = transfer;
}
fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: []const u8) !void {
fn httpHeaderCallback(transfer: *Http.Transfer, header: Http.Header) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
try self.response_headers.append(self.arena, try self.arena.dupe(u8, header));
const joined = try std.fmt.allocPrint(self.arena, "{s}: {s}", .{ header.name, header.value });
try self.response_headers.append(self.arena, joined);
}
fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !void {
fn httpHeaderDoneCallback(transfer: *Http.Transfer) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
const header = &transfer.response_header.?;
@@ -433,7 +435,7 @@ pub const XMLHttpRequest = struct {
self.dispatchEvt("readystatechange");
}
fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void {
fn httpDataCallback(transfer: *Http.Transfer, data: []const u8) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
try self.response_bytes.appendSlice(self.arena, data);

22
src/cdp/cdp.zig
View File

@@ -74,11 +74,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
// Used for processing notifications within a browser context.
notification_arena: std.heap.ArenaAllocator,
// Extra headers to add to all requests. TBD under which conditions this should be reset.
extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty,
intercept_state: InterceptState,
const Self = @This();
pub fn init(app: *App, client: TypeProvider.Client) !Self {
@@ -93,7 +88,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
.browser_context = null,
.message_arena = std.heap.ArenaAllocator.init(allocator),
.notification_arena = std.heap.ArenaAllocator.init(allocator),
.intercept_state = try InterceptState.init(allocator), // TBD or browser session arena?
};
}
@@ -101,7 +95,6 @@ pub fn CDPT(comptime TypeProvider: type) type {
if (self.browser_context) |*bc| {
bc.deinit();
}
self.intercept_state.deinit(); // TBD Should this live in BC?
self.browser.deinit();
self.message_arena.deinit();
self.notification_arena.deinit();
@@ -346,6 +339,11 @@ pub fn BrowserContext(comptime CDP_T: type) type {
http_proxy_changed: bool = false,
// Extra headers to add to all requests.
extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty,
intercept_state: InterceptState,
const Self = @This();
fn init(self: *Self, id: []const u8, cdp: *CDP_T) !void {
@@ -375,6 +373,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
.isolated_world = null,
.inspector = inspector,
.notification_arena = cdp.notification_arena.allocator(),
.intercept_state = try InterceptState.init(allocator),
};
self.node_search_list = Node.Search.List.init(allocator, &self.node_registry);
errdefer self.deinit();
@@ -388,6 +387,12 @@ pub fn BrowserContext(comptime CDP_T: type) type {
pub fn deinit(self: *Self) void {
self.inspector.deinit();
// abort all intercepted requests before closing the sesion/page
// since some of these might callback into the page/scriptmanager
for (self.intercept_state.pendingTransfers()) |transfer| {
transfer.abort();
}
// If the session has a page, we need to clear it first. The page
// context is always nested inside of the isolated world context,
// so we need to shutdown the page one first.
@@ -407,6 +412,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
log.warn(.http, "restoreOriginalProxy", .{ .err = err });
};
}
self.intercept_state.deinit();
}
pub fn reset(self: *Self) void {
@@ -495,7 +501,7 @@ pub fn BrowserContext(comptime CDP_T: type) type {
pub fn onHttpRequestIntercept(ctx: *anyopaque, data: *const Notification.RequestIntercept) !void {
const self: *Self = @alignCast(@ptrCast(ctx));
defer self.resetNotificationArena();
try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data);
try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, data);
}
pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void {

2
src/cdp/domains/dom.zig
View File

@@ -304,7 +304,7 @@ fn describeNode(cmd: anytype) !void {
pierce: bool = false,
})) orelse return error.InvalidParams;
if (params.depth != 1 or params.pierce) return error.NotYetImplementedParams;
if (params.depth != 1 or params.pierce) return error.NotImplemented;
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const node = try getNode(cmd.arena, bc, params.nodeId, params.backendNodeId, params.objectId);

238
src/cdp/domains/fetch.zig
View File

@@ -18,10 +18,12 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Notification = @import("../../notification.zig").Notification;
const log = @import("../../log.zig");
const Method = @import("../../http/Client.zig").Method;
const Transfer = @import("../../http/Client.zig").Transfer;
const network = @import("network.zig");
const Http = @import("../../http/Http.zig");
const Notification = @import("../../notification.zig").Notification;
pub fn processMessage(cmd: anytype) !void {
const action = std.meta.stringToEnum(enum {
@@ -29,6 +31,7 @@ pub fn processMessage(cmd: anytype) !void {
enable,
continueRequest,
failRequest,
fulfillRequest,
}, cmd.input.action) orelse return error.UnknownMethod;
switch (action) {
@@ -36,27 +39,48 @@ pub fn processMessage(cmd: anytype) !void {
.enable => return enable(cmd),
.continueRequest => return continueRequest(cmd),
.failRequest => return failRequest(cmd),
.fulfillRequest => return fulfillRequest(cmd),
}
}
// Stored in CDP
pub const InterceptState = struct {
const Self = @This();
waiting: std.AutoArrayHashMap(u64, *Transfer),
allocator: Allocator,
waiting: std.AutoArrayHashMapUnmanaged(u64, *Http.Transfer),
pub fn init(allocator: Allocator) !InterceptState {
return .{
.waiting = std.AutoArrayHashMap(u64, *Transfer).init(allocator),
.waiting = .empty,
.allocator = allocator,
};
}
pub fn deinit(self: *Self) void {
self.waiting.deinit();
pub fn empty(self: *const InterceptState) bool {
return self.waiting.count() == 0;
}
pub fn put(self: *InterceptState, transfer: *Http.Transfer) !void {
return self.waiting.put(self.allocator, transfer.id, transfer);
}
pub fn remove(self: *InterceptState, id: u64) ?*Http.Transfer {
const entry = self.waiting.fetchSwapRemove(id) orelse return null;
return entry.value;
}
pub fn deinit(self: *InterceptState) void {
self.waiting.deinit(self.allocator);
}
pub fn pendingTransfers(self: *const InterceptState) []*Http.Transfer {
return self.waiting.values();
}
};
const RequestPattern = struct {
urlPattern: []const u8 = "*", // Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed. Escape character is backslash. Omitting is equivalent to "*".
// Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed.
// Escape character is backslash. Omitting is equivalent to "*".
urlPattern: []const u8 = "*",
resourceType: ?ResourceType = null,
requestStage: RequestStage = .Request,
};
@@ -115,8 +139,14 @@ fn disable(cmd: anytype) !void {
fn enable(cmd: anytype) !void {
const params = (try cmd.params(EnableParam)) orelse EnableParam{};
if (params.patterns.len != 0) log.warn(.cdp, "Fetch.enable No patterns yet", .{});
if (params.handleAuthRequests) log.warn(.cdp, "Fetch.enable No auth yet", .{});
if (!arePatternsSupported(params.patterns)) {
log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable advanced patterns are not" });
return cmd.sendResult(null, .{});
}
if (params.handleAuthRequests) {
log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable handleAuthRequests is not supported yet" });
}
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
try bc.fetchEnable();
@@ -124,57 +154,67 @@ fn enable(cmd: anytype) !void {
return cmd.sendResult(null, .{});
}
pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void {
var cdp = bc.cdp;
fn arePatternsSupported(patterns: []RequestPattern) bool {
if (patterns.len == 0) {
return true;
}
if (patterns.len > 1) {
return false;
}
// While we don't support patterns, yet, both Playwright and Puppeteer send
// a default pattern which happens to be what we support:
// [{"urlPattern":"*","requestStage":"Request"}]
// So, rather than erroring on this case because we don't support patterns,
// we'll allow it, because this pattern is how it works as-is.
const pattern = patterns[0];
if (!std.mem.eql(u8, pattern.urlPattern, "*")) {
return false;
}
if (pattern.resourceType != null) {
return false;
}
if (pattern.requestStage != .Request) {
return false;
}
return true;
}
pub fn requestIntercept(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void {
// unreachable because we _have_ to have a page.
const session_id = bc.session_id orelse unreachable;
const target_id = bc.target_id orelse unreachable;
const page = bc.session.currentPage() orelse unreachable;
// We keep it around to wait for modifications to the request.
// NOTE: we assume whomever created the request created it with a lifetime of the Page.
// TODO: What to do when receiving replies for a previous page's requests?
const transfer = intercept.transfer;
try cdp.intercept_state.waiting.put(transfer.id, transfer);
try bc.intercept_state.put(transfer);
// NOTE: .request data preparation is duped from network.zig
const full_request_url = transfer.uri;
const request_url = try @import("network.zig").urlToString(arena, &full_request_url, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const request_fragment = try @import("network.zig").urlToString(arena, &full_request_url, .{
.fragment = true,
});
const headers = try transfer.req.headers.asHashMap(arena);
// End of duped code
try cdp.sendEvent("Fetch.requestPaused", .{
try bc.cdp.sendEvent("Fetch.requestPaused", .{
.requestId = try std.fmt.allocPrint(arena, "INTERCEPT-{d}", .{transfer.id}),
.request = .{
.url = request_url,
.urlFragment = request_fragment,
.method = @tagName(transfer.req.method),
.hasPostData = transfer.req.body != null,
.headers = std.json.ArrayHashMap([]const u8){ .map = headers },
},
.request = network.TransferAsRequestWriter.init(transfer),
.frameId = target_id,
.resourceType = ResourceType.Document, // TODO!
.resourceType = switch (transfer.req.resource_type) {
.script => "Script",
.xhr => "XHR",
.document => "Document",
},
.networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}),
}, .{ .session_id = session_id });
log.debug(.cdp, "request intercept", .{
.state = "paused",
.id = transfer.id,
.url = transfer.uri,
});
// Await either continueRequest, failRequest or fulfillRequest
intercept.wait_for_interception.* = true;
}
const HeaderEntry = struct {
name: []const u8,
value: []const u8,
};
intercept.wait_for_interception.* = true;
page.request_intercepted = true;
}
fn continueRequest(cmd: anytype) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
@@ -183,49 +223,129 @@ fn continueRequest(cmd: anytype) !void {
url: ?[]const u8 = null,
method: ?[]const u8 = null,
postData: ?[]const u8 = null,
headers: ?[]const HeaderEntry = null,
headers: ?[]const Http.Header = null,
interceptResponse: bool = false,
})) orelse return error.InvalidParams;
if (params.postData != null or params.headers != null or params.interceptResponse) return error.NotYetImplementedParams;
if (params.interceptResponse) {
return error.NotImplemented;
}
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
var intercept_state = &bc.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const entry = bc.cdp.intercept_state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound;
const transfer = entry.value;
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
log.debug(.cdp, "request intercept", .{
.state = "contiune",
.id = transfer.id,
.url = transfer.uri,
.new_url = params.url,
});
// Update the request with the new parameters
if (params.url) |url| {
// The request url must be modified in a way that's not observable by page. So page.url is not updated.
try transfer.updateURL(try bc.cdp.browser.page_arena.allocator().dupeZ(u8, url));
try transfer.updateURL(try page.arena.dupeZ(u8, url));
}
if (params.method) |method| {
transfer.req.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams;
transfer.req.method = std.meta.stringToEnum(Http.Method, method) orelse return error.InvalidParams;
}
if (params.headers) |headers| {
try transfer.replaceRequestHeaders(cmd.arena, headers);
}
if (params.postData) |b| {
const decoder = std.base64.standard.Decoder;
const body = try bc.arena.alloc(u8, try decoder.calcSizeForSlice(b));
try decoder.decode(body, b);
transfer.req.body = body;
}
log.info(.cdp, "Request continued by intercept", .{ .id = params.requestId });
try bc.cdp.browser.http_client.process(transfer);
if (intercept_state.empty()) {
page.request_intercepted = false;
}
return cmd.sendResult(null, .{});
}
fn fulfillRequest(cmd: anytype) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const params = (try cmd.params(struct {
requestId: []const u8, // "INTERCEPT-{d}"
responseCode: u16,
responseHeaders: ?[]const Http.Header = null,
binaryResponseHeaders: ?[]const u8 = null,
body: ?[]const u8 = null,
responsePhrase: ?[]const u8 = null,
})) orelse return error.InvalidParams;
if (params.binaryResponseHeaders != null) {
log.warn(.cdp, "not implemented", .{ .feature = "Fetch.fulfillRequest binaryResponseHeade" });
return error.NotImplemented;
}
var intercept_state = &bc.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
log.debug(.cdp, "request intercept", .{
.state = "fulfilled",
.id = transfer.id,
.url = transfer.uri,
.status = params.responseCode,
.body = params.body != null,
});
var body: ?[]const u8 = null;
if (params.body) |b| {
const decoder = std.base64.standard.Decoder;
const buf = try cmd.arena.alloc(u8, try decoder.calcSizeForSlice(b));
try decoder.decode(buf, b);
body = buf;
}
try transfer.fulfill(params.responseCode, params.responseHeaders orelse &.{}, body);
return cmd.sendResult(null, .{});
}
fn failRequest(cmd: anytype) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
var state = &bc.cdp.intercept_state;
const params = (try cmd.params(struct {
requestId: []const u8, // "INTERCEPT-{d}"
errorReason: ErrorReason,
})) orelse return error.InvalidParams;
const request_id = try idFromRequestId(params.requestId);
const entry = state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound;
// entry.value is the transfer
entry.value.abort();
const page = bc.session.currentPage() orelse return error.PageNotLoaded;
log.info(.cdp, "Request aborted by intercept", .{ .reason = params.errorReason });
var intercept_state = &bc.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
defer transfer.abort();
log.info(.cdp, "request intercept", .{
.state = "fail",
.id = request_id,
.url = transfer.uri,
.reason = params.errorReason,
});
if (intercept_state.empty()) {
page.request_intercepted = false;
}
return cmd.sendResult(null, .{});
}
// Get u64 from requestId which is formatted as: "INTERCEPT-{d}"
fn idFromRequestId(request_id: []const u8) !u64 {
if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) return error.InvalidParams;
if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) {
return error.InvalidParams;
}
return std.fmt.parseInt(u64, request_id[10..], 10) catch return error.InvalidParams;
}

212
src/cdp/domains/network.zig
View File

@@ -19,10 +19,10 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const Notification = @import("../../notification.zig").Notification;
const log = @import("../../log.zig");
const CdpStorage = @import("storage.zig");
const Transfer = @import("../../http/Client.zig").Transfer;
const Notification = @import("../../notification.zig").Notification;
pub fn processMessage(cmd: anytype) !void {
const action = std.meta.stringToEnum(enum {
@@ -83,7 +83,7 @@ fn setExtraHTTPHeaders(cmd: anytype) !void {
// Copy the headers onto the browser context arena
const arena = bc.arena;
const extra_headers = &bc.cdp.extra_headers;
const extra_headers = &bc.extra_headers;
extra_headers.clearRetainingCapacity();
try extra_headers.ensureTotalCapacity(arena, params.headers.map.count());
@@ -121,7 +121,7 @@ fn deleteCookies(cmd: anytype) !void {
path: ?[]const u8 = null,
partitionKey: ?CdpStorage.CookiePartitionKey = null,
})) orelse return error.InvalidParams;
if (params.partitionKey != null) return error.NotYetImplementedParams;
if (params.partitionKey != null) return error.NotImplemented;
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const cookies = &bc.session.cookie_jar.cookies;
@@ -235,50 +235,16 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification
const page = bc.session.currentPage() orelse unreachable;
// Modify request with extra CDP headers
for (cdp.extra_headers.items) |extra| {
for (bc.extra_headers.items) |extra| {
try data.transfer.req.headers.add(extra);
}
const document_url = try urlToString(arena, &page.url.uri, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const transfer = data.transfer;
const full_request_url = transfer.uri;
const request_url = try urlToString(arena, &full_request_url, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const request_fragment = try urlToString(arena, &full_request_url, .{
.fragment = true, // TODO since path is false, this likely does not work as intended
});
const headers = try transfer.req.headers.asHashMap(arena);
// We're missing a bunch of fields, but, for now, this seems like enough
try cdp.sendEvent("Network.requestWillBeSent", .{
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}),
.frameId = target_id,
.loaderId = bc.loader_id,
.documentUrl = document_url,
.request = .{
.url = request_url,
.urlFragment = request_fragment,
.method = @tagName(transfer.req.method),
.hasPostData = transfer.req.body != null,
.headers = std.json.ArrayHashMap([]const u8){ .map = headers },
},
}, .{ .session_id = session_id });
try cdp.sendEvent("Network.requestWillBeSent", .{ .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), .frameId = target_id, .loaderId = bc.loader_id, .documentUrl = DocumentUrlWriter.init(&page.url.uri), .request = TransferAsRequestWriter.init(transfer) }, .{ .session_id = session_id });
}
pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void {
pub fn httpHeadersDone(arena: Allocator, bc: anytype, data: *const Notification.ResponseHeadersDone) !void {
// Isn't possible to do a network request within a Browser (which our
// notification is tied to), without a page.
std.debug.assert(bc.session.page != null);
@@ -289,54 +255,157 @@ pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notificati
const session_id = bc.session_id orelse unreachable;
const target_id = bc.target_id orelse unreachable;
const url = try urlToString(arena, &request.transfer.uri, .{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
});
const status = request.transfer.response_header.?.status;
// We're missing a bunch of fields, but, for now, this seems like enough
try cdp.sendEvent("Network.responseReceived", .{
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.transfer.id}),
.requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}),
.loaderId = bc.loader_id,
.response = .{
.url = url,
.status = status,
.statusText = @as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown",
.headers = ResponseHeaderWriter.init(request.transfer),
},
.frameId = target_id,
.response = TransferAsResponseWriter.init(data.transfer),
}, .{ .session_id = session_id });
}
pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 {
var buf: std.ArrayListUnmanaged(u8) = .empty;
try url.writeToStream(opts, buf.writer(arena));
return buf.items;
}
const ResponseHeaderWriter = struct {
pub const TransferAsRequestWriter = struct {
transfer: *Transfer,
fn init(transfer: *Transfer) ResponseHeaderWriter {
pub fn init(transfer: *Transfer) TransferAsRequestWriter {
return .{
.transfer = transfer,
};
}
pub fn jsonStringify(self: *const ResponseHeaderWriter, writer: anytype) !void {
pub fn jsonStringify(self: *const TransferAsRequestWriter, writer: anytype) !void {
const stream = writer.stream;
const transfer = self.transfer;
try writer.beginObject();
var it = self.transfer.responseHeaderIterator();
{
try writer.objectField("url");
try writer.beginWriteRaw();
try stream.writeByte('\"');
try transfer.uri.writeToStream(.{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
}, stream);
try stream.writeByte('\"');
writer.endWriteRaw();
}
{
if (transfer.uri.fragment) |frag| {
try writer.objectField("urlFragment");
try writer.beginWriteRaw();
try stream.writeAll("\"#");
try stream.writeAll(frag.percent_encoded);
try stream.writeByte('\"');
writer.endWriteRaw();
}
}
{
try writer.objectField("method");
try writer.write(@tagName(transfer.req.method));
}
{
try writer.objectField("hasPostData");
try writer.write(transfer.req.body != null);
}
{
try writer.objectField("headers");
try writer.beginObject();
var it = transfer.req.headers.iterator();
while (it.next()) |hdr| {
try writer.objectField(hdr.name);
try writer.write(hdr.value);
}
try writer.endObject();
}
try writer.endObject();
}
};
const TransferAsResponseWriter = struct {
transfer: *Transfer,
fn init(transfer: *Transfer) TransferAsResponseWriter {
return .{
.transfer = transfer,
};
}
pub fn jsonStringify(self: *const TransferAsResponseWriter, writer: anytype) !void {
const stream = writer.stream;
const transfer = self.transfer;
try writer.beginObject();
{
try writer.objectField("url");
try writer.beginWriteRaw();
try stream.writeByte('\"');
try transfer.uri.writeToStream(.{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
}, stream);
try stream.writeByte('\"');
writer.endWriteRaw();
}
if (transfer.response_header) |*rh| {
// it should not be possible for this to be false, but I'm not
// feeling brave today.
const status = rh.status;
try writer.objectField("status");
try writer.write(status);
try writer.objectField("statusText");
try writer.write(@as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown");
}
{
try writer.objectField("headers");
try writer.beginObject();
var it = transfer.responseHeaderIterator();
while (it.next()) |hdr| {
try writer.objectField(hdr.name);
try writer.write(hdr.value);
}
try writer.endObject();
}
try writer.endObject();
}
};
const DocumentUrlWriter = struct {
uri: *std.Uri,
fn init(uri: *std.Uri) DocumentUrlWriter {
return .{
.uri = uri,
};
}
pub fn jsonStringify(self: *const DocumentUrlWriter, writer: anytype) !void {
const stream = writer.stream;
try writer.beginWriteRaw();
try stream.writeByte('\"');
try self.uri.writeToStream(.{
.scheme = true,
.authentication = true,
.authority = true,
.path = true,
.query = true,
}, stream);
try stream.writeByte('\"');
writer.endWriteRaw();
}
};
const testing = @import("../testing.zig");
@@ -344,8 +413,8 @@ test "cdp.network setExtraHTTPHeaders" {
var ctx = testing.context();
defer ctx.deinit();
// _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" });
try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } });
_ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" });
// try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } });
try ctx.processMessage(.{
.id = 3,
@@ -360,10 +429,7 @@ test "cdp.network setExtraHTTPHeaders" {
});
const bc = ctx.cdp().browser_context.?;
try testing.expectEqual(bc.cdp.extra_headers.items.len, 1);
try ctx.processMessage(.{ .id = 5, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } });
try testing.expectEqual(bc.cdp.extra_headers.items.len, 0);
try testing.expectEqual(bc.extra_headers.items.len, 1);
}
test "cdp.Network: cookies" {

2
src/cdp/domains/storage.zig
View File

@@ -129,7 +129,7 @@ pub const CdpCookie = struct {
pub fn setCdpCookie(cookie_jar: *CookieJar, param: CdpCookie) !void {
if (param.priority != .Medium or param.sameParty != null or param.sourceScheme != null or param.partitionKey != null) {
return error.NotYetImplementedParams;
return error.NotImplemented;
}
var arena = std.heap.ArenaAllocator.init(cookie_jar.allocator);

9
src/cdp/domains/target.zig
View File

@@ -73,7 +73,9 @@ fn createBrowserContext(cmd: anytype) !void {
originsWithUniversalNetworkAccess: ?[]const []const u8 = null,
});
if (params) |p| {
if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) std.debug.print("Target.createBrowserContext: Not implemented param set\n", .{});
if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) {
log.warn(.cdp, "not implemented", .{ .feature = "Target.createBrowserContext: Not implemented param set" });
}
}
const bc = cmd.createBrowserContext() catch |err| switch (err) {
@@ -407,8 +409,9 @@ fn doAttachtoTarget(cmd: anytype, target_id: []const u8) !void {
std.debug.assert(bc.session_id == null);
const session_id = cmd.cdp.session_id_gen.next();
// extra_headers should not be kept on a new page or tab, currently we have only 1 page, we clear it just in case
bc.cdp.extra_headers.clearRetainingCapacity();
// extra_headers should not be kept on a new page or tab,
// currently we have only 1 page, we clear it just in case
bc.extra_headers.clearRetainingCapacity();
try cmd.sendEvent("Target.attachedToTarget", AttachToTarget{
.sessionId = session_id,

110
src/http/Client.zig
View File

@@ -19,10 +19,10 @@
const std = @import("std");
const log = @import("../log.zig");
const builtin = @import("builtin");
const Http = @import("Http.zig");
pub const Headers = Http.Headers;
const Notification = @import("../notification.zig").Notification;
const storage = @import("../browser/storage/storage.zig");
const CookieJar = @import("../browser/storage/storage.zig").CookieJar;
const urlStitch = @import("../url.zig").stitch;
@@ -34,7 +34,7 @@ const ArenaAllocator = std.heap.ArenaAllocator;
const errorCheck = Http.errorCheck;
const errorMCheck = Http.errorMCheck;
pub const Method = Http.Method;
const Method = Http.Method;
// This is loosely tied to a browser Page. Loading all the <scripts>, doing
// XHR requests, and loading imports all happens through here. Sine the app
@@ -309,6 +309,10 @@ fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) !void {
try conn.setMethod(req.method);
if (req.body) |b| {
try conn.setBody(b);
} else if (req.method == .POST) {
// libcurl will crash if the method is POST but there's no body
// TODO: is there a setting for that..seems weird.
try conn.setBody("");
}
var header_list = req.headers;
@@ -496,7 +500,7 @@ pub const RequestCookie = struct {
origin: *const std.Uri,
jar: *@import("../browser/storage/cookie.zig").Jar,
pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Headers) !void {
pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Http.Headers) !void {
const uri = std.Uri.parse(url) catch |err| {
log.warn(.http, "invalid url", .{ .err = err, .url = url });
return error.InvalidUrl;
@@ -519,19 +523,26 @@ pub const RequestCookie = struct {
pub const Request = struct {
method: Method,
url: [:0]const u8,
headers: Headers,
headers: Http.Headers,
body: ?[]const u8 = null,
cookie_jar: *storage.CookieJar,
cookie_jar: *CookieJar,
resource_type: ResourceType,
// arbitrary data that can be associated with this request
ctx: *anyopaque = undefined,
start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null,
header_callback: ?*const fn (transfer: *Transfer, header: []const u8) anyerror!void = null,
header_callback: ?*const fn (transfer: *Transfer, header: Http.Header) anyerror!void = null,
header_done_callback: *const fn (transfer: *Transfer) anyerror!void,
data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void,
done_callback: *const fn (ctx: *anyopaque) anyerror!void,
error_callback: *const fn (ctx: *anyopaque, err: anyerror) void,
const ResourceType = enum {
document,
xhr,
script,
};
};
pub const Transfer = struct {
@@ -544,7 +555,7 @@ pub const Transfer = struct {
_notified_fail: bool = false,
// We'll store the response header here
response_header: ?Header = null,
response_header: ?ResponseHeader = null,
_handle: ?*Handle = null,
@@ -582,6 +593,22 @@ pub const Transfer = struct {
self.req.url = url;
}
pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const Http.Header) !void {
self.req.headers.deinit();
var buf: std.ArrayListUnmanaged(u8) = .empty;
var new_headers = try Http.Headers.init();
for (headers) |hdr| {
// safe to re-use this buffer, because Headers.add because curl copies
// the value we pass into curl_slist_append.
defer buf.clearRetainingCapacity();
try std.fmt.format(buf.writer(allocator), "{s}: {s}", .{ hdr.name, hdr.value });
try buf.append(allocator, 0); // null terminated
try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);
}
self.req.headers = new_headers;
}
pub fn abort(self: *Transfer) void {
self.client.requestFailed(self, error.Abort);
if (self._handle != null) {
@@ -697,7 +724,7 @@ pub const Transfer = struct {
if (getResponseHeader(easy, "content-type", 0)) |ct| {
var hdr = &transfer.response_header.?;
const value = ct.value;
const len = @min(value.len, hdr._content_type.len);
const len = @min(value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN);
hdr._content_type_len = len;
@memcpy(hdr._content_type[0..len], value[0..len]);
}
@@ -726,12 +753,14 @@ pub const Transfer = struct {
}
} else {
if (transfer.req.header_callback) |cb| {
cb(transfer, header) catch |err| {
if (Http.Headers.parseHeader(header)) |hdr| {
cb(transfer, hdr) catch |err| {
log.err(.http, "header_callback", .{ .err = err, .req = transfer });
return 0;
};
}
}
}
return buf_len;
}
@@ -768,15 +797,64 @@ pub const Transfer = struct {
try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_PRIVATE, &private));
return @alignCast(@ptrCast(private));
}
pub fn fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void {
if (transfer._handle != null) {
// should never happen, should have been intercepted/paused, and then
// either continued, aborted and fulfilled once.
@branchHint(.unlikely);
return error.RequestInProgress;
}
transfer._fulfill(status, headers, body) catch |err| {
transfer.req.error_callback(transfer.req.ctx, err);
return err;
};
}
fn _fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void {
const req = &transfer.req;
if (req.start_callback) |cb| {
try cb(transfer);
}
if (req.header_callback) |cb| {
for (headers) |hdr| {
try cb(transfer, hdr);
}
}
transfer.response_header = .{
.status = status,
.url = req.url,
};
for (headers) |hdr| {
if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) {
const len = @min(hdr.value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN);
@memcpy(transfer.response_header.?._content_type[0..len], hdr.value[0..len]);
break;
}
}
try req.header_done_callback(transfer);
if (body) |b| {
try req.data_callback(transfer, b);
}
try req.done_callback(req.ctx);
}
};
pub const Header = struct {
pub const ResponseHeader = struct {
const MAX_CONTENT_TYPE_LEN = 64;
status: u16,
url: [*c]const u8,
_content_type_len: usize = 0,
_content_type: [64]u8 = undefined,
_content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined,
pub fn contentType(self: *Header) ?[]u8 {
pub fn contentType(self: *ResponseHeader) ?[]u8 {
if (self._content_type_len == 0) {
return null;
}
@@ -788,7 +866,7 @@ const HeaderIterator = struct {
easy: *c.CURL,
prev: ?*c.curl_header = null,
pub fn next(self: *HeaderIterator) ?struct { name: []const u8, value: []const u8 } {
pub fn next(self: *HeaderIterator) ?Http.Header {
const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null;
self.prev = h;
@@ -800,12 +878,12 @@ const HeaderIterator = struct {
}
};
const ResponseHeader = struct {
const CurlHeaderValue = struct {
value: []const u8,
amount: usize,
};
fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?ResponseHeader {
fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?CurlHeaderValue {
var hdr: [*c]c.curl_header = null;
const result = c.curl_easy_header(easy, name, index, c.CURLH_HEADER, -1, &hdr);
if (result == c.CURLE_OK) {

63
src/http/Http.zig
View File

@@ -22,14 +22,15 @@ pub const c = @cImport({
@cInclude("curl/curl.h");
});
const Client = @import("Client.zig");
pub const ENABLE_DEBUG = false;
pub const Client = @import("Client.zig");
pub const Transfer = Client.Transfer;
const errors = @import("errors.zig");
const Allocator = std.mem.Allocator;
const ArenaAllocator = std.heap.ArenaAllocator;
pub const ENABLE_DEBUG = false;
// Client.zig does the bulk of the work and is loosely tied to a browser Page.
// But we still need something above Client.zig for the "utility" http stuff
// we need to do, like telemetry. The most important thing we want from this
@@ -216,6 +217,11 @@ pub const Connection = struct {
}
};
pub const Header = struct {
name: []const u8,
value: []const u8,
};
pub const Headers = struct {
headers: *c.curl_slist,
cookies: ?[*c]const u8,
@@ -237,25 +243,7 @@ pub const Headers = struct {
self.headers = updated_headers;
}
pub fn asHashMap(self: *const Headers, allocator: Allocator) !std.StringArrayHashMapUnmanaged([]const u8) {
var list: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
try list.ensureTotalCapacity(allocator, self.count());
var current: [*c]c.curl_slist = self.headers;
while (current) |node| {
const str = std.mem.span(@as([*:0]const u8, @ptrCast(node.*.data)));
const header = parseHeader(str) orelse return error.InvalidHeader;
list.putAssumeCapacity(header.name, header.value);
current = node.*.next;
}
// special case for cookies
if (self.cookies) |v| {
list.putAssumeCapacity("Cookie", std.mem.span(@as([*:0]const u8, @ptrCast(v))));
}
return list;
}
pub fn parseHeader(header_str: []const u8) ?std.http.Header {
pub fn parseHeader(header_str: []const u8) ?Header {
const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
@@ -264,19 +252,28 @@ pub const Headers = struct {
return .{ .name = name, .value = value };
}
pub fn count(self: *const Headers) usize {
var current: [*c]c.curl_slist = self.headers;
var num: usize = 0;
while (current) |node| {
num += 1;
current = node.*.next;
pub fn iterator(self: *Headers) Iterator {
return .{
.header = self.headers,
.cookies = self.cookies,
};
}
// special case for cookies
if (self.cookies != null) {
num += 1;
}
return num;
const Iterator = struct {
header: [*c]c.curl_slist,
cookies: ?[*c]const u8,
pub fn next(self: *Iterator) ?Header {
const h = self.header orelse {
const cookies = self.cookies orelse return null;
self.cookies = null;
return .{ .name = "Cookie", .value = std.mem.span(@as([*:0]const u8, cookies)) };
};
self.header = h.*.next;
return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data))));
}
};
};
pub fn errorCheck(code: c.CURLcode) errors.Error!void {