diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index aa457b46..4a27e054 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -24,8 +24,8 @@ const parser = @import("netsurf.zig"); const Env = @import("env.zig").Env; const Page = @import("page.zig").Page; const DataURI = @import("DataURI.zig"); +const Http = @import("../http/Http.zig"); const Browser = @import("browser.zig").Browser; -const HttpClient = @import("../http/Client.zig"); const URL = @import("../url.zig").URL; const Allocator = std.mem.Allocator; @@ -57,7 +57,7 @@ deferreds: OrderList, shutdown: bool = false, -client: *HttpClient, +client: *Http.Client, allocator: Allocator, buffer_pool: BufferPool, script_pool: std.heap.MemoryPool(PendingScript), @@ -229,7 +229,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { errdefer pending_script.deinit(); - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); try page.requestCookie(.{}).headersForRequest(page.arena, remote_url.?, &headers); try self.client.request(.{ @@ -297,7 +297,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { .buffer_pool = &self.buffer_pool, }; - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); try self.page.requestCookie(.{}).headersForRequest(self.page.arena, url, &headers); var client = self.client; @@ -425,7 +425,7 @@ fn getList(self: *ScriptManager, script: *const Script) *OrderList { return &self.scripts; } -fn startCallback(transfer: *HttpClient.Transfer) !void { +fn startCallback(transfer: *Http.Transfer) !void { const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx)); script.startCallback(transfer) catch |err| { log.err(.http, "SM.startCallback", .{ .err = err, .transfer = transfer }); @@ -433,7 +433,7 @@ fn startCallback(transfer: *HttpClient.Transfer) !void { }; } -fn headerCallback(transfer: *HttpClient.Transfer) !void { +fn headerCallback(transfer: *Http.Transfer) !void { const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx)); script.headerCallback(transfer) catch |err| { log.err(.http, "SM.headerCallback", .{ @@ -445,7 +445,7 @@ fn headerCallback(transfer: *HttpClient.Transfer) !void { }; } -fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { +fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void { const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx)); script.dataCallback(transfer, data) catch |err| { log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len }); @@ -490,12 +490,12 @@ const PendingScript = struct { } } - fn startCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void { + fn startCallback(self: *PendingScript, transfer: *Http.Transfer) !void { _ = self; log.debug(.http, "script fetch start", .{ .req = transfer }); } - fn headerCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void { + fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) !void { const header = &transfer.response_header.?; log.debug(.http, "script header", .{ .req = transfer, @@ -515,7 +515,7 @@ const PendingScript = struct { self.script.source = .{ .remote = self.manager.buffer_pool.get() }; } - fn dataCallback(self: *PendingScript, transfer: *HttpClient.Transfer, data: []const u8) !void { + fn dataCallback(self: *PendingScript, transfer: *Http.Transfer, data: []const u8) !void { _ = transfer; // too verbose // log.debug(.http, "script data chunk", .{ @@ -768,11 +768,11 @@ const Blocking = struct { done: BlockingResult, }; - fn startCallback(transfer: *HttpClient.Transfer) !void { + fn startCallback(transfer: *Http.Transfer) !void { log.debug(.http, "script fetch start", .{ .req = transfer, .blocking = true }); } - fn headerCallback(transfer: *HttpClient.Transfer) !void { + fn headerCallback(transfer: *Http.Transfer) !void { const header = &transfer.response_header.?; log.debug(.http, "script header", .{ .req = transfer, @@ -789,7 +789,7 @@ const Blocking = struct { self.buffer = self.buffer_pool.get(); } - fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { + fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void { // too verbose // log.debug(.http, "script data chunk", .{ // .req = transfer, diff --git a/src/browser/page.zig b/src/browser/page.zig index 69093485..0189ae82 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -30,7 +30,7 @@ const Renderer = @import("renderer.zig").Renderer; const Window = @import("html/window.zig").Window; const Walker = @import("dom/walker.zig").WalkerDepthFirst; const Scheduler = @import("Scheduler.zig"); -const HttpClient = @import("../http/Client.zig"); +const Http = @import("../http/Http.zig"); const ScriptManager = @import("ScriptManager.zig"); const HTMLDocument = @import("html/document.zig").HTMLDocument; @@ -87,7 +87,7 @@ pub const Page = struct { polyfill_loader: polyfill.Loader = .{}, scheduler: Scheduler, - http_client: *HttpClient, + http_client: *Http.Client, script_manager: ScriptManager, mode: Mode, @@ -375,7 +375,10 @@ pub const Page = struct { return; } }, - .err => |err| return err, + .err => |err| { + self.mode = .{ .raw_done = @errorName(err) }; + return err; + }, .raw_done => return, } @@ -394,7 +397,7 @@ pub const Page = struct { std.debug.print("\nactive requests: {d}\n", .{self.http_client.active}); var n_ = self.http_client.handles.in_use.first; while (n_) |n| { - const transfer = HttpClient.Transfer.fromEasy(n.data.conn.easy) catch |err| { + const transfer = Http.Transfer.fromEasy(n.data.conn.easy) catch |err| { std.debug.print(" - failed to load transfer: {any}\n", .{err}); break; }; @@ -467,7 +470,7 @@ pub const Page = struct { is_http: bool = true, is_navigation: bool = false, }; - pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie { + pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) Http.Client.RequestCookie { return .{ .jar = self.cookie_jar, .origin = &self.url.uri, @@ -505,7 +508,7 @@ pub const Page = struct { const owned_url = try self.arena.dupeZ(u8, request_url); self.url = try URL.parse(owned_url, null); - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); if (opts.header) |hdr| try headers.add(hdr); try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, owned_url, &headers); @@ -596,7 +599,7 @@ pub const Page = struct { ); } - fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !void { + fn pageHeaderDoneCallback(transfer: *Http.Transfer) !void { var self: *Page = @alignCast(@ptrCast(transfer.ctx)); // would be different than self.url in the case of a redirect @@ -611,7 +614,7 @@ pub const Page = struct { }); } - fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { + fn pageDataCallback(transfer: *Http.Transfer, data: []const u8) !void { var self: *Page = @alignCast(@ptrCast(transfer.ctx)); if (self.mode == .pre) { @@ -1035,7 +1038,7 @@ pub const NavigateReason = enum { pub const NavigateOpts = struct { cdp_id: ?i64 = null, reason: NavigateReason = .address_bar, - method: HttpClient.Method = .GET, + method: Http.Method = .GET, body: ?[]const u8 = null, header: ?[:0]const u8 = null, }; diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index 45dc71da..a48a956a 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -30,7 +30,7 @@ const URL = @import("../../url.zig").URL; const Mime = @import("../mime.zig").Mime; const parser = @import("../netsurf.zig"); const Page = @import("../page.zig").Page; -const HttpClient = @import("../../http/Client.zig"); +const Http = @import("../../http/Http.zig"); const CookieJar = @import("../storage/storage.zig").CookieJar; // XHR interfaces @@ -80,12 +80,12 @@ const XMLHttpRequestBodyInit = union(enum) { pub const XMLHttpRequest = struct { proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{}, arena: Allocator, - transfer: ?*HttpClient.Transfer = null, + transfer: ?*Http.Transfer = null, err: ?anyerror = null, last_dispatch: i64 = 0, send_flag: bool = false, - method: HttpClient.Method, + method: Http.Method, state: State, url: ?[:0]const u8 = null, @@ -320,7 +320,7 @@ pub const XMLHttpRequest = struct { } const methods = [_]struct { - tag: HttpClient.Method, + tag: Http.Method, name: []const u8, }{ .{ .tag = .DELETE, .name = "DELETE" }, @@ -330,7 +330,7 @@ pub const XMLHttpRequest = struct { .{ .tag = .POST, .name = "POST" }, .{ .tag = .PUT, .name = "PUT" }, }; - pub fn validMethod(m: []const u8) DOMError!HttpClient.Method { + pub fn validMethod(m: []const u8) DOMError!Http.Method { for (methods) |method| { if (std.ascii.eqlIgnoreCase(method.name, m)) { return method.tag; @@ -370,7 +370,7 @@ pub const XMLHttpRequest = struct { } } - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); for (self.headers.items) |hdr| { try headers.add(hdr); } @@ -393,18 +393,19 @@ pub const XMLHttpRequest = struct { }); } - fn httpStartCallback(transfer: *HttpClient.Transfer) !void { + fn httpStartCallback(transfer: *Http.Transfer) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" }); self.transfer = transfer; } - fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: []const u8) !void { + fn httpHeaderCallback(transfer: *Http.Transfer, header: Http.Header) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); - try self.response_headers.append(self.arena, try self.arena.dupe(u8, header)); + const joined = try std.fmt.allocPrint(self.arena, "{s}: {s}", .{ header.name, header.value }); + try self.response_headers.append(self.arena, joined); } - fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !void { + fn httpHeaderDoneCallback(transfer: *Http.Transfer) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); const header = &transfer.response_header.?; @@ -434,7 +435,7 @@ pub const XMLHttpRequest = struct { self.dispatchEvt("readystatechange"); } - fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { + fn httpDataCallback(transfer: *Http.Transfer, data: []const u8) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); try self.response_bytes.appendSlice(self.arena, data); diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 8e693718..1d7f19be 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -387,6 +387,12 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn deinit(self: *Self) void { self.inspector.deinit(); + // abort all intercepted requests before closing the sesion/page + // since some of these might callback into the page/scriptmanager + for (self.intercept_state.pendingTransfers()) |transfer| { + transfer.abort(); + } + // If the session has a page, we need to clear it first. The page // context is always nested inside of the isolated world context, // so we need to shutdown the page one first. @@ -406,10 +412,6 @@ pub fn BrowserContext(comptime CDP_T: type) type { log.warn(.http, "restoreOriginalProxy", .{ .err = err }); }; } - - for (self.intercept_state.pendingTransfers()) |transfer| { - transfer.abort(); - } self.intercept_state.deinit(); } diff --git a/src/cdp/domains/dom.zig b/src/cdp/domains/dom.zig index a88932fc..294f78a7 100644 --- a/src/cdp/domains/dom.zig +++ b/src/cdp/domains/dom.zig @@ -304,7 +304,7 @@ fn describeNode(cmd: anytype) !void { pierce: bool = false, })) orelse return error.InvalidParams; - if (params.depth != 1 or params.pierce) return error.NotYetImplementedParams; + if (params.depth != 1 or params.pierce) return error.NotImplemented; const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const node = try getNode(cmd.arena, bc, params.nodeId, params.backendNodeId, params.objectId); diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig index b13adc29..0d4743de 100644 --- a/src/cdp/domains/fetch.zig +++ b/src/cdp/domains/fetch.zig @@ -22,8 +22,7 @@ const Allocator = std.mem.Allocator; const log = @import("../../log.zig"); const network = @import("network.zig"); -const Method = @import("../../http/Client.zig").Method; -const Transfer = @import("../../http/Client.zig").Transfer; +const Http = @import("../../http/Http.zig"); const Notification = @import("../../notification.zig").Notification; pub fn processMessage(cmd: anytype) !void { @@ -32,6 +31,7 @@ pub fn processMessage(cmd: anytype) !void { enable, continueRequest, failRequest, + fulfillRequest, }, cmd.input.action) orelse return error.UnknownMethod; switch (action) { @@ -39,13 +39,14 @@ pub fn processMessage(cmd: anytype) !void { .enable => return enable(cmd), .continueRequest => return continueRequest(cmd), .failRequest => return failRequest(cmd), + .fulfillRequest => return fulfillRequest(cmd), } } // Stored in CDP pub const InterceptState = struct { allocator: Allocator, - waiting: std.AutoArrayHashMapUnmanaged(u64, *Transfer), + waiting: std.AutoArrayHashMapUnmanaged(u64, *Http.Transfer), pub fn init(allocator: Allocator) !InterceptState { return .{ @@ -58,11 +59,11 @@ pub const InterceptState = struct { return self.waiting.count() == 0; } - pub fn put(self: *InterceptState, transfer: *Transfer) !void { + pub fn put(self: *InterceptState, transfer: *Http.Transfer) !void { return self.waiting.put(self.allocator, transfer.id, transfer); } - pub fn remove(self: *InterceptState, id: u64) ?*Transfer { + pub fn remove(self: *InterceptState, id: u64) ?*Http.Transfer { const entry = self.waiting.fetchSwapRemove(id) orelse return null; return entry.value; } @@ -71,7 +72,7 @@ pub const InterceptState = struct { self.waiting.deinit(self.allocator); } - pub fn pendingTransfers(self: *const InterceptState) []*Transfer { + pub fn pendingTransfers(self: *const InterceptState) []*Http.Transfer { return self.waiting.values(); } }; @@ -204,17 +205,17 @@ pub fn requestIntercept(arena: Allocator, bc: anytype, intercept: *const Notific .networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), }, .{ .session_id = session_id }); + log.debug(.cdp, "request intercept", .{ + .state = "paused", + .id = transfer.id, + .url = transfer.uri, + }); // Await either continueRequest, failRequest or fulfillRequest intercept.wait_for_interception.* = true; page.request_intercepted = true; } -const HeaderEntry = struct { - name: []const u8, - value: []const u8, -}; - fn continueRequest(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const params = (try cmd.params(struct { @@ -222,12 +223,12 @@ fn continueRequest(cmd: anytype) !void { url: ?[]const u8 = null, method: ?[]const u8 = null, postData: ?[]const u8 = null, - headers: ?[]const HeaderEntry = null, + headers: ?[]const Http.Header = null, interceptResponse: bool = false, })) orelse return error.InvalidParams; - if (params.postData != null or params.headers != null or params.interceptResponse) { - return error.NotYetImplementedParams; + if (params.interceptResponse) { + return error.NotImplemented; } const page = bc.session.currentPage() orelse return error.PageNotLoaded; @@ -236,20 +237,32 @@ fn continueRequest(cmd: anytype) !void { const request_id = try idFromRequestId(params.requestId); const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound; + log.debug(.cdp, "request intercept", .{ + .state = "contiune", + .id = transfer.id, + .url = transfer.uri, + .new_url = params.url, + }); + // Update the request with the new parameters if (params.url) |url| { - // The request url must be modified in a way that's not observable by page. - // So page.url is not updated. try transfer.updateURL(try page.arena.dupeZ(u8, url)); } if (params.method) |method| { - transfer.req.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams; + transfer.req.method = std.meta.stringToEnum(Http.Method, method) orelse return error.InvalidParams; + } + + if (params.headers) |headers| { + try transfer.replaceRequestHeaders(cmd.arena, headers); + } + + if (params.postData) |b| { + const decoder = std.base64.standard.Decoder; + const body = try bc.arena.alloc(u8, try decoder.calcSizeForSlice(b)); + try decoder.decode(body, b); + transfer.req.body = body; } - log.info(.cdp, "Request continued by intercept", .{ - .id = params.requestId, - .url = transfer.uri, - }); try bc.cdp.browser.http_client.process(transfer); if (intercept_state.empty()) { @@ -259,6 +272,48 @@ fn continueRequest(cmd: anytype) !void { return cmd.sendResult(null, .{}); } +fn fulfillRequest(cmd: anytype) !void { + const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; + + const params = (try cmd.params(struct { + requestId: []const u8, // "INTERCEPT-{d}" + responseCode: u16, + responseHeaders: ?[]const Http.Header = null, + binaryResponseHeaders: ?[]const u8 = null, + body: ?[]const u8 = null, + responsePhrase: ?[]const u8 = null, + })) orelse return error.InvalidParams; + + if (params.binaryResponseHeaders != null) { + log.warn(.cdp, "not implemented", .{ .feature = "Fetch.fulfillRequest binaryResponseHeade" }); + return error.NotImplemented; + } + + var intercept_state = &bc.intercept_state; + const request_id = try idFromRequestId(params.requestId); + const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound; + + log.debug(.cdp, "request intercept", .{ + .state = "fulfilled", + .id = transfer.id, + .url = transfer.uri, + .status = params.responseCode, + .body = params.body != null, + }); + + var body: ?[]const u8 = null; + if (params.body) |b| { + const decoder = std.base64.standard.Decoder; + const buf = try cmd.arena.alloc(u8, try decoder.calcSizeForSlice(b)); + try decoder.decode(buf, b); + body = buf; + } + + try transfer.fulfill(params.responseCode, params.responseHeaders orelse &.{}, body); + + return cmd.sendResult(null, .{}); +} + fn failRequest(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const params = (try cmd.params(struct { @@ -272,13 +327,18 @@ fn failRequest(cmd: anytype) !void { const request_id = try idFromRequestId(params.requestId); const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound; - transfer.abort(); + defer transfer.abort(); + + log.info(.cdp, "request intercept", .{ + .state = "fail", + .id = request_id, + .url = transfer.uri, + .reason = params.errorReason, + }); if (intercept_state.empty()) { page.request_intercepted = false; } - - log.info(.cdp, "Request aborted by intercept", .{ .reason = params.errorReason }); return cmd.sendResult(null, .{}); } diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 7efb6cf6..81329ff0 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -121,7 +121,7 @@ fn deleteCookies(cmd: anytype) !void { path: ?[]const u8 = null, partitionKey: ?CdpStorage.CookiePartitionKey = null, })) orelse return error.InvalidParams; - if (params.partitionKey != null) return error.NotYetImplementedParams; + if (params.partitionKey != null) return error.NotImplemented; const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const cookies = &bc.session.cookie_jar.cookies; @@ -413,8 +413,8 @@ test "cdp.network setExtraHTTPHeaders" { var ctx = testing.context(); defer ctx.deinit(); - // _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" }); - try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } }); + _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" }); + // try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } }); try ctx.processMessage(.{ .id = 3, @@ -430,9 +430,6 @@ test "cdp.network setExtraHTTPHeaders" { const bc = ctx.cdp().browser_context.?; try testing.expectEqual(bc.extra_headers.items.len, 1); - - try ctx.processMessage(.{ .id = 5, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } }); - try testing.expectEqual(bc.extra_headers.items.len, 0); } test "cdp.Network: cookies" { diff --git a/src/cdp/domains/storage.zig b/src/cdp/domains/storage.zig index f26bdfef..53e8320c 100644 --- a/src/cdp/domains/storage.zig +++ b/src/cdp/domains/storage.zig @@ -129,7 +129,7 @@ pub const CdpCookie = struct { pub fn setCdpCookie(cookie_jar: *CookieJar, param: CdpCookie) !void { if (param.priority != .Medium or param.sameParty != null or param.sourceScheme != null or param.partitionKey != null) { - return error.NotYetImplementedParams; + return error.NotImplemented; } var arena = std.heap.ArenaAllocator.init(cookie_jar.allocator); diff --git a/src/http/Client.zig b/src/http/Client.zig index d03fae76..cddb9483 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -19,10 +19,10 @@ const std = @import("std"); const log = @import("../log.zig"); const builtin = @import("builtin"); + const Http = @import("Http.zig"); -pub const Headers = Http.Headers; const Notification = @import("../notification.zig").Notification; -const storage = @import("../browser/storage/storage.zig"); +const CookieJar = @import("../browser/storage/storage.zig").CookieJar; const c = Http.c; @@ -32,7 +32,7 @@ const ArenaAllocator = std.heap.ArenaAllocator; const errorCheck = Http.errorCheck; const errorMCheck = Http.errorMCheck; -pub const Method = Http.Method; +const Method = Http.Method; // This is loosely tied to a browser Page. Loading all the , doing // XHR requests, and loading imports all happens through here. Sine the app @@ -314,6 +314,10 @@ fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) !void { try conn.setMethod(req.method); if (req.body) |b| { try conn.setBody(b); + } else if (req.method == .POST) { + // libcurl will crash if the method is POST but there's no body + // TODO: is there a setting for that..seems weird. + try conn.setBody(""); } var header_list = req.headers; @@ -503,7 +507,7 @@ pub const RequestCookie = struct { origin: *const std.Uri, jar: *@import("../browser/storage/cookie.zig").Jar, - pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Headers) !void { + pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Http.Headers) !void { const uri = std.Uri.parse(url) catch |err| { log.warn(.http, "invalid url", .{ .err = err, .url = url }); return error.InvalidUrl; @@ -526,16 +530,16 @@ pub const RequestCookie = struct { pub const Request = struct { method: Method, url: [:0]const u8, - headers: Headers, + headers: Http.Headers, body: ?[]const u8 = null, - cookie_jar: *storage.CookieJar, + cookie_jar: *CookieJar, resource_type: ResourceType, // arbitrary data that can be associated with this request ctx: *anyopaque = undefined, start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null, - header_callback: ?*const fn (transfer: *Transfer, header: []const u8) anyerror!void = null, + header_callback: ?*const fn (transfer: *Transfer, header: Http.Header) anyerror!void = null, header_done_callback: *const fn (transfer: *Transfer) anyerror!void, data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void, done_callback: *const fn (ctx: *anyopaque) anyerror!void, @@ -557,7 +561,7 @@ pub const Transfer = struct { _notified_fail: bool = false, // We'll store the response header here - response_header: ?Header = null, + response_header: ?ResponseHeader = null, _handle: ?*Handle = null, @@ -594,6 +598,22 @@ pub const Transfer = struct { self.req.url = url; } + pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const Http.Header) !void { + self.req.headers.deinit(); + + var buf: std.ArrayListUnmanaged(u8) = .empty; + var new_headers = try Http.Headers.init(); + for (headers) |hdr| { + // safe to re-use this buffer, because Headers.add because curl copies + // the value we pass into curl_slist_append. + defer buf.clearRetainingCapacity(); + try std.fmt.format(buf.writer(allocator), "{s}: {s}", .{ hdr.name, hdr.value }); + try buf.append(allocator, 0); // null terminated + try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]); + } + self.req.headers = new_headers; + } + pub fn abort(self: *Transfer) void { self.client.requestFailed(self, error.Abort); if (self._handle != null) { @@ -675,7 +695,7 @@ pub const Transfer = struct { if (getResponseHeader(easy, "content-type", 0)) |ct| { var hdr = &transfer.response_header.?; const value = ct.value; - const len = @min(value.len, hdr._content_type.len); + const len = @min(value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN); hdr._content_type_len = len; @memcpy(hdr._content_type[0..len], value[0..len]); } @@ -704,10 +724,12 @@ pub const Transfer = struct { } } else { if (transfer.req.header_callback) |cb| { - cb(transfer, header) catch |err| { - log.err(.http, "header_callback", .{ .err = err, .req = transfer }); - return 0; - }; + if (Http.Headers.parseHeader(header)) |hdr| { + cb(transfer, hdr) catch |err| { + log.err(.http, "header_callback", .{ .err = err, .req = transfer }); + return 0; + }; + } } } return buf_len; @@ -746,15 +768,64 @@ pub const Transfer = struct { try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_PRIVATE, &private)); return @alignCast(@ptrCast(private)); } + + pub fn fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void { + if (transfer._handle != null) { + // should never happen, should have been intercepted/paused, and then + // either continued, aborted and fulfilled once. + @branchHint(.unlikely); + return error.RequestInProgress; + } + + transfer._fulfill(status, headers, body) catch |err| { + transfer.req.error_callback(transfer.req.ctx, err); + return err; + }; + } + + fn _fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void { + const req = &transfer.req; + if (req.start_callback) |cb| { + try cb(transfer); + } + + if (req.header_callback) |cb| { + for (headers) |hdr| { + try cb(transfer, hdr); + } + } + + transfer.response_header = .{ + .status = status, + .url = req.url, + }; + for (headers) |hdr| { + if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { + const len = @min(hdr.value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN); + @memcpy(transfer.response_header.?._content_type[0..len], hdr.value[0..len]); + break; + } + } + + try req.header_done_callback(transfer); + + if (body) |b| { + try req.data_callback(transfer, b); + } + + try req.done_callback(req.ctx); + } }; -pub const Header = struct { +pub const ResponseHeader = struct { + const MAX_CONTENT_TYPE_LEN = 64; + status: u16, url: [*c]const u8, _content_type_len: usize = 0, - _content_type: [64]u8 = undefined, + _content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined, - pub fn contentType(self: *Header) ?[]u8 { + pub fn contentType(self: *ResponseHeader) ?[]u8 { if (self._content_type_len == 0) { return null; } @@ -766,7 +837,7 @@ const HeaderIterator = struct { easy: *c.CURL, prev: ?*c.curl_header = null, - pub fn next(self: *HeaderIterator) ?struct { name: []const u8, value: []const u8 } { + pub fn next(self: *HeaderIterator) ?Http.Header { const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null; self.prev = h; @@ -778,12 +849,12 @@ const HeaderIterator = struct { } }; -const ResponseHeader = struct { +const CurlHeaderValue = struct { value: []const u8, amount: usize, }; -fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?ResponseHeader { +fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?CurlHeaderValue { var hdr: [*c]c.curl_header = null; const result = c.curl_easy_header(easy, name, index, c.CURLH_HEADER, -1, &hdr); if (result == c.CURLE_OK) { diff --git a/src/http/Http.zig b/src/http/Http.zig index 85187d6b..5bfd8950 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -22,14 +22,15 @@ pub const c = @cImport({ @cInclude("curl/curl.h"); }); -const Client = @import("Client.zig"); +pub const ENABLE_DEBUG = false; +pub const Client = @import("Client.zig"); +pub const Transfer = Client.Transfer; + const errors = @import("errors.zig"); const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; -pub const ENABLE_DEBUG = false; - // Client.zig does the bulk of the work and is loosely tied to a browser Page. // But we still need something above Client.zig for the "utility" http stuff // we need to do, like telemetry. The most important thing we want from this @@ -221,15 +222,15 @@ pub const Connection = struct { } }; +pub const Header = struct { + name: []const u8, + value: []const u8, +}; + pub const Headers = struct { headers: *c.curl_slist, cookies: ?[*c]const u8, - const Header = struct { - name: []const u8, - value: []const u8, - }; - pub fn init() !Headers { const header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0"); if (header_list == null) return error.OutOfMemory;