diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index d637418c..4a27e054 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -24,8 +24,8 @@ const parser = @import("netsurf.zig"); const Env = @import("env.zig").Env; const Page = @import("page.zig").Page; const DataURI = @import("DataURI.zig"); +const Http = @import("../http/Http.zig"); const Browser = @import("browser.zig").Browser; -const HttpClient = @import("../http/Client.zig"); const URL = @import("../url.zig").URL; const Allocator = std.mem.Allocator; @@ -57,7 +57,7 @@ deferreds: OrderList, shutdown: bool = false, -client: *HttpClient, +client: *Http.Client, allocator: Allocator, buffer_pool: BufferPool, script_pool: std.heap.MemoryPool(PendingScript), @@ -229,7 +229,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { errdefer pending_script.deinit(); - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); try page.requestCookie(.{}).headersForRequest(page.arena, remote_url.?, &headers); try self.client.request(.{ @@ -238,6 +238,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { .method = .GET, .headers = headers, .cookie_jar = page.cookie_jar, + .resource_type = .script, .start_callback = if (log.enabled(.http, .debug)) startCallback else null, .header_done_callback = headerCallback, .data_callback = dataCallback, @@ -296,7 +297,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { .buffer_pool = &self.buffer_pool, }; - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); try self.page.requestCookie(.{}).headersForRequest(self.page.arena, url, &headers); var client = self.client; @@ -306,6 +307,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { .headers = headers, .cookie_jar = self.page.cookie_jar, .ctx = &blocking, + .resource_type = .script, .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null, .header_done_callback = Blocking.headerCallback, .data_callback = Blocking.dataCallback, @@ -423,7 +425,7 @@ fn getList(self: *ScriptManager, script: *const Script) *OrderList { return &self.scripts; } -fn startCallback(transfer: *HttpClient.Transfer) !void { +fn startCallback(transfer: *Http.Transfer) !void { const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx)); script.startCallback(transfer) catch |err| { log.err(.http, "SM.startCallback", .{ .err = err, .transfer = transfer }); @@ -431,7 +433,7 @@ fn startCallback(transfer: *HttpClient.Transfer) !void { }; } -fn headerCallback(transfer: *HttpClient.Transfer) !void { +fn headerCallback(transfer: *Http.Transfer) !void { const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx)); script.headerCallback(transfer) catch |err| { log.err(.http, "SM.headerCallback", .{ @@ -443,7 +445,7 @@ fn headerCallback(transfer: *HttpClient.Transfer) !void { }; } -fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { +fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void { const script: *PendingScript = @alignCast(@ptrCast(transfer.ctx)); script.dataCallback(transfer, data) catch |err| { log.err(.http, "SM.dataCallback", .{ .err = err, .transfer = transfer, .len = data.len }); @@ -488,12 +490,12 @@ const PendingScript = struct { } } - fn startCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void { + fn startCallback(self: *PendingScript, transfer: *Http.Transfer) !void { _ = self; log.debug(.http, "script fetch start", .{ .req = transfer }); } - fn headerCallback(self: *PendingScript, transfer: *HttpClient.Transfer) !void { + fn headerCallback(self: *PendingScript, transfer: *Http.Transfer) !void { const header = &transfer.response_header.?; log.debug(.http, "script header", .{ .req = transfer, @@ -513,7 +515,7 @@ const PendingScript = struct { self.script.source = .{ .remote = self.manager.buffer_pool.get() }; } - fn dataCallback(self: *PendingScript, transfer: *HttpClient.Transfer, data: []const u8) !void { + fn dataCallback(self: *PendingScript, transfer: *Http.Transfer, data: []const u8) !void { _ = transfer; // too verbose // log.debug(.http, "script data chunk", .{ @@ -766,11 +768,11 @@ const Blocking = struct { done: BlockingResult, }; - fn startCallback(transfer: *HttpClient.Transfer) !void { + fn startCallback(transfer: *Http.Transfer) !void { log.debug(.http, "script fetch start", .{ .req = transfer, .blocking = true }); } - fn headerCallback(transfer: *HttpClient.Transfer) !void { + fn headerCallback(transfer: *Http.Transfer) !void { const header = &transfer.response_header.?; log.debug(.http, "script header", .{ .req = transfer, @@ -787,7 +789,7 @@ const Blocking = struct { self.buffer = self.buffer_pool.get(); } - fn dataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { + fn dataCallback(transfer: *Http.Transfer, data: []const u8) !void { // too verbose // log.debug(.http, "script data chunk", .{ // .req = transfer, diff --git a/src/browser/page.zig b/src/browser/page.zig index a02a447b..0189ae82 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -30,7 +30,7 @@ const Renderer = @import("renderer.zig").Renderer; const Window = @import("html/window.zig").Window; const Walker = @import("dom/walker.zig").WalkerDepthFirst; const Scheduler = @import("Scheduler.zig"); -const HttpClient = @import("../http/Client.zig"); +const Http = @import("../http/Http.zig"); const ScriptManager = @import("ScriptManager.zig"); const HTMLDocument = @import("html/document.zig").HTMLDocument; @@ -87,13 +87,23 @@ pub const Page = struct { polyfill_loader: polyfill.Loader = .{}, scheduler: Scheduler, - http_client: *HttpClient, + http_client: *Http.Client, script_manager: ScriptManager, mode: Mode, load_state: LoadState = .parsing, + // Page.wait balances waiting for resources / tasks and producing an output. + // Up until a timeout, Page.wait will always wait for inflight or pending + // HTTP requests, via the Http.Client.active counter. However, intercepted + // requests (via CDP, but it could be anything), aren't considered "active" + // connection. So it's possible that we have intercepted requests (which are + // pending on some driver to continue/abort) while Http.Client.active == 0. + // This boolean exists to supplment Http.Client.active and inform Page.wait + // of pending connections. + request_intercepted: bool = false, + const Mode = union(enum) { pre: void, err: anyerror, @@ -275,16 +285,26 @@ pub const Page = struct { while (true) { SW: switch (self.mode) { .pre, .raw => { + if (self.request_intercepted) { + // the page request was intercepted. + + // there shouldn't be any active requests; + std.debug.assert(http_client.active == 0); + + // nothing we can do for this, need to kick the can up + // the chain and wait for activity (e.g. a CDP message) + // to unblock this. + return; + } + // The main page hasn't started/finished navigating. // There's no JS to run, and no reason to run the scheduler. - if (http_client.active == 0) { // haven't started navigating, I guess. return; } // There should only be 1 active http transfer, the main page - std.debug.assert(http_client.active == 1); try http_client.tick(ms_remaining); }, .html, .parsed => { @@ -330,20 +350,35 @@ pub const Page = struct { _ = try scheduler.runLowPriority(); - // We'll block here, waiting for network IO. We know - // when the next timeout is scheduled, and we know how long - // the caller wants to wait for, so we can pick a good wait - // duration - const ms_to_wait = @min(ms_remaining, ms_to_next_task orelse 1000); + const request_intercepted = self.request_intercepted; + + // We want to prioritize processing intercepted requests + // because, the sooner they get unblocked, the sooner we + // can start the HTTP request. But we still want to advanced + // existing HTTP requests, if possible. So, if we have + // intercepted requests, we'll still look at existing HTTP + // requests, but we won't block waiting for more data. + const ms_to_wait = + if (request_intercepted) 0 + + // But if we have no intercepted requests, we'll wait + // for as long as we can for data to our existing + // inflight requests + else @min(ms_remaining, ms_to_next_task orelse 1000); + try http_client.tick(ms_to_wait); - if (try_catch.hasCaught()) { - const msg = (try try_catch.err(self.arena)) orelse "unknown"; - log.warn(.user_script, "page wait", .{ .err = msg, .src = "data" }); - return error.JsError; + if (request_intercepted) { + // Again, proritizing intercepted requests. Exit this + // loop so that our caller can hopefully resolve them + // (i.e. continue or abort them); + return; } }, - .err => |err| return err, + .err => |err| { + self.mode = .{ .raw_done = @errorName(err) }; + return err; + }, .raw_done => return, } @@ -362,7 +397,7 @@ pub const Page = struct { std.debug.print("\nactive requests: {d}\n", .{self.http_client.active}); var n_ = self.http_client.handles.in_use.first; while (n_) |n| { - const transfer = HttpClient.Transfer.fromEasy(n.data.conn.easy) catch |err| { + const transfer = Http.Transfer.fromEasy(n.data.conn.easy) catch |err| { std.debug.print(" - failed to load transfer: {any}\n", .{err}); break; }; @@ -435,7 +470,7 @@ pub const Page = struct { is_http: bool = true, is_navigation: bool = false, }; - pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie { + pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) Http.Client.RequestCookie { return .{ .jar = self.cookie_jar, .origin = &self.url.uri, @@ -473,7 +508,7 @@ pub const Page = struct { const owned_url = try self.arena.dupeZ(u8, request_url); self.url = try URL.parse(owned_url, null); - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); if (opts.header) |hdr| try headers.add(hdr); try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, owned_url, &headers); @@ -484,6 +519,7 @@ pub const Page = struct { .headers = headers, .body = opts.body, .cookie_jar = self.cookie_jar, + .resource_type = .document, .header_done_callback = pageHeaderDoneCallback, .data_callback = pageDataCallback, .done_callback = pageDoneCallback, @@ -563,7 +599,7 @@ pub const Page = struct { ); } - fn pageHeaderDoneCallback(transfer: *HttpClient.Transfer) !void { + fn pageHeaderDoneCallback(transfer: *Http.Transfer) !void { var self: *Page = @alignCast(@ptrCast(transfer.ctx)); // would be different than self.url in the case of a redirect @@ -578,7 +614,7 @@ pub const Page = struct { }); } - fn pageDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { + fn pageDataCallback(transfer: *Http.Transfer, data: []const u8) !void { var self: *Page = @alignCast(@ptrCast(transfer.ctx)); if (self.mode == .pre) { @@ -1002,7 +1038,7 @@ pub const NavigateReason = enum { pub const NavigateOpts = struct { cdp_id: ?i64 = null, reason: NavigateReason = .address_bar, - method: HttpClient.Method = .GET, + method: Http.Method = .GET, body: ?[]const u8 = null, header: ?[:0]const u8 = null, }; diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index 720b813b..a48a956a 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -30,7 +30,7 @@ const URL = @import("../../url.zig").URL; const Mime = @import("../mime.zig").Mime; const parser = @import("../netsurf.zig"); const Page = @import("../page.zig").Page; -const HttpClient = @import("../../http/Client.zig"); +const Http = @import("../../http/Http.zig"); const CookieJar = @import("../storage/storage.zig").CookieJar; // XHR interfaces @@ -80,12 +80,12 @@ const XMLHttpRequestBodyInit = union(enum) { pub const XMLHttpRequest = struct { proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{}, arena: Allocator, - transfer: ?*HttpClient.Transfer = null, + transfer: ?*Http.Transfer = null, err: ?anyerror = null, last_dispatch: i64 = 0, send_flag: bool = false, - method: HttpClient.Method, + method: Http.Method, state: State, url: ?[:0]const u8 = null, @@ -320,7 +320,7 @@ pub const XMLHttpRequest = struct { } const methods = [_]struct { - tag: HttpClient.Method, + tag: Http.Method, name: []const u8, }{ .{ .tag = .DELETE, .name = "DELETE" }, @@ -330,7 +330,7 @@ pub const XMLHttpRequest = struct { .{ .tag = .POST, .name = "POST" }, .{ .tag = .PUT, .name = "PUT" }, }; - pub fn validMethod(m: []const u8) DOMError!HttpClient.Method { + pub fn validMethod(m: []const u8) DOMError!Http.Method { for (methods) |method| { if (std.ascii.eqlIgnoreCase(method.name, m)) { return method.tag; @@ -370,7 +370,7 @@ pub const XMLHttpRequest = struct { } } - var headers = try HttpClient.Headers.init(); + var headers = try Http.Headers.init(); for (self.headers.items) |hdr| { try headers.add(hdr); } @@ -383,6 +383,7 @@ pub const XMLHttpRequest = struct { .headers = headers, .body = self.request_body, .cookie_jar = page.cookie_jar, + .resource_type = .xhr, .start_callback = httpStartCallback, .header_callback = httpHeaderCallback, .header_done_callback = httpHeaderDoneCallback, @@ -392,18 +393,19 @@ pub const XMLHttpRequest = struct { }); } - fn httpStartCallback(transfer: *HttpClient.Transfer) !void { + fn httpStartCallback(transfer: *Http.Transfer) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" }); self.transfer = transfer; } - fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: []const u8) !void { + fn httpHeaderCallback(transfer: *Http.Transfer, header: Http.Header) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); - try self.response_headers.append(self.arena, try self.arena.dupe(u8, header)); + const joined = try std.fmt.allocPrint(self.arena, "{s}: {s}", .{ header.name, header.value }); + try self.response_headers.append(self.arena, joined); } - fn httpHeaderDoneCallback(transfer: *HttpClient.Transfer) !void { + fn httpHeaderDoneCallback(transfer: *Http.Transfer) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); const header = &transfer.response_header.?; @@ -433,7 +435,7 @@ pub const XMLHttpRequest = struct { self.dispatchEvt("readystatechange"); } - fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { + fn httpDataCallback(transfer: *Http.Transfer, data: []const u8) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); try self.response_bytes.appendSlice(self.arena, data); diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 2435f7fd..1d7f19be 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -74,11 +74,6 @@ pub fn CDPT(comptime TypeProvider: type) type { // Used for processing notifications within a browser context. notification_arena: std.heap.ArenaAllocator, - // Extra headers to add to all requests. TBD under which conditions this should be reset. - extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty, - - intercept_state: InterceptState, - const Self = @This(); pub fn init(app: *App, client: TypeProvider.Client) !Self { @@ -93,7 +88,6 @@ pub fn CDPT(comptime TypeProvider: type) type { .browser_context = null, .message_arena = std.heap.ArenaAllocator.init(allocator), .notification_arena = std.heap.ArenaAllocator.init(allocator), - .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena? }; } @@ -101,7 +95,6 @@ pub fn CDPT(comptime TypeProvider: type) type { if (self.browser_context) |*bc| { bc.deinit(); } - self.intercept_state.deinit(); // TBD Should this live in BC? self.browser.deinit(); self.message_arena.deinit(); self.notification_arena.deinit(); @@ -346,6 +339,11 @@ pub fn BrowserContext(comptime CDP_T: type) type { http_proxy_changed: bool = false, + // Extra headers to add to all requests. + extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty, + + intercept_state: InterceptState, + const Self = @This(); fn init(self: *Self, id: []const u8, cdp: *CDP_T) !void { @@ -375,6 +373,7 @@ pub fn BrowserContext(comptime CDP_T: type) type { .isolated_world = null, .inspector = inspector, .notification_arena = cdp.notification_arena.allocator(), + .intercept_state = try InterceptState.init(allocator), }; self.node_search_list = Node.Search.List.init(allocator, &self.node_registry); errdefer self.deinit(); @@ -388,6 +387,12 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn deinit(self: *Self) void { self.inspector.deinit(); + // abort all intercepted requests before closing the sesion/page + // since some of these might callback into the page/scriptmanager + for (self.intercept_state.pendingTransfers()) |transfer| { + transfer.abort(); + } + // If the session has a page, we need to clear it first. The page // context is always nested inside of the isolated world context, // so we need to shutdown the page one first. @@ -407,6 +412,7 @@ pub fn BrowserContext(comptime CDP_T: type) type { log.warn(.http, "restoreOriginalProxy", .{ .err = err }); }; } + self.intercept_state.deinit(); } pub fn reset(self: *Self) void { @@ -495,7 +501,7 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn onHttpRequestIntercept(ctx: *anyopaque, data: *const Notification.RequestIntercept) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); - try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data); + try @import("domains/fetch.zig").requestIntercept(self.notification_arena, self, data); } pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void { diff --git a/src/cdp/domains/dom.zig b/src/cdp/domains/dom.zig index a88932fc..294f78a7 100644 --- a/src/cdp/domains/dom.zig +++ b/src/cdp/domains/dom.zig @@ -304,7 +304,7 @@ fn describeNode(cmd: anytype) !void { pierce: bool = false, })) orelse return error.InvalidParams; - if (params.depth != 1 or params.pierce) return error.NotYetImplementedParams; + if (params.depth != 1 or params.pierce) return error.NotImplemented; const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const node = try getNode(cmd.arena, bc, params.nodeId, params.backendNodeId, params.objectId); diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig index 6b02ae58..0d4743de 100644 --- a/src/cdp/domains/fetch.zig +++ b/src/cdp/domains/fetch.zig @@ -18,10 +18,12 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const Notification = @import("../../notification.zig").Notification; + const log = @import("../../log.zig"); -const Method = @import("../../http/Client.zig").Method; -const Transfer = @import("../../http/Client.zig").Transfer; +const network = @import("network.zig"); + +const Http = @import("../../http/Http.zig"); +const Notification = @import("../../notification.zig").Notification; pub fn processMessage(cmd: anytype) !void { const action = std.meta.stringToEnum(enum { @@ -29,6 +31,7 @@ pub fn processMessage(cmd: anytype) !void { enable, continueRequest, failRequest, + fulfillRequest, }, cmd.input.action) orelse return error.UnknownMethod; switch (action) { @@ -36,27 +39,48 @@ pub fn processMessage(cmd: anytype) !void { .enable => return enable(cmd), .continueRequest => return continueRequest(cmd), .failRequest => return failRequest(cmd), + .fulfillRequest => return fulfillRequest(cmd), } } // Stored in CDP pub const InterceptState = struct { - const Self = @This(); - waiting: std.AutoArrayHashMap(u64, *Transfer), + allocator: Allocator, + waiting: std.AutoArrayHashMapUnmanaged(u64, *Http.Transfer), pub fn init(allocator: Allocator) !InterceptState { return .{ - .waiting = std.AutoArrayHashMap(u64, *Transfer).init(allocator), + .waiting = .empty, + .allocator = allocator, }; } - pub fn deinit(self: *Self) void { - self.waiting.deinit(); + pub fn empty(self: *const InterceptState) bool { + return self.waiting.count() == 0; + } + + pub fn put(self: *InterceptState, transfer: *Http.Transfer) !void { + return self.waiting.put(self.allocator, transfer.id, transfer); + } + + pub fn remove(self: *InterceptState, id: u64) ?*Http.Transfer { + const entry = self.waiting.fetchSwapRemove(id) orelse return null; + return entry.value; + } + + pub fn deinit(self: *InterceptState) void { + self.waiting.deinit(self.allocator); + } + + pub fn pendingTransfers(self: *const InterceptState) []*Http.Transfer { + return self.waiting.values(); } }; const RequestPattern = struct { - urlPattern: []const u8 = "*", // Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed. Escape character is backslash. Omitting is equivalent to "*". + // Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed. + // Escape character is backslash. Omitting is equivalent to "*". + urlPattern: []const u8 = "*", resourceType: ?ResourceType = null, requestStage: RequestStage = .Request, }; @@ -115,8 +139,14 @@ fn disable(cmd: anytype) !void { fn enable(cmd: anytype) !void { const params = (try cmd.params(EnableParam)) orelse EnableParam{}; - if (params.patterns.len != 0) log.warn(.cdp, "Fetch.enable No patterns yet", .{}); - if (params.handleAuthRequests) log.warn(.cdp, "Fetch.enable No auth yet", .{}); + if (!arePatternsSupported(params.patterns)) { + log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable advanced patterns are not" }); + return cmd.sendResult(null, .{}); + } + + if (params.handleAuthRequests) { + log.warn(.cdp, "not implemented", .{ .feature = "Fetch.enable handleAuthRequests is not supported yet" }); + } const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; try bc.fetchEnable(); @@ -124,57 +154,67 @@ fn enable(cmd: anytype) !void { return cmd.sendResult(null, .{}); } -pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void { - var cdp = bc.cdp; +fn arePatternsSupported(patterns: []RequestPattern) bool { + if (patterns.len == 0) { + return true; + } + if (patterns.len > 1) { + return false; + } + // While we don't support patterns, yet, both Playwright and Puppeteer send + // a default pattern which happens to be what we support: + // [{"urlPattern":"*","requestStage":"Request"}] + // So, rather than erroring on this case because we don't support patterns, + // we'll allow it, because this pattern is how it works as-is. + const pattern = patterns[0]; + if (!std.mem.eql(u8, pattern.urlPattern, "*")) { + return false; + } + if (pattern.resourceType != null) { + return false; + } + if (pattern.requestStage != .Request) { + return false; + } + return true; +} + +pub fn requestIntercept(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void { // unreachable because we _have_ to have a page. const session_id = bc.session_id orelse unreachable; const target_id = bc.target_id orelse unreachable; + const page = bc.session.currentPage() orelse unreachable; // We keep it around to wait for modifications to the request. // NOTE: we assume whomever created the request created it with a lifetime of the Page. // TODO: What to do when receiving replies for a previous page's requests? const transfer = intercept.transfer; - try cdp.intercept_state.waiting.put(transfer.id, transfer); + try bc.intercept_state.put(transfer); - // NOTE: .request data preparation is duped from network.zig - const full_request_url = transfer.uri; - const request_url = try @import("network.zig").urlToString(arena, &full_request_url, .{ - .scheme = true, - .authentication = true, - .authority = true, - .path = true, - .query = true, - }); - const request_fragment = try @import("network.zig").urlToString(arena, &full_request_url, .{ - .fragment = true, - }); - const headers = try transfer.req.headers.asHashMap(arena); - // End of duped code - - try cdp.sendEvent("Fetch.requestPaused", .{ + try bc.cdp.sendEvent("Fetch.requestPaused", .{ .requestId = try std.fmt.allocPrint(arena, "INTERCEPT-{d}", .{transfer.id}), - .request = .{ - .url = request_url, - .urlFragment = request_fragment, - .method = @tagName(transfer.req.method), - .hasPostData = transfer.req.body != null, - .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, - }, + .request = network.TransferAsRequestWriter.init(transfer), .frameId = target_id, - .resourceType = ResourceType.Document, // TODO! + .resourceType = switch (transfer.req.resource_type) { + .script => "Script", + .xhr => "XHR", + .document => "Document", + }, .networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), }, .{ .session_id = session_id }); + log.debug(.cdp, "request intercept", .{ + .state = "paused", + .id = transfer.id, + .url = transfer.uri, + }); // Await either continueRequest, failRequest or fulfillRequest - intercept.wait_for_interception.* = true; -} -const HeaderEntry = struct { - name: []const u8, - value: []const u8, -}; + intercept.wait_for_interception.* = true; + page.request_intercepted = true; +} fn continueRequest(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; @@ -183,49 +223,129 @@ fn continueRequest(cmd: anytype) !void { url: ?[]const u8 = null, method: ?[]const u8 = null, postData: ?[]const u8 = null, - headers: ?[]const HeaderEntry = null, + headers: ?[]const Http.Header = null, interceptResponse: bool = false, })) orelse return error.InvalidParams; - if (params.postData != null or params.headers != null or params.interceptResponse) return error.NotYetImplementedParams; + if (params.interceptResponse) { + return error.NotImplemented; + } + + const page = bc.session.currentPage() orelse return error.PageNotLoaded; + + var intercept_state = &bc.intercept_state; const request_id = try idFromRequestId(params.requestId); - const entry = bc.cdp.intercept_state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound; - const transfer = entry.value; + const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound; + + log.debug(.cdp, "request intercept", .{ + .state = "contiune", + .id = transfer.id, + .url = transfer.uri, + .new_url = params.url, + }); // Update the request with the new parameters if (params.url) |url| { - // The request url must be modified in a way that's not observable by page. So page.url is not updated. - try transfer.updateURL(try bc.cdp.browser.page_arena.allocator().dupeZ(u8, url)); + try transfer.updateURL(try page.arena.dupeZ(u8, url)); } if (params.method) |method| { - transfer.req.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams; + transfer.req.method = std.meta.stringToEnum(Http.Method, method) orelse return error.InvalidParams; + } + + if (params.headers) |headers| { + try transfer.replaceRequestHeaders(cmd.arena, headers); + } + + if (params.postData) |b| { + const decoder = std.base64.standard.Decoder; + const body = try bc.arena.alloc(u8, try decoder.calcSizeForSlice(b)); + try decoder.decode(body, b); + transfer.req.body = body; } - log.info(.cdp, "Request continued by intercept", .{ .id = params.requestId }); try bc.cdp.browser.http_client.process(transfer); + if (intercept_state.empty()) { + page.request_intercepted = false; + } + + return cmd.sendResult(null, .{}); +} + +fn fulfillRequest(cmd: anytype) !void { + const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; + + const params = (try cmd.params(struct { + requestId: []const u8, // "INTERCEPT-{d}" + responseCode: u16, + responseHeaders: ?[]const Http.Header = null, + binaryResponseHeaders: ?[]const u8 = null, + body: ?[]const u8 = null, + responsePhrase: ?[]const u8 = null, + })) orelse return error.InvalidParams; + + if (params.binaryResponseHeaders != null) { + log.warn(.cdp, "not implemented", .{ .feature = "Fetch.fulfillRequest binaryResponseHeade" }); + return error.NotImplemented; + } + + var intercept_state = &bc.intercept_state; + const request_id = try idFromRequestId(params.requestId); + const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound; + + log.debug(.cdp, "request intercept", .{ + .state = "fulfilled", + .id = transfer.id, + .url = transfer.uri, + .status = params.responseCode, + .body = params.body != null, + }); + + var body: ?[]const u8 = null; + if (params.body) |b| { + const decoder = std.base64.standard.Decoder; + const buf = try cmd.arena.alloc(u8, try decoder.calcSizeForSlice(b)); + try decoder.decode(buf, b); + body = buf; + } + + try transfer.fulfill(params.responseCode, params.responseHeaders orelse &.{}, body); + return cmd.sendResult(null, .{}); } fn failRequest(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; - var state = &bc.cdp.intercept_state; const params = (try cmd.params(struct { requestId: []const u8, // "INTERCEPT-{d}" errorReason: ErrorReason, })) orelse return error.InvalidParams; - const request_id = try idFromRequestId(params.requestId); - const entry = state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound; - // entry.value is the transfer - entry.value.abort(); + const page = bc.session.currentPage() orelse return error.PageNotLoaded; - log.info(.cdp, "Request aborted by intercept", .{ .reason = params.errorReason }); + var intercept_state = &bc.intercept_state; + const request_id = try idFromRequestId(params.requestId); + + const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound; + defer transfer.abort(); + + log.info(.cdp, "request intercept", .{ + .state = "fail", + .id = request_id, + .url = transfer.uri, + .reason = params.errorReason, + }); + + if (intercept_state.empty()) { + page.request_intercepted = false; + } return cmd.sendResult(null, .{}); } // Get u64 from requestId which is formatted as: "INTERCEPT-{d}" fn idFromRequestId(request_id: []const u8) !u64 { - if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) return error.InvalidParams; + if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) { + return error.InvalidParams; + } return std.fmt.parseInt(u64, request_id[10..], 10) catch return error.InvalidParams; } diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index d3f30be2..81329ff0 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -19,10 +19,10 @@ const std = @import("std"); const Allocator = std.mem.Allocator; -const Notification = @import("../../notification.zig").Notification; const log = @import("../../log.zig"); const CdpStorage = @import("storage.zig"); const Transfer = @import("../../http/Client.zig").Transfer; +const Notification = @import("../../notification.zig").Notification; pub fn processMessage(cmd: anytype) !void { const action = std.meta.stringToEnum(enum { @@ -83,7 +83,7 @@ fn setExtraHTTPHeaders(cmd: anytype) !void { // Copy the headers onto the browser context arena const arena = bc.arena; - const extra_headers = &bc.cdp.extra_headers; + const extra_headers = &bc.extra_headers; extra_headers.clearRetainingCapacity(); try extra_headers.ensureTotalCapacity(arena, params.headers.map.count()); @@ -121,7 +121,7 @@ fn deleteCookies(cmd: anytype) !void { path: ?[]const u8 = null, partitionKey: ?CdpStorage.CookiePartitionKey = null, })) orelse return error.InvalidParams; - if (params.partitionKey != null) return error.NotYetImplementedParams; + if (params.partitionKey != null) return error.NotImplemented; const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const cookies = &bc.session.cookie_jar.cookies; @@ -235,50 +235,16 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification const page = bc.session.currentPage() orelse unreachable; // Modify request with extra CDP headers - for (cdp.extra_headers.items) |extra| { + for (bc.extra_headers.items) |extra| { try data.transfer.req.headers.add(extra); } - const document_url = try urlToString(arena, &page.url.uri, .{ - .scheme = true, - .authentication = true, - .authority = true, - .path = true, - .query = true, - }); - const transfer = data.transfer; - const full_request_url = transfer.uri; - const request_url = try urlToString(arena, &full_request_url, .{ - .scheme = true, - .authentication = true, - .authority = true, - .path = true, - .query = true, - }); - const request_fragment = try urlToString(arena, &full_request_url, .{ - .fragment = true, // TODO since path is false, this likely does not work as intended - }); - - const headers = try transfer.req.headers.asHashMap(arena); - // We're missing a bunch of fields, but, for now, this seems like enough - try cdp.sendEvent("Network.requestWillBeSent", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), - .frameId = target_id, - .loaderId = bc.loader_id, - .documentUrl = document_url, - .request = .{ - .url = request_url, - .urlFragment = request_fragment, - .method = @tagName(transfer.req.method), - .hasPostData = transfer.req.body != null, - .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, - }, - }, .{ .session_id = session_id }); + try cdp.sendEvent("Network.requestWillBeSent", .{ .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), .frameId = target_id, .loaderId = bc.loader_id, .documentUrl = DocumentUrlWriter.init(&page.url.uri), .request = TransferAsRequestWriter.init(transfer) }, .{ .session_id = session_id }); } -pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void { +pub fn httpHeadersDone(arena: Allocator, bc: anytype, data: *const Notification.ResponseHeadersDone) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. std.debug.assert(bc.session.page != null); @@ -289,63 +255,166 @@ pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notificati const session_id = bc.session_id orelse unreachable; const target_id = bc.target_id orelse unreachable; - const url = try urlToString(arena, &request.transfer.uri, .{ - .scheme = true, - .authentication = true, - .authority = true, - .path = true, - .query = true, - }); - - const status = request.transfer.response_header.?.status; - // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.responseReceived", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.transfer.id}), + .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}), .loaderId = bc.loader_id, - .response = .{ - .url = url, - .status = status, - .statusText = @as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown", - .headers = ResponseHeaderWriter.init(request.transfer), - }, .frameId = target_id, + .response = TransferAsResponseWriter.init(data.transfer), }, .{ .session_id = session_id }); } -pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 { - var buf: std.ArrayListUnmanaged(u8) = .empty; - try url.writeToStream(opts, buf.writer(arena)); - return buf.items; -} - -const ResponseHeaderWriter = struct { +pub const TransferAsRequestWriter = struct { transfer: *Transfer, - fn init(transfer: *Transfer) ResponseHeaderWriter { + pub fn init(transfer: *Transfer) TransferAsRequestWriter { return .{ .transfer = transfer, }; } - pub fn jsonStringify(self: *const ResponseHeaderWriter, writer: anytype) !void { + pub fn jsonStringify(self: *const TransferAsRequestWriter, writer: anytype) !void { + const stream = writer.stream; + const transfer = self.transfer; + try writer.beginObject(); - var it = self.transfer.responseHeaderIterator(); - while (it.next()) |hdr| { - try writer.objectField(hdr.name); - try writer.write(hdr.value); + { + try writer.objectField("url"); + try writer.beginWriteRaw(); + try stream.writeByte('\"'); + try transfer.uri.writeToStream(.{ + .scheme = true, + .authentication = true, + .authority = true, + .path = true, + .query = true, + }, stream); + try stream.writeByte('\"'); + writer.endWriteRaw(); + } + + { + if (transfer.uri.fragment) |frag| { + try writer.objectField("urlFragment"); + try writer.beginWriteRaw(); + try stream.writeAll("\"#"); + try stream.writeAll(frag.percent_encoded); + try stream.writeByte('\"'); + writer.endWriteRaw(); + } + } + + { + try writer.objectField("method"); + try writer.write(@tagName(transfer.req.method)); + } + + { + try writer.objectField("hasPostData"); + try writer.write(transfer.req.body != null); + } + + { + try writer.objectField("headers"); + try writer.beginObject(); + var it = transfer.req.headers.iterator(); + while (it.next()) |hdr| { + try writer.objectField(hdr.name); + try writer.write(hdr.value); + } + try writer.endObject(); } try writer.endObject(); } }; +const TransferAsResponseWriter = struct { + transfer: *Transfer, + + fn init(transfer: *Transfer) TransferAsResponseWriter { + return .{ + .transfer = transfer, + }; + } + + pub fn jsonStringify(self: *const TransferAsResponseWriter, writer: anytype) !void { + const stream = writer.stream; + const transfer = self.transfer; + + try writer.beginObject(); + { + try writer.objectField("url"); + try writer.beginWriteRaw(); + try stream.writeByte('\"'); + try transfer.uri.writeToStream(.{ + .scheme = true, + .authentication = true, + .authority = true, + .path = true, + .query = true, + }, stream); + try stream.writeByte('\"'); + writer.endWriteRaw(); + } + + if (transfer.response_header) |*rh| { + // it should not be possible for this to be false, but I'm not + // feeling brave today. + const status = rh.status; + try writer.objectField("status"); + try writer.write(status); + + try writer.objectField("statusText"); + try writer.write(@as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown"); + } + + { + try writer.objectField("headers"); + try writer.beginObject(); + var it = transfer.responseHeaderIterator(); + while (it.next()) |hdr| { + try writer.objectField(hdr.name); + try writer.write(hdr.value); + } + try writer.endObject(); + } + try writer.endObject(); + } +}; + +const DocumentUrlWriter = struct { + uri: *std.Uri, + + fn init(uri: *std.Uri) DocumentUrlWriter { + return .{ + .uri = uri, + }; + } + + pub fn jsonStringify(self: *const DocumentUrlWriter, writer: anytype) !void { + const stream = writer.stream; + + try writer.beginWriteRaw(); + try stream.writeByte('\"'); + try self.uri.writeToStream(.{ + .scheme = true, + .authentication = true, + .authority = true, + .path = true, + .query = true, + }, stream); + try stream.writeByte('\"'); + writer.endWriteRaw(); + } +}; + const testing = @import("../testing.zig"); test "cdp.network setExtraHTTPHeaders" { var ctx = testing.context(); defer ctx.deinit(); - // _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" }); - try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } }); + _ = try ctx.loadBrowserContext(.{ .id = "NID-A", .session_id = "NESI-A" }); + // try ctx.processMessage(.{ .id = 10, .method = "Target.createTarget", .params = .{ .url = "about/blank" } }); try ctx.processMessage(.{ .id = 3, @@ -360,10 +429,7 @@ test "cdp.network setExtraHTTPHeaders" { }); const bc = ctx.cdp().browser_context.?; - try testing.expectEqual(bc.cdp.extra_headers.items.len, 1); - - try ctx.processMessage(.{ .id = 5, .method = "Target.attachToTarget", .params = .{ .targetId = bc.target_id.? } }); - try testing.expectEqual(bc.cdp.extra_headers.items.len, 0); + try testing.expectEqual(bc.extra_headers.items.len, 1); } test "cdp.Network: cookies" { diff --git a/src/cdp/domains/storage.zig b/src/cdp/domains/storage.zig index f26bdfef..53e8320c 100644 --- a/src/cdp/domains/storage.zig +++ b/src/cdp/domains/storage.zig @@ -129,7 +129,7 @@ pub const CdpCookie = struct { pub fn setCdpCookie(cookie_jar: *CookieJar, param: CdpCookie) !void { if (param.priority != .Medium or param.sameParty != null or param.sourceScheme != null or param.partitionKey != null) { - return error.NotYetImplementedParams; + return error.NotImplemented; } var arena = std.heap.ArenaAllocator.init(cookie_jar.allocator); diff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zig index 0a3ed2e0..fdf01be4 100644 --- a/src/cdp/domains/target.zig +++ b/src/cdp/domains/target.zig @@ -73,7 +73,9 @@ fn createBrowserContext(cmd: anytype) !void { originsWithUniversalNetworkAccess: ?[]const []const u8 = null, }); if (params) |p| { - if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) std.debug.print("Target.createBrowserContext: Not implemented param set\n", .{}); + if (p.disposeOnDetach or p.proxyBypassList != null or p.originsWithUniversalNetworkAccess != null) { + log.warn(.cdp, "not implemented", .{ .feature = "Target.createBrowserContext: Not implemented param set" }); + } } const bc = cmd.createBrowserContext() catch |err| switch (err) { @@ -407,8 +409,9 @@ fn doAttachtoTarget(cmd: anytype, target_id: []const u8) !void { std.debug.assert(bc.session_id == null); const session_id = cmd.cdp.session_id_gen.next(); - // extra_headers should not be kept on a new page or tab, currently we have only 1 page, we clear it just in case - bc.cdp.extra_headers.clearRetainingCapacity(); + // extra_headers should not be kept on a new page or tab, + // currently we have only 1 page, we clear it just in case + bc.extra_headers.clearRetainingCapacity(); try cmd.sendEvent("Target.attachedToTarget", AttachToTarget{ .sessionId = session_id, diff --git a/src/http/Client.zig b/src/http/Client.zig index deebd84d..2f5fafe5 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -19,10 +19,10 @@ const std = @import("std"); const log = @import("../log.zig"); const builtin = @import("builtin"); + const Http = @import("Http.zig"); -pub const Headers = Http.Headers; const Notification = @import("../notification.zig").Notification; -const storage = @import("../browser/storage/storage.zig"); +const CookieJar = @import("../browser/storage/storage.zig").CookieJar; const urlStitch = @import("../url.zig").stitch; @@ -34,7 +34,7 @@ const ArenaAllocator = std.heap.ArenaAllocator; const errorCheck = Http.errorCheck; const errorMCheck = Http.errorMCheck; -pub const Method = Http.Method; +const Method = Http.Method; // This is loosely tied to a browser Page. Loading all the , doing // XHR requests, and loading imports all happens through here. Sine the app @@ -309,6 +309,10 @@ fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) !void { try conn.setMethod(req.method); if (req.body) |b| { try conn.setBody(b); + } else if (req.method == .POST) { + // libcurl will crash if the method is POST but there's no body + // TODO: is there a setting for that..seems weird. + try conn.setBody(""); } var header_list = req.headers; @@ -496,7 +500,7 @@ pub const RequestCookie = struct { origin: *const std.Uri, jar: *@import("../browser/storage/cookie.zig").Jar, - pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Headers) !void { + pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Http.Headers) !void { const uri = std.Uri.parse(url) catch |err| { log.warn(.http, "invalid url", .{ .err = err, .url = url }); return error.InvalidUrl; @@ -519,19 +523,26 @@ pub const RequestCookie = struct { pub const Request = struct { method: Method, url: [:0]const u8, - headers: Headers, + headers: Http.Headers, body: ?[]const u8 = null, - cookie_jar: *storage.CookieJar, + cookie_jar: *CookieJar, + resource_type: ResourceType, // arbitrary data that can be associated with this request ctx: *anyopaque = undefined, start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null, - header_callback: ?*const fn (transfer: *Transfer, header: []const u8) anyerror!void = null, + header_callback: ?*const fn (transfer: *Transfer, header: Http.Header) anyerror!void = null, header_done_callback: *const fn (transfer: *Transfer) anyerror!void, data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void, done_callback: *const fn (ctx: *anyopaque) anyerror!void, error_callback: *const fn (ctx: *anyopaque, err: anyerror) void, + + const ResourceType = enum { + document, + xhr, + script, + }; }; pub const Transfer = struct { @@ -544,7 +555,7 @@ pub const Transfer = struct { _notified_fail: bool = false, // We'll store the response header here - response_header: ?Header = null, + response_header: ?ResponseHeader = null, _handle: ?*Handle = null, @@ -582,6 +593,22 @@ pub const Transfer = struct { self.req.url = url; } + pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const Http.Header) !void { + self.req.headers.deinit(); + + var buf: std.ArrayListUnmanaged(u8) = .empty; + var new_headers = try Http.Headers.init(); + for (headers) |hdr| { + // safe to re-use this buffer, because Headers.add because curl copies + // the value we pass into curl_slist_append. + defer buf.clearRetainingCapacity(); + try std.fmt.format(buf.writer(allocator), "{s}: {s}", .{ hdr.name, hdr.value }); + try buf.append(allocator, 0); // null terminated + try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]); + } + self.req.headers = new_headers; + } + pub fn abort(self: *Transfer) void { self.client.requestFailed(self, error.Abort); if (self._handle != null) { @@ -697,7 +724,7 @@ pub const Transfer = struct { if (getResponseHeader(easy, "content-type", 0)) |ct| { var hdr = &transfer.response_header.?; const value = ct.value; - const len = @min(value.len, hdr._content_type.len); + const len = @min(value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN); hdr._content_type_len = len; @memcpy(hdr._content_type[0..len], value[0..len]); } @@ -726,10 +753,12 @@ pub const Transfer = struct { } } else { if (transfer.req.header_callback) |cb| { - cb(transfer, header) catch |err| { - log.err(.http, "header_callback", .{ .err = err, .req = transfer }); - return 0; - }; + if (Http.Headers.parseHeader(header)) |hdr| { + cb(transfer, hdr) catch |err| { + log.err(.http, "header_callback", .{ .err = err, .req = transfer }); + return 0; + }; + } } } return buf_len; @@ -768,15 +797,64 @@ pub const Transfer = struct { try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_PRIVATE, &private)); return @alignCast(@ptrCast(private)); } + + pub fn fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void { + if (transfer._handle != null) { + // should never happen, should have been intercepted/paused, and then + // either continued, aborted and fulfilled once. + @branchHint(.unlikely); + return error.RequestInProgress; + } + + transfer._fulfill(status, headers, body) catch |err| { + transfer.req.error_callback(transfer.req.ctx, err); + return err; + }; + } + + fn _fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void { + const req = &transfer.req; + if (req.start_callback) |cb| { + try cb(transfer); + } + + if (req.header_callback) |cb| { + for (headers) |hdr| { + try cb(transfer, hdr); + } + } + + transfer.response_header = .{ + .status = status, + .url = req.url, + }; + for (headers) |hdr| { + if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { + const len = @min(hdr.value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN); + @memcpy(transfer.response_header.?._content_type[0..len], hdr.value[0..len]); + break; + } + } + + try req.header_done_callback(transfer); + + if (body) |b| { + try req.data_callback(transfer, b); + } + + try req.done_callback(req.ctx); + } }; -pub const Header = struct { +pub const ResponseHeader = struct { + const MAX_CONTENT_TYPE_LEN = 64; + status: u16, url: [*c]const u8, _content_type_len: usize = 0, - _content_type: [64]u8 = undefined, + _content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined, - pub fn contentType(self: *Header) ?[]u8 { + pub fn contentType(self: *ResponseHeader) ?[]u8 { if (self._content_type_len == 0) { return null; } @@ -788,7 +866,7 @@ const HeaderIterator = struct { easy: *c.CURL, prev: ?*c.curl_header = null, - pub fn next(self: *HeaderIterator) ?struct { name: []const u8, value: []const u8 } { + pub fn next(self: *HeaderIterator) ?Http.Header { const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null; self.prev = h; @@ -800,12 +878,12 @@ const HeaderIterator = struct { } }; -const ResponseHeader = struct { +const CurlHeaderValue = struct { value: []const u8, amount: usize, }; -fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?ResponseHeader { +fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?CurlHeaderValue { var hdr: [*c]c.curl_header = null; const result = c.curl_easy_header(easy, name, index, c.CURLH_HEADER, -1, &hdr); if (result == c.CURLE_OK) { diff --git a/src/http/Http.zig b/src/http/Http.zig index 162d7fea..59b5b621 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -22,14 +22,15 @@ pub const c = @cImport({ @cInclude("curl/curl.h"); }); -const Client = @import("Client.zig"); +pub const ENABLE_DEBUG = false; +pub const Client = @import("Client.zig"); +pub const Transfer = Client.Transfer; + const errors = @import("errors.zig"); const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; -pub const ENABLE_DEBUG = false; - // Client.zig does the bulk of the work and is loosely tied to a browser Page. // But we still need something above Client.zig for the "utility" http stuff // we need to do, like telemetry. The most important thing we want from this @@ -216,6 +217,11 @@ pub const Connection = struct { } }; +pub const Header = struct { + name: []const u8, + value: []const u8, +}; + pub const Headers = struct { headers: *c.curl_slist, cookies: ?[*c]const u8, @@ -237,25 +243,7 @@ pub const Headers = struct { self.headers = updated_headers; } - pub fn asHashMap(self: *const Headers, allocator: Allocator) !std.StringArrayHashMapUnmanaged([]const u8) { - var list: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - try list.ensureTotalCapacity(allocator, self.count()); - - var current: [*c]c.curl_slist = self.headers; - while (current) |node| { - const str = std.mem.span(@as([*:0]const u8, @ptrCast(node.*.data))); - const header = parseHeader(str) orelse return error.InvalidHeader; - list.putAssumeCapacity(header.name, header.value); - current = node.*.next; - } - // special case for cookies - if (self.cookies) |v| { - list.putAssumeCapacity("Cookie", std.mem.span(@as([*:0]const u8, @ptrCast(v)))); - } - return list; - } - - pub fn parseHeader(header_str: []const u8) ?std.http.Header { + pub fn parseHeader(header_str: []const u8) ?Header { const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null; const name = std.mem.trim(u8, header_str[0..colon_pos], " \t"); @@ -264,19 +252,28 @@ pub const Headers = struct { return .{ .name = name, .value = value }; } - pub fn count(self: *const Headers) usize { - var current: [*c]c.curl_slist = self.headers; - var num: usize = 0; - while (current) |node| { - num += 1; - current = node.*.next; - } - // special case for cookies - if (self.cookies != null) { - num += 1; - } - return num; + pub fn iterator(self: *Headers) Iterator { + return .{ + .header = self.headers, + .cookies = self.cookies, + }; } + + const Iterator = struct { + header: [*c]c.curl_slist, + cookies: ?[*c]const u8, + + pub fn next(self: *Iterator) ?Header { + const h = self.header orelse { + const cookies = self.cookies orelse return null; + self.cookies = null; + return .{ .name = "Cookie", .value = std.mem.span(@as([*:0]const u8, cookies)) }; + }; + + self.header = h.*.next; + return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data)))); + } + }; }; pub fn errorCheck(code: c.CURLcode) errors.Error!void {