diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 3c8ec621..c1f6b4fb 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -55,7 +55,7 @@ pub const HeaderIterator = http.HeaderIterator; pub const Client = @This(); // Count of active requests -active: usize, +active: usize = 0, // Count of intercepted requests. This is to help deal with intercepted requests. // The client doesn't track intercepted transfers. If a request is intercepted, @@ -64,7 +64,7 @@ active: usize, // no more network activity when, with interecepted requests, there might be more // in the future. (We really only need this to properly emit a 'networkIdle' and // 'networkAlmostIdle' Page.lifecycleEvent in CDP). -intercepted: usize, +intercepted: usize = 0, // Our curl multi handle. handles: http.Handles, @@ -82,12 +82,13 @@ performing: bool = false, next_request_id: u32 = 0, // When handles has no more available easys, requests get queued. -queue: TransferQueue, +queue: std.DoublyLinkedList = .{}, // The main app allocator allocator: Allocator, network: *Runtime, + // Queue of requests that depend on a robots.txt. // Allows us to fetch the robots.txt just once. pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empty, @@ -97,7 +98,7 @@ pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empt // request. These wil come and go with each request. transfer_pool: std.heap.MemoryPool(Transfer), -// The current proxy. CDP can change it, restoreOriginalProxy restores +// The current proxy. CDP can change it, changeProxy(null) restores // from config. http_proxy: ?[:0]const u8 = null, @@ -131,8 +132,6 @@ pub const CDPClient = struct { blocking_read_end: *const fn (*anyopaque) bool, }; -const TransferQueue = std.DoublyLinkedList; - pub fn init(allocator: Allocator, network: *Runtime) !*Client { var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator); errdefer transfer_pool.deinit(); @@ -146,17 +145,15 @@ pub fn init(allocator: Allocator, network: *Runtime) !*Client { const http_proxy = network.config.httpProxy(); client.* = .{ - .queue = .{}, - .active = 0, - .intercepted = 0, .handles = handles, - .allocator = allocator, .network = network, - .http_proxy = http_proxy, + .allocator = allocator, + .transfer_pool = transfer_pool, + .use_proxy = http_proxy != null, + .http_proxy = http_proxy, .tls_verify = network.config.tlsVerifyHost(), .obey_robots = network.config.obeyRobots(), - .transfer_pool = transfer_pool, }; return client; @@ -177,6 +174,34 @@ pub fn deinit(self: *Client) void { self.allocator.destroy(self); } +// Enable TLS verification on all connections. +pub fn setTlsVerify(self: *Client, verify: bool) !void { + // Remove inflight connections check on enable TLS b/c chromiumoxide calls + // the command during navigate and Curl seems to accept it... + + var it = self.in_use.first; + while (it) |node| : (it = node.next) { + const conn: *http.Connection = @fieldParentPtr("node", node); + try conn.setTlsVerify(verify, self.use_proxy); + } + self.tls_verify = verify; +} + +// Restrictive since it'll only work if there are no inflight requests. In some +// cases, the libcurl documentation is clear that changing settings while a +// connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY, +// but better to be safe than sorry. +// For now, this restriction is ok, since it's only called by CDP on +// createBrowserContext, at which point, if we do have an active connection, +// that's probably a bug (a previous abort failed?). But if we need to call this +// at any point in time, it could be worth digging into libcurl to see if this +// can be changed at any point in the easy's lifecycle. +pub fn changeProxy(self: *Client, proxy: ?[:0]const u8) !void { + try self.ensureNoActiveConnection(); + self.http_proxy = proxy orelse self.network.config.httpProxy(); + self.use_proxy = self.http_proxy != null; +} + pub fn newHeaders(self: *const Client) !http.Headers { return http.Headers.init(self.network.config.http_headers.user_agent_header); } @@ -193,8 +218,7 @@ pub fn abortFrame(self: *Client, frame_id: u32) void { // but abort can avoid the frame_id check at comptime. fn _abort(self: *Client, comptime abort_all: bool, frame_id: u32) void { { - var q = &self.in_use; - var n = q.first; + var n = self.in_use.first; while (n) |node| { n = node.next; const conn: *http.Connection = @fieldParentPtr("node", node); @@ -207,7 +231,6 @@ fn _abort(self: *Client, comptime abort_all: bool, frame_id: u32) void { if (comptime abort_all) { transfer.kill(); } else if (transfer.req.frame_id == frame_id) { - q.remove(node); transfer.kill(); } } @@ -252,9 +275,10 @@ pub fn tick(self: *Client, timeout_ms: u32) !PerformStatus { self.queue.prepend(queue_node); break; }; - const transfer: *Transfer = @fieldParentPtr("_node", queue_node); - try self.makeRequest(conn, transfer); + + try self.makeRequest(conn, @fieldParentPtr("_node", queue_node)); } + return self.perform(@intCast(timeout_ms)); } @@ -617,68 +641,7 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer { return transfer; } -fn requestFailed(transfer: *Transfer, err: anyerror, comptime execute_callback: bool) void { - if (transfer._notified_fail) { - // we can force a failed request within a callback, which will eventually - // result in this being called again in the more general loop. We do this - // because we can raise a more specific error inside a callback in some cases - return; - } - - transfer._notified_fail = true; - - transfer.req.notification.dispatch(.http_request_fail, &.{ - .transfer = transfer, - .err = err, - }); - - if (execute_callback) { - transfer.req.error_callback(transfer.ctx, err); - } else if (transfer.req.shutdown_callback) |cb| { - cb(transfer.ctx); - } -} - -// Restrictive since it'll only work if there are no inflight requests. In some -// cases, the libcurl documentation is clear that changing settings while a -// connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY, -// but better to be safe than sorry. -// For now, this restriction is ok, since it's only called by CDP on -// createBrowserContext, at which point, if we do have an active connection, -// that's probably a bug (a previous abort failed?). But if we need to call this -// at any point in time, it could be worth digging into libcurl to see if this -// can be changed at any point in the easy's lifecycle. -pub fn changeProxy(self: *Client, proxy: [:0]const u8) !void { - try self.ensureNoActiveConnection(); - self.http_proxy = proxy; - self.use_proxy = true; -} - -// Same restriction as changeProxy. Should be ok since this is only called on -// BrowserContext deinit. -pub fn restoreOriginalProxy(self: *Client) !void { - try self.ensureNoActiveConnection(); - - self.http_proxy = self.network.config.httpProxy(); - self.use_proxy = self.http_proxy != null; -} - -// Enable TLS verification on all connections. -pub fn setTlsVerify(self: *Client, verify: bool) !void { - // Remove inflight connections check on enable TLS b/c chromiumoxide calls - // the command during navigate and Curl seems to accept it... - - var it = self.in_use.first; - while (it) |node| : (it = node.next) { - const conn: *http.Connection = @fieldParentPtr("node", node); - try conn.setTlsVerify(verify, self.use_proxy); - } - self.tls_verify = verify; -} - fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyerror!void { - const req = &transfer.req; - { transfer._conn = conn; errdefer { @@ -687,45 +650,7 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyer self.releaseConn(conn); } - // Set callbacks and per-client settings on the pooled connection. - try conn.setCallbacks(Transfer.dataCallback); - try conn.setFollowLocation(false); - try conn.setProxy(self.http_proxy); - try conn.setTlsVerify(self.tls_verify, self.use_proxy); - - try conn.setURL(req.url); - try conn.setMethod(req.method); - if (req.body) |b| { - try conn.setBody(b); - } else { - try conn.setGetMode(); - } - - var header_list = req.headers; - try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts - try conn.setHeaders(&header_list); - - // If we have WebBotAuth, sign our request. - if (self.network.web_bot_auth) |*wba| { - const authority = URL.getHost(req.url); - try wba.signRequest(transfer.arena.allocator(), &header_list, authority); - } - - // Add cookies. - if (header_list.cookies) |cookies| { - try conn.setCookies(cookies); - } - - try conn.setPrivate(transfer); - - // add credentials - if (req.credentials) |creds| { - if (transfer._auth_challenge != null and transfer._auth_challenge.?.source == .proxy) { - try conn.setProxyCredentials(creds); - } else { - try conn.setCredentials(creds); - } - } + try transfer.configureConn(conn); } // As soon as this is called, our "perform" loop is responsible for @@ -741,15 +666,14 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyer self.releaseConn(conn); return err; }; + self.active += 1; - if (req.start_callback) |cb| { + if (transfer.req.start_callback) |cb| { cb(transfer) catch |err| { transfer.deinit(); return err; }; } - - self.active += 1; _ = try self.perform(0); } @@ -758,7 +682,7 @@ pub const PerformStatus = enum { normal, }; -fn perform(self: *Client, timeout_ms: c_int) !PerformStatus { +fn perform(self: *Client, timeout_ms: c_int) anyerror!PerformStatus { const running = blk: { self.performing = true; defer self.performing = false; @@ -827,7 +751,7 @@ fn processMessages(self: *Client) !bool { // to process it now. We can end the transfer, which will // release the easy handle back into the pool. The transfer // is still valid/alive (just has no handle). - self.endTransfer(transfer); + transfer.releaseConn(); if (!transfer.req.blocking) { // In the case of an async request, we can just "forget" // about this transfer until it gets updated asynchronously @@ -841,7 +765,7 @@ fn processMessages(self: *Client) !bool { // we've been asked to continue with the request // we can't process it here, since we're already inside // a process, so we need to queue it and wait for the - // next tick (this is why it was safe to endTransfer + // next tick (this is why it was safe to releaseConn // above, because even in the "blocking" path, we still // only process it on the next tick). self.queue.append(&transfer._node); @@ -853,31 +777,33 @@ fn processMessages(self: *Client) !bool { } } - // Handle redirects: extract data from conn before releasing it. + // Handle redirects: reuse the same connection to preserve TCP state. if (msg.err == null) { const status = try msg.conn.getResponseCode(); if (status >= 300 and status <= 399) { - transfer.handleRedirect(&msg.conn) catch |err| { - requestFailed(transfer, err, true); - self.endTransfer(transfer); + transfer.handleRedirect() catch |err| { + transfer.requestFailed(err, true); transfer.deinit(); continue; }; - self.endTransfer(transfer); + + const conn = transfer._conn.?; + + try self.handles.remove(conn); transfer.reset(); - try self.process(transfer); + try transfer.configureConn(conn); + try self.handles.add(conn); + + _ = try self.perform(0); + continue; } } - // release it ASAP so that it's available; some done_callbacks - // will load more resources. - self.endTransfer(transfer); - defer transfer.deinit(); if (msg.err) |err| { - requestFailed(transfer, err, true); + transfer.requestFailed(err, true); } else blk: { // make sure the transfer can't be immediately aborted from a callback // since we still need it here. @@ -889,18 +815,23 @@ fn processMessages(self: *Client) !bool { // callback now. const proceed = transfer.headerDoneCallback(&msg.conn) catch |err| { lp.log.err(.http, "header_done_callback2", .{ .err = err }); - requestFailed(transfer, err, true); + transfer.requestFailed(err, true); continue; }; if (!proceed) { - requestFailed(transfer, error.Abort, true); + transfer.requestFailed(error.Abort, true); break :blk; } } + + // release conn ASAP so that it's available; some done_callbacks + // will load more resources. + transfer.releaseConn(); + transfer.req.done_callback(transfer.ctx) catch |err| { // transfer isn't valid at this point, don't use it. lp.log.err(.http, "done_callback", .{ .err = err }); - requestFailed(transfer, err, true); + transfer.requestFailed(err, true); continue; }; @@ -913,15 +844,9 @@ fn processMessages(self: *Client) !bool { return processed; } -fn endTransfer(self: *Client, transfer: *Transfer) void { - const conn = transfer._conn.?; - self.removeConn(conn); - transfer._conn = null; - self.active -= 1; -} - fn removeConn(self: *Client, conn: *http.Connection) void { self.in_use.remove(&conn.node); + self.active -= 1; if (self.handles.remove(conn)) { self.releaseConn(conn); } else |_| { @@ -1012,8 +937,6 @@ pub const Request = struct { }; }; -const AuthChallenge = http.AuthChallenge; - pub const Transfer = struct { arena: ArenaAllocator, id: u32 = 0, @@ -1039,7 +962,7 @@ pub const Transfer = struct { _conn: ?*http.Connection = null, - _auth_challenge: ?AuthChallenge = null, + _auth_challenge: ?http.AuthChallenge = null, // number of times the transfer has been tried. // incremented by reset func. @@ -1059,6 +982,116 @@ pub const Transfer = struct { fulfilled, }; + fn releaseConn(self: *Transfer) void { + if (self._conn) |conn| { + self.client.removeConn(conn); + self._conn = null; + } + } + + fn deinit(self: *Transfer) void { + if (self._conn) |conn| { + self.client.removeConn(conn); + self._conn = null; + } + + self.req.headers.deinit(); + self.arena.deinit(); + self.client.transfer_pool.destroy(self); + } + + pub fn abort(self: *Transfer, err: anyerror) void { + self.requestFailed(err, true); + + if (self._performing or self.client.performing) { + // We're currently in a curl_multi_perform. We cannot call + // curl_multi_remove_handle from a curl callback. Instead, we flag + // this transfer and our callbacks will check for this flag. + self.aborted = true; + return; + } + + self.deinit(); + } + + pub fn terminate(self: *Transfer) void { + self.requestFailed(error.Shutdown, false); + self.deinit(); + } + + // internal, when the page is shutting down. Doesn't have the same ceremony + // as abort (doesn't send a notification, doesn't invoke an error callback) + fn kill(self: *Transfer) void { + if (self.req.shutdown_callback) |cb| { + cb(self.ctx); + } + self.deinit(); + } + + // We can force a failed request within a callback, which will eventually + // result in this being called again in the more general loop. We do this + // because we can raise a more specific error inside a callback in some cases. + fn requestFailed(self: *Transfer, err: anyerror, comptime execute_callback: bool) void { + if (self._notified_fail) return; + self._notified_fail = true; + + self.req.notification.dispatch(.http_request_fail, &.{ + .transfer = self, + .err = err, + }); + + if (execute_callback) { + self.req.error_callback(self.ctx, err); + } else if (self.req.shutdown_callback) |cb| { + cb(self.ctx); + } + } + + fn configureConn(self: *Transfer, conn: *http.Connection) anyerror!void { + const client = self.client; + const req = &self.req; + + // Set callbacks and per-client settings on the pooled connection. + try conn.setCallbacks(Transfer.dataCallback); + try conn.setFollowLocation(false); + try conn.setProxy(client.http_proxy); + try conn.setTlsVerify(client.tls_verify, client.use_proxy); + + try conn.setURL(req.url); + try conn.setMethod(req.method); + if (req.body) |b| { + try conn.setBody(b); + } else { + try conn.setGetMode(); + } + + var header_list = req.headers; + try conn.secretHeaders(&header_list, &client.network.config.http_headers); + try conn.setHeaders(&header_list); + + // If we have WebBotAuth, sign our request. + if (client.network.web_bot_auth) |*wba| { + const authority = URL.getHost(req.url); + try wba.signRequest(self.arena.allocator(), &header_list, authority); + } + + // Add cookies. + if (header_list.cookies) |cookies| { + try conn.setCookies(cookies); + } + + try conn.setPrivate(self); + + // add credentials + if (req.credentials) |creds| { + if (self._auth_challenge != null and self._auth_challenge.?.source == .proxy) { + try conn.setProxyCredentials(creds); + } else { + try conn.setCredentials(creds); + } + } + } + pub fn reset(self: *Transfer) void { self._auth_challenge = null; self._notified_fail = false; @@ -1067,15 +1100,6 @@ pub const Transfer = struct { self._tries += 1; } - fn deinit(self: *Transfer) void { - self.req.headers.deinit(); - if (self._conn) |conn| { - self.client.removeConn(conn); - } - self.arena.deinit(); - self.client.transfer_pool.destroy(self); - } - fn buildResponseHeader(self: *Transfer, conn: *const http.Connection) !void { if (comptime IS_DEBUG) { std.debug.assert(self.response_header == null); @@ -1116,8 +1140,9 @@ pub const Transfer = struct { self.req.url = url; } - fn handleRedirect(transfer: *Transfer, conn: *const http.Connection) !void { + fn handleRedirect(transfer: *Transfer) !void { const req = &transfer.req; + const conn = transfer._conn.?; const arena = transfer.arena.allocator(); transfer._redirect_count += 1; @@ -1125,15 +1150,18 @@ pub const Transfer = struct { return error.TooManyRedirects; } + lp.log.warn(.bug, "Redirecting...", .{}); + // retrieve cookies from the redirect's response. if (req.cookie_jar) |jar| { var i: usize = 0; - while (true) { - const ct = conn.getResponseHeader("set-cookie", i); - if (ct == null) break; - try jar.populateFromResponse(transfer.url, ct.?.value); - i += 1; - if (i >= ct.?.amount) break; + while (conn.getResponseHeader("set-cookie", i)) |ct| : (i += 1) { + lp.log.warn(.bug, "set-cookie", .{ i, ct.value }); + try jar.populateFromResponse(transfer.url, ct.value); + + if (i >= ct.amount) { + break; + } } } @@ -1168,6 +1196,7 @@ pub const Transfer = struct { } else { req.headers.cookies = null; } + lp.log.warn(.bug, "cookie", .{cookies.items[0..]}); } } @@ -1179,9 +1208,9 @@ pub const Transfer = struct { } if (conn.getResponseHeader("WWW-Authenticate", 0)) |hdr| { - transfer._auth_challenge = AuthChallenge.parse(status, .server, hdr.value) catch null; + transfer._auth_challenge = http.AuthChallenge.parse(status, .server, hdr.value) catch null; } else if (conn.getResponseHeader("Proxy-Authenticate", 0)) |hdr| { - transfer._auth_challenge = AuthChallenge.parse(status, .proxy, hdr.value) catch null; + transfer._auth_challenge = http.AuthChallenge.parse(status, .proxy, hdr.value) catch null; } else { transfer._auth_challenge = .{ .status = status, .source = null, .scheme = null, .realm = null }; } @@ -1207,48 +1236,8 @@ pub const Transfer = struct { self.req.headers = new_headers; } - pub fn abort(self: *Transfer, err: anyerror) void { - requestFailed(self, err, true); - - const client = self.client; - if (self._performing or client.performing) { - // We're currently in a curl_multi_perform. We cannot call endTransfer - // as that calls curl_multi_remove_handle, and you can't do that - // from a curl callback. Instead, we flag this transfer and all of - // our callbacks will check for this flag and abort the transfer for - // us - self.aborted = true; - return; - } - - if (self._conn != null) { - client.endTransfer(self); - } - self.deinit(); - } - - pub fn terminate(self: *Transfer) void { - requestFailed(self, error.Shutdown, false); - if (self._conn != null) { - self.client.endTransfer(self); - } - self.deinit(); - } - - // internal, when the page is shutting down. Doesn't have the same ceremony - // as abort (doesn't send a notification, doesn't invoke an error callback) - fn kill(self: *Transfer) void { - if (self._conn != null) { - self.client.endTransfer(self); - } - if (self.req.shutdown_callback) |cb| { - cb(self.ctx); - } - self.deinit(); - } - // abortAuthChallenge is called when an auth challenge interception is - // abort. We don't call self.client.endTransfer here b/c it has been done + // abort. We don't call self.releaseConn here b/c it has been done // before interception process. pub fn abortAuthChallenge(self: *Transfer) void { if (comptime IS_DEBUG) { @@ -1320,8 +1309,8 @@ pub const Transfer = struct { std.debug.assert(chunk_count == 1); } - const conn: http.Connection = .{ .easy = @ptrCast(@alignCast(data)) }; - var transfer = fromConnection(&conn) catch |err| { + const conn: *http.Connection = @ptrCast(@alignCast(data)); + var transfer = fromConnection(conn) catch |err| { lp.log.err(.http, "get private info", .{ .err = err, .source = "body callback" }); return http.writefunc_error; }; @@ -1333,7 +1322,7 @@ pub const Transfer = struct { } if (!transfer._header_done_called) { - const proceed = transfer.headerDoneCallback(&conn) catch |err| { + const proceed = transfer.headerDoneCallback(conn) catch |err| { lp.log.err(.http, "header_done_callback", .{ .err = err, .req = transfer }); return http.writefunc_error; }; @@ -1346,7 +1335,7 @@ pub const Transfer = struct { transfer.bytes_received += chunk_len; if (transfer.max_response_size) |max_size| { if (transfer.bytes_received > max_size) { - requestFailed(transfer, error.ResponseTooLarge, true); + transfer.requestFailed(error.ResponseTooLarge, true); return http.writefunc_error; } } @@ -1382,7 +1371,7 @@ pub const Transfer = struct { return .{ .list = .{ .list = self.response_header.?._injected_headers } }; } - pub fn fromConnection(conn: *const http.Connection) !*Transfer { + fn fromConnection(conn: *const http.Connection) !*Transfer { const private = try conn.getPrivate(); return @ptrCast(@alignCast(private)); } diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 08cc85ff..fc38acf0 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -739,7 +739,7 @@ pub const Script = struct { .b5 = transfer._notified_fail, .b7 = @intFromEnum(transfer._intercept_state), .b8 = transfer._auth_challenge != null, - .b9 = if (transfer._conn) |c| @intFromPtr(c.easy) else 0, + .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0, }); self.header_callback_called = true; self.debug_transfer_id = transfer.id; @@ -749,7 +749,7 @@ pub const Script = struct { self.debug_transfer_notified_fail = transfer._notified_fail; self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state); self.debug_transfer_auth_challenge = transfer._auth_challenge != null; - self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c.easy) else 0; + self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0; } lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity }); diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 58ed11b9..0f33cc65 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -475,8 +475,8 @@ pub fn BrowserContext(comptime CDP_T: type) type { if (self.http_proxy_changed) { // has to be called after browser.closeSession, since it won't // work if there are active connections. - browser.http_client.restoreOriginalProxy() catch |err| { - log.warn(.http, "restoreOriginalProxy", .{ .err = err }); + browser.http_client.changeProxy(null) catch |err| { + log.warn(.http, "changeProxy", .{ .err = err }); }; } self.intercept_state.deinit(); diff --git a/src/network/Runtime.zig b/src/network/Runtime.zig index 72aebe81..fb3bd9f6 100644 --- a/src/network/Runtime.zig +++ b/src/network/Runtime.zig @@ -461,7 +461,7 @@ fn drainQueue(self: *Runtime) void { self.releaseConnection(conn); continue; }; - libcurl.curl_multi_add_handle(multi, conn.easy) catch |err| { + libcurl.curl_multi_add_handle(multi, conn._easy) catch |err| { lp.log.err(.app, "curl multi add", .{ .err = err }); self.releaseConnection(conn); }; @@ -565,7 +565,7 @@ pub fn getConnection(self: *Runtime) ?*net_http.Connection { } pub fn releaseConnection(self: *Runtime, conn: *net_http.Connection) void { - conn.reset() catch |err| { + conn.reset(self.config, self.ca_blob) catch |err| { lp.assert(false, "couldn't reset curl easy", .{ .err = err }); }; diff --git a/src/network/http.zig b/src/network/http.zig index 48c3d619..d4338104 100644 --- a/src/network/http.zig +++ b/src/network/http.zig @@ -142,7 +142,7 @@ pub const HeaderIterator = union(enum) { prev: ?*libcurl.CurlHeader = null, pub fn next(self: *CurlHeaderIterator) ?Header { - const h = libcurl.curl_easy_nextheader(self.conn.easy, .header, -1, self.prev) orelse return null; + const h = libcurl.curl_easy_nextheader(self.conn._easy, .header, -1, self.prev) orelse return null; self.prev = h; const header = h.*; @@ -227,77 +227,28 @@ pub const ResponseHead = struct { }; pub const Connection = struct { - easy: *libcurl.Curl, + _easy: *libcurl.Curl, node: std.DoublyLinkedList.Node = .{}, pub fn init( - ca_blob_: ?libcurl.CurlBlob, + ca_blob: ?libcurl.CurlBlob, config: *const Config, ) !Connection { const easy = libcurl.curl_easy_init() orelse return error.FailedToInitializeEasy; - errdefer libcurl.curl_easy_cleanup(easy); - // timeouts - try libcurl.curl_easy_setopt(easy, .timeout_ms, config.httpTimeout()); - try libcurl.curl_easy_setopt(easy, .connect_timeout_ms, config.httpConnectTimeout()); + const self = Connection{ ._easy = easy }; + errdefer self.deinit(); - // redirect behavior - try libcurl.curl_easy_setopt(easy, .max_redirs, config.httpMaxRedirects()); - try libcurl.curl_easy_setopt(easy, .follow_location, 2); - try libcurl.curl_easy_setopt(easy, .redir_protocols_str, "HTTP,HTTPS"); // remove FTP and FTPS from the default - - // proxy - const http_proxy = config.httpProxy(); - if (http_proxy) |proxy| { - try libcurl.curl_easy_setopt(easy, .proxy, proxy.ptr); - } - - // tls - if (ca_blob_) |ca_blob| { - try libcurl.curl_easy_setopt(easy, .ca_info_blob, ca_blob); - if (http_proxy != null) { - try libcurl.curl_easy_setopt(easy, .proxy_ca_info_blob, ca_blob); - } - } else { - assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{}); - - try libcurl.curl_easy_setopt(easy, .ssl_verify_host, false); - try libcurl.curl_easy_setopt(easy, .ssl_verify_peer, false); - - if (http_proxy != null) { - try libcurl.curl_easy_setopt(easy, .proxy_ssl_verify_host, false); - try libcurl.curl_easy_setopt(easy, .proxy_ssl_verify_peer, false); - } - } - - // compression, don't remove this. CloudFront will send gzip content - // even if we don't support it, and then it won't be decompressed. - // empty string means: use whatever's available - try libcurl.curl_easy_setopt(easy, .accept_encoding, ""); - - // debug - if (comptime ENABLE_DEBUG) { - try libcurl.curl_easy_setopt(easy, .verbose, true); - - // Sometimes the default debug output hides some useful data. You can - // uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to - // get more control over the data (specifically, the `CURLINFO_TEXT` - // can include useful data). - - // try libcurl.curl_easy_setopt(easy, .debug_function, debugCallback); - } - - return .{ - .easy = easy, - }; + try self.reset(config, ca_blob); + return self; } pub fn deinit(self: *const Connection) void { - libcurl.curl_easy_cleanup(self.easy); + libcurl.curl_easy_cleanup(self._easy); } pub fn setURL(self: *const Connection, url: [:0]const u8) !void { - try libcurl.curl_easy_setopt(self.easy, .url, url.ptr); + try libcurl.curl_easy_setopt(self._easy, .url, url.ptr); } // a libcurl request has 2 methods. The first is the method that @@ -320,7 +271,7 @@ pub const Connection = struct { // can infer that based on the presence of the body, but we also reset it // to be safe); pub fn setMethod(self: *const Connection, method: Method) !void { - const easy = self.easy; + const easy = self._easy; const m: [:0]const u8 = switch (method) { .GET => "GET", .POST => "POST", @@ -335,50 +286,97 @@ pub const Connection = struct { } pub fn setBody(self: *const Connection, body: []const u8) !void { - const easy = self.easy; + const easy = self._easy; try libcurl.curl_easy_setopt(easy, .post, true); try libcurl.curl_easy_setopt(easy, .post_field_size, body.len); try libcurl.curl_easy_setopt(easy, .copy_post_fields, body.ptr); } pub fn setGetMode(self: *const Connection) !void { - try libcurl.curl_easy_setopt(self.easy, .http_get, true); + try libcurl.curl_easy_setopt(self._easy, .http_get, true); } pub fn setHeaders(self: *const Connection, headers: *Headers) !void { - try libcurl.curl_easy_setopt(self.easy, .http_header, headers.headers); + try libcurl.curl_easy_setopt(self._easy, .http_header, headers.headers); } pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void { - try libcurl.curl_easy_setopt(self.easy, .cookie, cookies); + try libcurl.curl_easy_setopt(self._easy, .cookie, cookies); } pub fn setPrivate(self: *const Connection, ptr: *anyopaque) !void { - try libcurl.curl_easy_setopt(self.easy, .private, ptr); + try libcurl.curl_easy_setopt(self._easy, .private, ptr); } pub fn setProxyCredentials(self: *const Connection, creds: [:0]const u8) !void { - try libcurl.curl_easy_setopt(self.easy, .proxy_user_pwd, creds.ptr); + try libcurl.curl_easy_setopt(self._easy, .proxy_user_pwd, creds.ptr); } pub fn setCredentials(self: *const Connection, creds: [:0]const u8) !void { - try libcurl.curl_easy_setopt(self.easy, .user_pwd, creds.ptr); + try libcurl.curl_easy_setopt(self._easy, .user_pwd, creds.ptr); } pub fn setCallbacks( - self: *const Connection, + self: *Connection, comptime data_cb: libcurl.CurlWriteFunction, ) !void { - try libcurl.curl_easy_setopt(self.easy, .write_data, self.easy); - try libcurl.curl_easy_setopt(self.easy, .write_function, data_cb); + try libcurl.curl_easy_setopt(self._easy, .write_data, self); + try libcurl.curl_easy_setopt(self._easy, .write_function, data_cb); } - pub fn reset(self: *const Connection) !void { - try libcurl.curl_easy_setopt(self.easy, .proxy, null); - try libcurl.curl_easy_setopt(self.easy, .http_header, null); + pub fn reset( + self: *const Connection, + config: *const Config, + ca_blob: ?libcurl.CurlBlob, + ) !void { + libcurl.curl_easy_reset(self._easy); - try libcurl.curl_easy_setopt(self.easy, .write_data, null); - try libcurl.curl_easy_setopt(self.easy, .write_function, discardBody); + // timeouts + try libcurl.curl_easy_setopt(self._easy, .timeout_ms, config.httpTimeout()); + try libcurl.curl_easy_setopt(self._easy, .connect_timeout_ms, config.httpConnectTimeout()); + + // compression, don't remove this. CloudFront will send gzip content + // even if we don't support it, and then it won't be decompressed. + // empty string means: use whatever's available + try libcurl.curl_easy_setopt(self._easy, .accept_encoding, ""); + + // proxy + const http_proxy = config.httpProxy(); + if (http_proxy) |proxy| { + try libcurl.curl_easy_setopt(self._easy, .proxy, proxy.ptr); + } else { + try libcurl.curl_easy_setopt(self._easy, .proxy, null); + } + + // tls + if (ca_blob) |ca| { + try libcurl.curl_easy_setopt(self._easy, .ca_info_blob, ca); + if (http_proxy != null) { + try libcurl.curl_easy_setopt(self._easy, .proxy_ca_info_blob, ca); + } + } else { + assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{}); + + try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false); + try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false); + + if (http_proxy != null) { + try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, false); + try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, false); + } + } + + // debug + if (comptime ENABLE_DEBUG) { + try libcurl.curl_easy_setopt(self._easy, .verbose, true); + + // Sometimes the default debug output hides some useful data. You can + // uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to + // get more control over the data (specifically, the `CURLINFO_TEXT` + // can include useful data). + + // try libcurl.curl_easy_setopt(easy, .debug_function, debugCallback); + } } fn discardBody(_: [*]const u8, count: usize, len: usize, _: ?*anyopaque) usize { @@ -386,31 +384,31 @@ pub const Connection = struct { } pub fn setProxy(self: *const Connection, proxy: ?[:0]const u8) !void { - try libcurl.curl_easy_setopt(self.easy, .proxy, if (proxy) |p| p.ptr else null); + try libcurl.curl_easy_setopt(self._easy, .proxy, if (proxy) |p| p.ptr else null); } pub fn setFollowLocation(self: *const Connection, follow: bool) !void { - try libcurl.curl_easy_setopt(self.easy, .follow_location, @as(c_long, if (follow) 2 else 0)); + try libcurl.curl_easy_setopt(self._easy, .follow_location, @as(c_long, if (follow) 2 else 0)); } pub fn setTlsVerify(self: *const Connection, verify: bool, use_proxy: bool) !void { - try libcurl.curl_easy_setopt(self.easy, .ssl_verify_host, verify); - try libcurl.curl_easy_setopt(self.easy, .ssl_verify_peer, verify); + try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, verify); + try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, verify); if (use_proxy) { - try libcurl.curl_easy_setopt(self.easy, .proxy_ssl_verify_host, verify); - try libcurl.curl_easy_setopt(self.easy, .proxy_ssl_verify_peer, verify); + try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, verify); + try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, verify); } } pub fn getEffectiveUrl(self: *const Connection) ![*c]const u8 { var url: [*c]u8 = undefined; - try libcurl.curl_easy_getinfo(self.easy, .effective_url, &url); + try libcurl.curl_easy_getinfo(self._easy, .effective_url, &url); return url; } pub fn getResponseCode(self: *const Connection) !u16 { var status: c_long = undefined; - try libcurl.curl_easy_getinfo(self.easy, .response_code, &status); + try libcurl.curl_easy_getinfo(self._easy, .response_code, &status); if (status < 0 or status > std.math.maxInt(u16)) { return 0; } @@ -419,13 +417,13 @@ pub const Connection = struct { pub fn getRedirectCount(self: *const Connection) !u32 { var count: c_long = undefined; - try libcurl.curl_easy_getinfo(self.easy, .redirect_count, &count); + try libcurl.curl_easy_getinfo(self._easy, .redirect_count, &count); return @intCast(count); } pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue { var hdr: ?*libcurl.CurlHeader = null; - libcurl.curl_easy_header(self.easy, name, index, .header, -1, &hdr) catch |err| { + libcurl.curl_easy_header(self._easy, name, index, .header, -1, &hdr) catch |err| { // ErrorHeader includes OutOfMemory — rare but real errors from curl internals. // Logged and returned as null since callers don't expect errors. log.err(.http, "get response header", .{ @@ -443,7 +441,7 @@ pub const Connection = struct { pub fn getPrivate(self: *const Connection) !*anyopaque { var private: *anyopaque = undefined; - try libcurl.curl_easy_getinfo(self.easy, .private, &private); + try libcurl.curl_easy_getinfo(self._easy, .private, &private); return private; } @@ -465,7 +463,7 @@ pub const Connection = struct { try self.setCookies(cookies); } - try libcurl.curl_easy_perform(self.easy); + try libcurl.curl_easy_perform(self._easy); return self.getResponseCode(); } }; @@ -487,11 +485,11 @@ pub const Handles = struct { } pub fn add(self: *Handles, conn: *const Connection) !void { - try libcurl.curl_multi_add_handle(self.multi, conn.easy); + try libcurl.curl_multi_add_handle(self.multi, conn._easy); } pub fn remove(self: *Handles, conn: *const Connection) !void { - try libcurl.curl_multi_remove_handle(self.multi, conn.easy); + try libcurl.curl_multi_remove_handle(self.multi, conn._easy); } pub fn perform(self: *Handles) !c_int { @@ -514,7 +512,7 @@ pub const Handles = struct { const msg = libcurl.curl_multi_info_read(self.multi, &messages_count) orelse return null; return switch (msg.data) { .done => |err| .{ - .conn = .{ .easy = msg.easy_handle }, + .conn = .{ ._easy = msg.easy_handle }, .err = err, }, else => unreachable, diff --git a/src/sys/libcurl.zig b/src/sys/libcurl.zig index 1c6f9f13..b37e5142 100644 --- a/src/sys/libcurl.zig +++ b/src/sys/libcurl.zig @@ -516,6 +516,10 @@ pub fn curl_easy_cleanup(easy: *Curl) void { c.curl_easy_cleanup(easy); } +pub fn curl_easy_reset(easy: *Curl) void { + c.curl_easy_reset(easy); +} + pub fn curl_easy_perform(easy: *Curl) Error!void { try errorCheck(c.curl_easy_perform(easy)); }