public/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-03-22 04:34:44 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Fix cleanup connections in HttpClient

Browse Source
This commit is contained in:
Nikolay Govorov
2026-03-18 19:10:11 +00:00
parent f1a96bab5b
commit 6e38aa9414
6 changed files with 297 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

411
src/browser/HttpClient.zig
View File

@@ -55,7 +55,7 @@ pub const HeaderIterator = http.HeaderIterator;
pub const Client = @This(); pub const Client = @This();
// Count of active requests // Count of active requests
active: usize, active: usize = 0,
// Count of intercepted requests. This is to help deal with intercepted requests. // Count of intercepted requests. This is to help deal with intercepted requests.
// The client doesn't track intercepted transfers. If a request is intercepted, // The client doesn't track intercepted transfers. If a request is intercepted,
@@ -64,7 +64,7 @@ active: usize,
// no more network activity when, with interecepted requests, there might be more // no more network activity when, with interecepted requests, there might be more
// in the future. (We really only need this to properly emit a 'networkIdle' and // in the future. (We really only need this to properly emit a 'networkIdle' and
// 'networkAlmostIdle' Page.lifecycleEvent in CDP). // 'networkAlmostIdle' Page.lifecycleEvent in CDP).
intercepted: usize, intercepted: usize = 0,
// Our curl multi handle. // Our curl multi handle.
handles: http.Handles, handles: http.Handles,
@@ -82,12 +82,13 @@ performing: bool = false,
next_request_id: u32 = 0, next_request_id: u32 = 0,
// When handles has no more available easys, requests get queued. // When handles has no more available easys, requests get queued.
queue: TransferQueue, queue: std.DoublyLinkedList = .{},
// The main app allocator // The main app allocator
allocator: Allocator, allocator: Allocator,
network: *Runtime, network: *Runtime,
// Queue of requests that depend on a robots.txt. // Queue of requests that depend on a robots.txt.
// Allows us to fetch the robots.txt just once. // Allows us to fetch the robots.txt just once.
pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empty, pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empty,
@@ -97,7 +98,7 @@ pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empt
// request. These wil come and go with each request. // request. These wil come and go with each request.
transfer_pool: std.heap.MemoryPool(Transfer), transfer_pool: std.heap.MemoryPool(Transfer),
// The current proxy. CDP can change it, restoreOriginalProxy restores // The current proxy. CDP can change it, changeProxy(null) restores
// from config. // from config.
http_proxy: ?[:0]const u8 = null, http_proxy: ?[:0]const u8 = null,
@@ -131,8 +132,6 @@ pub const CDPClient = struct {
blocking_read_end: *const fn (*anyopaque) bool, blocking_read_end: *const fn (*anyopaque) bool,
}; };
const TransferQueue = std.DoublyLinkedList;
pub fn init(allocator: Allocator, network: *Runtime) !*Client { pub fn init(allocator: Allocator, network: *Runtime) !*Client {
var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator); var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
errdefer transfer_pool.deinit(); errdefer transfer_pool.deinit();
@@ -146,17 +145,15 @@ pub fn init(allocator: Allocator, network: *Runtime) !*Client {
const http_proxy = network.config.httpProxy(); const http_proxy = network.config.httpProxy();
client.* = .{ client.* = .{
.queue = .{},
.active = 0,
.intercepted = 0,
.handles = handles, .handles = handles,
.allocator = allocator,
.network = network, .network = network,
.http_proxy = http_proxy, .allocator = allocator,
.transfer_pool = transfer_pool,
.use_proxy = http_proxy != null, .use_proxy = http_proxy != null,
.http_proxy = http_proxy,
.tls_verify = network.config.tlsVerifyHost(), .tls_verify = network.config.tlsVerifyHost(),
.obey_robots = network.config.obeyRobots(), .obey_robots = network.config.obeyRobots(),
.transfer_pool = transfer_pool,
}; };
return client; return client;
@@ -177,6 +174,34 @@ pub fn deinit(self: *Client) void {
self.allocator.destroy(self); self.allocator.destroy(self);
} }
// Enable TLS verification on all connections.
pub fn setTlsVerify(self: *Client, verify: bool) !void {
// Remove inflight connections check on enable TLS b/c chromiumoxide calls
// the command during navigate and Curl seems to accept it...
var it = self.in_use.first;
while (it) |node| : (it = node.next) {
const conn: *http.Connection = @fieldParentPtr("node", node);
try conn.setTlsVerify(verify, self.use_proxy);
}
self.tls_verify = verify;
}
// Restrictive since it'll only work if there are no inflight requests. In some
// cases, the libcurl documentation is clear that changing settings while a
// connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY,
// but better to be safe than sorry.
// For now, this restriction is ok, since it's only called by CDP on
// createBrowserContext, at which point, if we do have an active connection,
// that's probably a bug (a previous abort failed?). But if we need to call this
// at any point in time, it could be worth digging into libcurl to see if this
// can be changed at any point in the easy's lifecycle.
pub fn changeProxy(self: *Client, proxy: ?[:0]const u8) !void {
try self.ensureNoActiveConnection();
self.http_proxy = proxy orelse self.network.config.httpProxy();
self.use_proxy = self.http_proxy != null;
}
pub fn newHeaders(self: *const Client) !http.Headers { pub fn newHeaders(self: *const Client) !http.Headers {
return http.Headers.init(self.network.config.http_headers.user_agent_header); return http.Headers.init(self.network.config.http_headers.user_agent_header);
} }
@@ -193,8 +218,7 @@ pub fn abortFrame(self: *Client, frame_id: u32) void {
// but abort can avoid the frame_id check at comptime. // but abort can avoid the frame_id check at comptime.
fn _abort(self: *Client, comptime abort_all: bool, frame_id: u32) void { fn _abort(self: *Client, comptime abort_all: bool, frame_id: u32) void {
{ {
var q = &self.in_use; var n = self.in_use.first;
var n = q.first;
while (n) |node| { while (n) |node| {
n = node.next; n = node.next;
const conn: *http.Connection = @fieldParentPtr("node", node); const conn: *http.Connection = @fieldParentPtr("node", node);
@@ -207,7 +231,6 @@ fn _abort(self: *Client, comptime abort_all: bool, frame_id: u32) void {
if (comptime abort_all) { if (comptime abort_all) {
transfer.kill(); transfer.kill();
} else if (transfer.req.frame_id == frame_id) { } else if (transfer.req.frame_id == frame_id) {
q.remove(node);
transfer.kill(); transfer.kill();
} }
} }
@@ -252,9 +275,10 @@ pub fn tick(self: *Client, timeout_ms: u32) !PerformStatus {
self.queue.prepend(queue_node); self.queue.prepend(queue_node);
break; break;
}; };
const transfer: *Transfer = @fieldParentPtr("_node", queue_node);
try self.makeRequest(conn, transfer); try self.makeRequest(conn, @fieldParentPtr("_node", queue_node));
} }
return self.perform(@intCast(timeout_ms)); return self.perform(@intCast(timeout_ms));
} }
@@ -617,68 +641,7 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer {
return transfer; return transfer;
} }
fn requestFailed(transfer: *Transfer, err: anyerror, comptime execute_callback: bool) void {
if (transfer._notified_fail) {
// we can force a failed request within a callback, which will eventually
// result in this being called again in the more general loop. We do this
// because we can raise a more specific error inside a callback in some cases
return;
}
transfer._notified_fail = true;
transfer.req.notification.dispatch(.http_request_fail, &.{
.transfer = transfer,
.err = err,
});
if (execute_callback) {
transfer.req.error_callback(transfer.ctx, err);
} else if (transfer.req.shutdown_callback) |cb| {
cb(transfer.ctx);
}
}
// Restrictive since it'll only work if there are no inflight requests. In some
// cases, the libcurl documentation is clear that changing settings while a
// connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY,
// but better to be safe than sorry.
// For now, this restriction is ok, since it's only called by CDP on
// createBrowserContext, at which point, if we do have an active connection,
// that's probably a bug (a previous abort failed?). But if we need to call this
// at any point in time, it could be worth digging into libcurl to see if this
// can be changed at any point in the easy's lifecycle.
pub fn changeProxy(self: *Client, proxy: [:0]const u8) !void {
try self.ensureNoActiveConnection();
self.http_proxy = proxy;
self.use_proxy = true;
}
// Same restriction as changeProxy. Should be ok since this is only called on
// BrowserContext deinit.
pub fn restoreOriginalProxy(self: *Client) !void {
try self.ensureNoActiveConnection();
self.http_proxy = self.network.config.httpProxy();
self.use_proxy = self.http_proxy != null;
}
// Enable TLS verification on all connections.
pub fn setTlsVerify(self: *Client, verify: bool) !void {
// Remove inflight connections check on enable TLS b/c chromiumoxide calls
// the command during navigate and Curl seems to accept it...
var it = self.in_use.first;
while (it) |node| : (it = node.next) {
const conn: *http.Connection = @fieldParentPtr("node", node);
try conn.setTlsVerify(verify, self.use_proxy);
}
self.tls_verify = verify;
}
fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyerror!void { fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyerror!void {
const req = &transfer.req;
{ {
transfer._conn = conn; transfer._conn = conn;
errdefer { errdefer {
@@ -687,45 +650,7 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyer
self.releaseConn(conn); self.releaseConn(conn);
} }
// Set callbacks and per-client settings on the pooled connection. try transfer.configureConn(conn);
try conn.setCallbacks(Transfer.dataCallback);
try conn.setFollowLocation(false);
try conn.setProxy(self.http_proxy);
try conn.setTlsVerify(self.tls_verify, self.use_proxy);
try conn.setURL(req.url);
try conn.setMethod(req.method);
if (req.body) |b| {
try conn.setBody(b);
} else {
try conn.setGetMode();
}
var header_list = req.headers;
try conn.secretHeaders(&header_list, &self.network.config.http_headers); // Add headers that must be hidden from intercepts
try conn.setHeaders(&header_list);
// If we have WebBotAuth, sign our request.
if (self.network.web_bot_auth) |*wba| {
const authority = URL.getHost(req.url);
try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
}
// Add cookies.
if (header_list.cookies) |cookies| {
try conn.setCookies(cookies);
}
try conn.setPrivate(transfer);
// add credentials
if (req.credentials) |creds| {
if (transfer._auth_challenge != null and transfer._auth_challenge.?.source == .proxy) {
try conn.setProxyCredentials(creds);
} else {
try conn.setCredentials(creds);
}
}
} }
// As soon as this is called, our "perform" loop is responsible for // As soon as this is called, our "perform" loop is responsible for
@@ -741,15 +666,14 @@ fn makeRequest(self: *Client, conn: *http.Connection, transfer: *Transfer) anyer
self.releaseConn(conn); self.releaseConn(conn);
return err; return err;
}; };
self.active += 1;
if (req.start_callback) |cb| { if (transfer.req.start_callback) |cb| {
cb(transfer) catch |err| { cb(transfer) catch |err| {
transfer.deinit(); transfer.deinit();
return err; return err;
}; };
} }
self.active += 1;
_ = try self.perform(0); _ = try self.perform(0);
} }
@@ -758,7 +682,7 @@ pub const PerformStatus = enum {
normal, normal,
}; };
fn perform(self: *Client, timeout_ms: c_int) !PerformStatus { fn perform(self: *Client, timeout_ms: c_int) anyerror!PerformStatus {
const running = blk: { const running = blk: {
self.performing = true; self.performing = true;
defer self.performing = false; defer self.performing = false;
@@ -827,7 +751,7 @@ fn processMessages(self: *Client) !bool {
// to process it now. We can end the transfer, which will // to process it now. We can end the transfer, which will
// release the easy handle back into the pool. The transfer // release the easy handle back into the pool. The transfer
// is still valid/alive (just has no handle). // is still valid/alive (just has no handle).
self.endTransfer(transfer); transfer.releaseConn();
if (!transfer.req.blocking) { if (!transfer.req.blocking) {
// In the case of an async request, we can just "forget" // In the case of an async request, we can just "forget"
// about this transfer until it gets updated asynchronously // about this transfer until it gets updated asynchronously
@@ -841,7 +765,7 @@ fn processMessages(self: *Client) !bool {
// we've been asked to continue with the request // we've been asked to continue with the request
// we can't process it here, since we're already inside // we can't process it here, since we're already inside
// a process, so we need to queue it and wait for the // a process, so we need to queue it and wait for the
// next tick (this is why it was safe to endTransfer // next tick (this is why it was safe to releaseConn
// above, because even in the "blocking" path, we still // above, because even in the "blocking" path, we still
// only process it on the next tick). // only process it on the next tick).
self.queue.append(&transfer._node); self.queue.append(&transfer._node);
@@ -853,31 +777,33 @@ fn processMessages(self: *Client) !bool {
} }
} }
// Handle redirects: extract data from conn before releasing it. // Handle redirects: reuse the same connection to preserve TCP state.
if (msg.err == null) { if (msg.err == null) {
const status = try msg.conn.getResponseCode(); const status = try msg.conn.getResponseCode();
if (status >= 300 and status <= 399) { if (status >= 300 and status <= 399) {
transfer.handleRedirect(&msg.conn) catch |err| { transfer.handleRedirect() catch |err| {
requestFailed(transfer, err, true); transfer.requestFailed(err, true);
self.endTransfer(transfer);
transfer.deinit(); transfer.deinit();
continue; continue;
}; };
self.endTransfer(transfer);
const conn = transfer._conn.?;
try self.handles.remove(conn);
transfer.reset(); transfer.reset();
try self.process(transfer); try transfer.configureConn(conn);
try self.handles.add(conn);
_ = try self.perform(0);
continue; continue;
} }
} }
// release it ASAP so that it's available; some done_callbacks
// will load more resources.
self.endTransfer(transfer);
defer transfer.deinit(); defer transfer.deinit();
if (msg.err) |err| { if (msg.err) |err| {
requestFailed(transfer, err, true); transfer.requestFailed(err, true);
} else blk: { } else blk: {
// make sure the transfer can't be immediately aborted from a callback // make sure the transfer can't be immediately aborted from a callback
// since we still need it here. // since we still need it here.
@@ -889,18 +815,23 @@ fn processMessages(self: *Client) !bool {
// callback now. // callback now.
const proceed = transfer.headerDoneCallback(&msg.conn) catch |err| { const proceed = transfer.headerDoneCallback(&msg.conn) catch |err| {
lp.log.err(.http, "header_done_callback2", .{ .err = err }); lp.log.err(.http, "header_done_callback2", .{ .err = err });
requestFailed(transfer, err, true); transfer.requestFailed(err, true);
continue; continue;
}; };
if (!proceed) { if (!proceed) {
requestFailed(transfer, error.Abort, true); transfer.requestFailed(error.Abort, true);
break :blk; break :blk;
} }
} }
// release conn ASAP so that it's available; some done_callbacks
// will load more resources.
transfer.releaseConn();
transfer.req.done_callback(transfer.ctx) catch |err| { transfer.req.done_callback(transfer.ctx) catch |err| {
// transfer isn't valid at this point, don't use it. // transfer isn't valid at this point, don't use it.
lp.log.err(.http, "done_callback", .{ .err = err }); lp.log.err(.http, "done_callback", .{ .err = err });
requestFailed(transfer, err, true); transfer.requestFailed(err, true);
continue; continue;
}; };
@@ -913,15 +844,9 @@ fn processMessages(self: *Client) !bool {
return processed; return processed;
} }
fn endTransfer(self: *Client, transfer: *Transfer) void {
const conn = transfer._conn.?;
self.removeConn(conn);
transfer._conn = null;
self.active -= 1;
}
fn removeConn(self: *Client, conn: *http.Connection) void { fn removeConn(self: *Client, conn: *http.Connection) void {
self.in_use.remove(&conn.node); self.in_use.remove(&conn.node);
self.active -= 1;
if (self.handles.remove(conn)) { if (self.handles.remove(conn)) {
self.releaseConn(conn); self.releaseConn(conn);
} else |_| { } else |_| {
@@ -1012,8 +937,6 @@ pub const Request = struct {
}; };
}; };
const AuthChallenge = http.AuthChallenge;
pub const Transfer = struct { pub const Transfer = struct {
arena: ArenaAllocator, arena: ArenaAllocator,
id: u32 = 0, id: u32 = 0,
@@ -1039,7 +962,7 @@ pub const Transfer = struct {
_conn: ?*http.Connection = null, _conn: ?*http.Connection = null,
_auth_challenge: ?AuthChallenge = null, _auth_challenge: ?http.AuthChallenge = null,
// number of times the transfer has been tried. // number of times the transfer has been tried.
// incremented by reset func. // incremented by reset func.
@@ -1059,6 +982,116 @@ pub const Transfer = struct {
fulfilled, fulfilled,
}; };
fn releaseConn(self: *Transfer) void {
if (self._conn) |conn| {
self.client.removeConn(conn);
self._conn = null;
}
}
fn deinit(self: *Transfer) void {
if (self._conn) |conn| {
self.client.removeConn(conn);
self._conn = null;
}
self.req.headers.deinit();
self.arena.deinit();
self.client.transfer_pool.destroy(self);
}
pub fn abort(self: *Transfer, err: anyerror) void {
self.requestFailed(err, true);
if (self._performing or self.client.performing) {
// We're currently in a curl_multi_perform. We cannot call
// curl_multi_remove_handle from a curl callback. Instead, we flag
// this transfer and our callbacks will check for this flag.
self.aborted = true;
return;
}
self.deinit();
}
pub fn terminate(self: *Transfer) void {
self.requestFailed(error.Shutdown, false);
self.deinit();
}
// internal, when the page is shutting down. Doesn't have the same ceremony
// as abort (doesn't send a notification, doesn't invoke an error callback)
fn kill(self: *Transfer) void {
if (self.req.shutdown_callback) |cb| {
cb(self.ctx);
}
self.deinit();
}
// We can force a failed request within a callback, which will eventually
// result in this being called again in the more general loop. We do this
// because we can raise a more specific error inside a callback in some cases.
fn requestFailed(self: *Transfer, err: anyerror, comptime execute_callback: bool) void {
if (self._notified_fail) return;
self._notified_fail = true;
self.req.notification.dispatch(.http_request_fail, &.{
.transfer = self,
.err = err,
});
if (execute_callback) {
self.req.error_callback(self.ctx, err);
} else if (self.req.shutdown_callback) |cb| {
cb(self.ctx);
}
}
fn configureConn(self: *Transfer, conn: *http.Connection) anyerror!void {
const client = self.client;
const req = &self.req;
// Set callbacks and per-client settings on the pooled connection.
try conn.setCallbacks(Transfer.dataCallback);
try conn.setFollowLocation(false);
try conn.setProxy(client.http_proxy);
try conn.setTlsVerify(client.tls_verify, client.use_proxy);
try conn.setURL(req.url);
try conn.setMethod(req.method);
if (req.body) |b| {
try conn.setBody(b);
} else {
try conn.setGetMode();
}
var header_list = req.headers;
try conn.secretHeaders(&header_list, &client.network.config.http_headers);
try conn.setHeaders(&header_list);
// If we have WebBotAuth, sign our request.
if (client.network.web_bot_auth) |*wba| {
const authority = URL.getHost(req.url);
try wba.signRequest(self.arena.allocator(), &header_list, authority);
}
// Add cookies.
if (header_list.cookies) |cookies| {
try conn.setCookies(cookies);
}
try conn.setPrivate(self);
// add credentials
if (req.credentials) |creds| {
if (self._auth_challenge != null and self._auth_challenge.?.source == .proxy) {
try conn.setProxyCredentials(creds);
} else {
try conn.setCredentials(creds);
}
}
}
pub fn reset(self: *Transfer) void { pub fn reset(self: *Transfer) void {
self._auth_challenge = null; self._auth_challenge = null;
self._notified_fail = false; self._notified_fail = false;
@@ -1067,15 +1100,6 @@ pub const Transfer = struct {
self._tries += 1; self._tries += 1;
} }
fn deinit(self: *Transfer) void {
self.req.headers.deinit();
if (self._conn) |conn| {
self.client.removeConn(conn);
}
self.arena.deinit();
self.client.transfer_pool.destroy(self);
}
fn buildResponseHeader(self: *Transfer, conn: *const http.Connection) !void { fn buildResponseHeader(self: *Transfer, conn: *const http.Connection) !void {
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
std.debug.assert(self.response_header == null); std.debug.assert(self.response_header == null);
@@ -1116,8 +1140,9 @@ pub const Transfer = struct {
self.req.url = url; self.req.url = url;
} }
fn handleRedirect(transfer: *Transfer, conn: *const http.Connection) !void { fn handleRedirect(transfer: *Transfer) !void {
const req = &transfer.req; const req = &transfer.req;
const conn = transfer._conn.?;
const arena = transfer.arena.allocator(); const arena = transfer.arena.allocator();
transfer._redirect_count += 1; transfer._redirect_count += 1;
@@ -1125,15 +1150,18 @@ pub const Transfer = struct {
return error.TooManyRedirects; return error.TooManyRedirects;
} }
lp.log.warn(.bug, "Redirecting...", .{});
// retrieve cookies from the redirect's response. // retrieve cookies from the redirect's response.
if (req.cookie_jar) |jar| { if (req.cookie_jar) |jar| {
var i: usize = 0; var i: usize = 0;
while (true) { while (conn.getResponseHeader("set-cookie", i)) |ct| : (i += 1) {
const ct = conn.getResponseHeader("set-cookie", i); lp.log.warn(.bug, "set-cookie", .{ i, ct.value });
if (ct == null) break; try jar.populateFromResponse(transfer.url, ct.value);
try jar.populateFromResponse(transfer.url, ct.?.value);
i += 1; if (i >= ct.amount) {
if (i >= ct.?.amount) break; break;
}
} }
} }
@@ -1168,6 +1196,7 @@ pub const Transfer = struct {
} else { } else {
req.headers.cookies = null; req.headers.cookies = null;
} }
lp.log.warn(.bug, "cookie", .{cookies.items[0..]});
} }
} }
@@ -1179,9 +1208,9 @@ pub const Transfer = struct {
} }
if (conn.getResponseHeader("WWW-Authenticate", 0)) |hdr| { if (conn.getResponseHeader("WWW-Authenticate", 0)) |hdr| {
transfer._auth_challenge = AuthChallenge.parse(status, .server, hdr.value) catch null; transfer._auth_challenge = http.AuthChallenge.parse(status, .server, hdr.value) catch null;
} else if (conn.getResponseHeader("Proxy-Authenticate", 0)) |hdr| { } else if (conn.getResponseHeader("Proxy-Authenticate", 0)) |hdr| {
transfer._auth_challenge = AuthChallenge.parse(status, .proxy, hdr.value) catch null; transfer._auth_challenge = http.AuthChallenge.parse(status, .proxy, hdr.value) catch null;
} else { } else {
transfer._auth_challenge = .{ .status = status, .source = null, .scheme = null, .realm = null }; transfer._auth_challenge = .{ .status = status, .source = null, .scheme = null, .realm = null };
} }
@@ -1207,48 +1236,8 @@ pub const Transfer = struct {
self.req.headers = new_headers; self.req.headers = new_headers;
} }
pub fn abort(self: *Transfer, err: anyerror) void {
requestFailed(self, err, true);
const client = self.client;
if (self._performing or client.performing) {
// We're currently in a curl_multi_perform. We cannot call endTransfer
// as that calls curl_multi_remove_handle, and you can't do that
// from a curl callback. Instead, we flag this transfer and all of
// our callbacks will check for this flag and abort the transfer for
// us
self.aborted = true;
return;
}
if (self._conn != null) {
client.endTransfer(self);
}
self.deinit();
}
pub fn terminate(self: *Transfer) void {
requestFailed(self, error.Shutdown, false);
if (self._conn != null) {
self.client.endTransfer(self);
}
self.deinit();
}
// internal, when the page is shutting down. Doesn't have the same ceremony
// as abort (doesn't send a notification, doesn't invoke an error callback)
fn kill(self: *Transfer) void {
if (self._conn != null) {
self.client.endTransfer(self);
}
if (self.req.shutdown_callback) |cb| {
cb(self.ctx);
}
self.deinit();
}
// abortAuthChallenge is called when an auth challenge interception is // abortAuthChallenge is called when an auth challenge interception is
// abort. We don't call self.client.endTransfer here b/c it has been done // abort. We don't call self.releaseConn here b/c it has been done
// before interception process. // before interception process.
pub fn abortAuthChallenge(self: *Transfer) void { pub fn abortAuthChallenge(self: *Transfer) void {
if (comptime IS_DEBUG) { if (comptime IS_DEBUG) {
@@ -1320,8 +1309,8 @@ pub const Transfer = struct {
std.debug.assert(chunk_count == 1); std.debug.assert(chunk_count == 1);
} }
const conn: http.Connection = .{ .easy = @ptrCast(@alignCast(data)) }; const conn: *http.Connection = @ptrCast(@alignCast(data));
var transfer = fromConnection(&conn) catch |err| { var transfer = fromConnection(conn) catch |err| {
lp.log.err(.http, "get private info", .{ .err = err, .source = "body callback" }); lp.log.err(.http, "get private info", .{ .err = err, .source = "body callback" });
return http.writefunc_error; return http.writefunc_error;
}; };
@@ -1333,7 +1322,7 @@ pub const Transfer = struct {
} }
if (!transfer._header_done_called) { if (!transfer._header_done_called) {
const proceed = transfer.headerDoneCallback(&conn) catch |err| { const proceed = transfer.headerDoneCallback(conn) catch |err| {
lp.log.err(.http, "header_done_callback", .{ .err = err, .req = transfer }); lp.log.err(.http, "header_done_callback", .{ .err = err, .req = transfer });
return http.writefunc_error; return http.writefunc_error;
}; };
@@ -1346,7 +1335,7 @@ pub const Transfer = struct {
transfer.bytes_received += chunk_len; transfer.bytes_received += chunk_len;
if (transfer.max_response_size) |max_size| { if (transfer.max_response_size) |max_size| {
if (transfer.bytes_received > max_size) { if (transfer.bytes_received > max_size) {
requestFailed(transfer, error.ResponseTooLarge, true); transfer.requestFailed(error.ResponseTooLarge, true);
return http.writefunc_error; return http.writefunc_error;
} }
} }
@@ -1382,7 +1371,7 @@ pub const Transfer = struct {
return .{ .list = .{ .list = self.response_header.?._injected_headers } }; return .{ .list = .{ .list = self.response_header.?._injected_headers } };
} }
pub fn fromConnection(conn: *const http.Connection) !*Transfer { fn fromConnection(conn: *const http.Connection) !*Transfer {
const private = try conn.getPrivate(); const private = try conn.getPrivate();
return @ptrCast(@alignCast(private)); return @ptrCast(@alignCast(private));
} }

4
src/browser/ScriptManager.zig
View File

@@ -739,7 +739,7 @@ pub const Script = struct {
.b5 = transfer._notified_fail, .b5 = transfer._notified_fail,
.b7 = @intFromEnum(transfer._intercept_state), .b7 = @intFromEnum(transfer._intercept_state),
.b8 = transfer._auth_challenge != null, .b8 = transfer._auth_challenge != null,
.b9 = if (transfer._conn) |c| @intFromPtr(c.easy) else 0, .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
}); });
self.header_callback_called = true; self.header_callback_called = true;
self.debug_transfer_id = transfer.id; self.debug_transfer_id = transfer.id;
@@ -749,7 +749,7 @@ pub const Script = struct {
self.debug_transfer_notified_fail = transfer._notified_fail; self.debug_transfer_notified_fail = transfer._notified_fail;
self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state); self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
self.debug_transfer_auth_challenge = transfer._auth_challenge != null; self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c.easy) else 0; self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
} }
lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity }); lp.assert(self.source.remote.capacity == 0, "ScriptManager.Header buffer", .{ .capacity = self.source.remote.capacity });

4
src/cdp/cdp.zig
View File

@@ -475,8 +475,8 @@ pub fn BrowserContext(comptime CDP_T: type) type {
if (self.http_proxy_changed) { if (self.http_proxy_changed) {
// has to be called after browser.closeSession, since it won't // has to be called after browser.closeSession, since it won't
// work if there are active connections. // work if there are active connections.
browser.http_client.restoreOriginalProxy() catch |err| { browser.http_client.changeProxy(null) catch |err| {
log.warn(.http, "restoreOriginalProxy", .{ .err = err }); log.warn(.http, "changeProxy", .{ .err = err });
}; };
} }
self.intercept_state.deinit(); self.intercept_state.deinit();

4
src/network/Runtime.zig
View File

@@ -461,7 +461,7 @@ fn drainQueue(self: *Runtime) void {
self.releaseConnection(conn); self.releaseConnection(conn);
continue; continue;
}; };
libcurl.curl_multi_add_handle(multi, conn.easy) catch |err| { libcurl.curl_multi_add_handle(multi, conn._easy) catch |err| {
lp.log.err(.app, "curl multi add", .{ .err = err }); lp.log.err(.app, "curl multi add", .{ .err = err });
self.releaseConnection(conn); self.releaseConnection(conn);
}; };
@@ -565,7 +565,7 @@ pub fn getConnection(self: *Runtime) ?*net_http.Connection {
} }
pub fn releaseConnection(self: *Runtime, conn: *net_http.Connection) void { pub fn releaseConnection(self: *Runtime, conn: *net_http.Connection) void {
conn.reset() catch |err| { conn.reset(self.config, self.ca_blob) catch |err| {
lp.assert(false, "couldn't reset curl easy", .{ .err = err }); lp.assert(false, "couldn't reset curl easy", .{ .err = err });
}; };

176
src/network/http.zig
View File

@@ -142,7 +142,7 @@ pub const HeaderIterator = union(enum) {
prev: ?*libcurl.CurlHeader = null, prev: ?*libcurl.CurlHeader = null,
pub fn next(self: *CurlHeaderIterator) ?Header { pub fn next(self: *CurlHeaderIterator) ?Header {
const h = libcurl.curl_easy_nextheader(self.conn.easy, .header, -1, self.prev) orelse return null; const h = libcurl.curl_easy_nextheader(self.conn._easy, .header, -1, self.prev) orelse return null;
self.prev = h; self.prev = h;
const header = h.*; const header = h.*;
@@ -227,77 +227,28 @@ pub const ResponseHead = struct {
}; };
pub const Connection = struct { pub const Connection = struct {
easy: *libcurl.Curl, _easy: *libcurl.Curl,
node: std.DoublyLinkedList.Node = .{}, node: std.DoublyLinkedList.Node = .{},
pub fn init( pub fn init(
ca_blob_: ?libcurl.CurlBlob, ca_blob: ?libcurl.CurlBlob,
config: *const Config, config: *const Config,
) !Connection { ) !Connection {
const easy = libcurl.curl_easy_init() orelse return error.FailedToInitializeEasy; const easy = libcurl.curl_easy_init() orelse return error.FailedToInitializeEasy;
errdefer libcurl.curl_easy_cleanup(easy);
// timeouts const self = Connection{ ._easy = easy };
try libcurl.curl_easy_setopt(easy, .timeout_ms, config.httpTimeout()); errdefer self.deinit();
try libcurl.curl_easy_setopt(easy, .connect_timeout_ms, config.httpConnectTimeout());
// redirect behavior try self.reset(config, ca_blob);
try libcurl.curl_easy_setopt(easy, .max_redirs, config.httpMaxRedirects()); return self;
try libcurl.curl_easy_setopt(easy, .follow_location, 2);
try libcurl.curl_easy_setopt(easy, .redir_protocols_str, "HTTP,HTTPS"); // remove FTP and FTPS from the default
// proxy
const http_proxy = config.httpProxy();
if (http_proxy) |proxy| {
try libcurl.curl_easy_setopt(easy, .proxy, proxy.ptr);
}
// tls
if (ca_blob_) |ca_blob| {
try libcurl.curl_easy_setopt(easy, .ca_info_blob, ca_blob);
if (http_proxy != null) {
try libcurl.curl_easy_setopt(easy, .proxy_ca_info_blob, ca_blob);
}
} else {
assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{});
try libcurl.curl_easy_setopt(easy, .ssl_verify_host, false);
try libcurl.curl_easy_setopt(easy, .ssl_verify_peer, false);
if (http_proxy != null) {
try libcurl.curl_easy_setopt(easy, .proxy_ssl_verify_host, false);
try libcurl.curl_easy_setopt(easy, .proxy_ssl_verify_peer, false);
}
}
// compression, don't remove this. CloudFront will send gzip content
// even if we don't support it, and then it won't be decompressed.
// empty string means: use whatever's available
try libcurl.curl_easy_setopt(easy, .accept_encoding, "");
// debug
if (comptime ENABLE_DEBUG) {
try libcurl.curl_easy_setopt(easy, .verbose, true);
// Sometimes the default debug output hides some useful data. You can
// uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to
// get more control over the data (specifically, the `CURLINFO_TEXT`
// can include useful data).
// try libcurl.curl_easy_setopt(easy, .debug_function, debugCallback);
}
return .{
.easy = easy,
};
} }
pub fn deinit(self: *const Connection) void { pub fn deinit(self: *const Connection) void {
libcurl.curl_easy_cleanup(self.easy); libcurl.curl_easy_cleanup(self._easy);
} }
pub fn setURL(self: *const Connection, url: [:0]const u8) !void { pub fn setURL(self: *const Connection, url: [:0]const u8) !void {
try libcurl.curl_easy_setopt(self.easy, .url, url.ptr); try libcurl.curl_easy_setopt(self._easy, .url, url.ptr);
} }
// a libcurl request has 2 methods. The first is the method that // a libcurl request has 2 methods. The first is the method that
@@ -320,7 +271,7 @@ pub const Connection = struct {
// can infer that based on the presence of the body, but we also reset it // can infer that based on the presence of the body, but we also reset it
// to be safe); // to be safe);
pub fn setMethod(self: *const Connection, method: Method) !void { pub fn setMethod(self: *const Connection, method: Method) !void {
const easy = self.easy; const easy = self._easy;
const m: [:0]const u8 = switch (method) { const m: [:0]const u8 = switch (method) {
.GET => "GET", .GET => "GET",
.POST => "POST", .POST => "POST",
@@ -335,50 +286,97 @@ pub const Connection = struct {
} }
pub fn setBody(self: *const Connection, body: []const u8) !void { pub fn setBody(self: *const Connection, body: []const u8) !void {
const easy = self.easy; const easy = self._easy;
try libcurl.curl_easy_setopt(easy, .post, true); try libcurl.curl_easy_setopt(easy, .post, true);
try libcurl.curl_easy_setopt(easy, .post_field_size, body.len); try libcurl.curl_easy_setopt(easy, .post_field_size, body.len);
try libcurl.curl_easy_setopt(easy, .copy_post_fields, body.ptr); try libcurl.curl_easy_setopt(easy, .copy_post_fields, body.ptr);
} }
pub fn setGetMode(self: *const Connection) !void { pub fn setGetMode(self: *const Connection) !void {
try libcurl.curl_easy_setopt(self.easy, .http_get, true); try libcurl.curl_easy_setopt(self._easy, .http_get, true);
} }
pub fn setHeaders(self: *const Connection, headers: *Headers) !void { pub fn setHeaders(self: *const Connection, headers: *Headers) !void {
try libcurl.curl_easy_setopt(self.easy, .http_header, headers.headers); try libcurl.curl_easy_setopt(self._easy, .http_header, headers.headers);
} }
pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void { pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void {
try libcurl.curl_easy_setopt(self.easy, .cookie, cookies); try libcurl.curl_easy_setopt(self._easy, .cookie, cookies);
} }
pub fn setPrivate(self: *const Connection, ptr: *anyopaque) !void { pub fn setPrivate(self: *const Connection, ptr: *anyopaque) !void {
try libcurl.curl_easy_setopt(self.easy, .private, ptr); try libcurl.curl_easy_setopt(self._easy, .private, ptr);
} }
pub fn setProxyCredentials(self: *const Connection, creds: [:0]const u8) !void { pub fn setProxyCredentials(self: *const Connection, creds: [:0]const u8) !void {
try libcurl.curl_easy_setopt(self.easy, .proxy_user_pwd, creds.ptr); try libcurl.curl_easy_setopt(self._easy, .proxy_user_pwd, creds.ptr);
} }
pub fn setCredentials(self: *const Connection, creds: [:0]const u8) !void { pub fn setCredentials(self: *const Connection, creds: [:0]const u8) !void {
try libcurl.curl_easy_setopt(self.easy, .user_pwd, creds.ptr); try libcurl.curl_easy_setopt(self._easy, .user_pwd, creds.ptr);
} }
pub fn setCallbacks( pub fn setCallbacks(
self: *const Connection, self: *Connection,
comptime data_cb: libcurl.CurlWriteFunction, comptime data_cb: libcurl.CurlWriteFunction,
) !void { ) !void {
try libcurl.curl_easy_setopt(self.easy, .write_data, self.easy); try libcurl.curl_easy_setopt(self._easy, .write_data, self);
try libcurl.curl_easy_setopt(self.easy, .write_function, data_cb); try libcurl.curl_easy_setopt(self._easy, .write_function, data_cb);
} }
pub fn reset(self: *const Connection) !void { pub fn reset(
try libcurl.curl_easy_setopt(self.easy, .proxy, null); self: *const Connection,
try libcurl.curl_easy_setopt(self.easy, .http_header, null); config: *const Config,
ca_blob: ?libcurl.CurlBlob,
) !void {
libcurl.curl_easy_reset(self._easy);
try libcurl.curl_easy_setopt(self.easy, .write_data, null); // timeouts
try libcurl.curl_easy_setopt(self.easy, .write_function, discardBody); try libcurl.curl_easy_setopt(self._easy, .timeout_ms, config.httpTimeout());
try libcurl.curl_easy_setopt(self._easy, .connect_timeout_ms, config.httpConnectTimeout());
// compression, don't remove this. CloudFront will send gzip content
// even if we don't support it, and then it won't be decompressed.
// empty string means: use whatever's available
try libcurl.curl_easy_setopt(self._easy, .accept_encoding, "");
// proxy
const http_proxy = config.httpProxy();
if (http_proxy) |proxy| {
try libcurl.curl_easy_setopt(self._easy, .proxy, proxy.ptr);
} else {
try libcurl.curl_easy_setopt(self._easy, .proxy, null);
}
// tls
if (ca_blob) |ca| {
try libcurl.curl_easy_setopt(self._easy, .ca_info_blob, ca);
if (http_proxy != null) {
try libcurl.curl_easy_setopt(self._easy, .proxy_ca_info_blob, ca);
}
} else {
assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{});
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);
if (http_proxy != null) {
try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, false);
try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, false);
}
}
// debug
if (comptime ENABLE_DEBUG) {
try libcurl.curl_easy_setopt(self._easy, .verbose, true);
// Sometimes the default debug output hides some useful data. You can
// uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to
// get more control over the data (specifically, the `CURLINFO_TEXT`
// can include useful data).
// try libcurl.curl_easy_setopt(easy, .debug_function, debugCallback);
}
} }
fn discardBody(_: [*]const u8, count: usize, len: usize, _: ?*anyopaque) usize { fn discardBody(_: [*]const u8, count: usize, len: usize, _: ?*anyopaque) usize {
@@ -386,31 +384,31 @@ pub const Connection = struct {
} }
pub fn setProxy(self: *const Connection, proxy: ?[:0]const u8) !void { pub fn setProxy(self: *const Connection, proxy: ?[:0]const u8) !void {
try libcurl.curl_easy_setopt(self.easy, .proxy, if (proxy) |p| p.ptr else null); try libcurl.curl_easy_setopt(self._easy, .proxy, if (proxy) |p| p.ptr else null);
} }
pub fn setFollowLocation(self: *const Connection, follow: bool) !void { pub fn setFollowLocation(self: *const Connection, follow: bool) !void {
try libcurl.curl_easy_setopt(self.easy, .follow_location, @as(c_long, if (follow) 2 else 0)); try libcurl.curl_easy_setopt(self._easy, .follow_location, @as(c_long, if (follow) 2 else 0));
} }
pub fn setTlsVerify(self: *const Connection, verify: bool, use_proxy: bool) !void { pub fn setTlsVerify(self: *const Connection, verify: bool, use_proxy: bool) !void {
try libcurl.curl_easy_setopt(self.easy, .ssl_verify_host, verify); try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, verify);
try libcurl.curl_easy_setopt(self.easy, .ssl_verify_peer, verify); try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, verify);
if (use_proxy) { if (use_proxy) {
try libcurl.curl_easy_setopt(self.easy, .proxy_ssl_verify_host, verify); try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_host, verify);
try libcurl.curl_easy_setopt(self.easy, .proxy_ssl_verify_peer, verify); try libcurl.curl_easy_setopt(self._easy, .proxy_ssl_verify_peer, verify);
} }
} }
pub fn getEffectiveUrl(self: *const Connection) ![*c]const u8 { pub fn getEffectiveUrl(self: *const Connection) ![*c]const u8 {
var url: [*c]u8 = undefined; var url: [*c]u8 = undefined;
try libcurl.curl_easy_getinfo(self.easy, .effective_url, &url); try libcurl.curl_easy_getinfo(self._easy, .effective_url, &url);
return url; return url;
} }
pub fn getResponseCode(self: *const Connection) !u16 { pub fn getResponseCode(self: *const Connection) !u16 {
var status: c_long = undefined; var status: c_long = undefined;
try libcurl.curl_easy_getinfo(self.easy, .response_code, &status); try libcurl.curl_easy_getinfo(self._easy, .response_code, &status);
if (status < 0 or status > std.math.maxInt(u16)) { if (status < 0 or status > std.math.maxInt(u16)) {
return 0; return 0;
} }
@@ -419,13 +417,13 @@ pub const Connection = struct {
pub fn getRedirectCount(self: *const Connection) !u32 { pub fn getRedirectCount(self: *const Connection) !u32 {
var count: c_long = undefined; var count: c_long = undefined;
try libcurl.curl_easy_getinfo(self.easy, .redirect_count, &count); try libcurl.curl_easy_getinfo(self._easy, .redirect_count, &count);
return @intCast(count); return @intCast(count);
} }
pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue { pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
var hdr: ?*libcurl.CurlHeader = null; var hdr: ?*libcurl.CurlHeader = null;
libcurl.curl_easy_header(self.easy, name, index, .header, -1, &hdr) catch |err| { libcurl.curl_easy_header(self._easy, name, index, .header, -1, &hdr) catch |err| {
// ErrorHeader includes OutOfMemory — rare but real errors from curl internals. // ErrorHeader includes OutOfMemory — rare but real errors from curl internals.
// Logged and returned as null since callers don't expect errors. // Logged and returned as null since callers don't expect errors.
log.err(.http, "get response header", .{ log.err(.http, "get response header", .{
@@ -443,7 +441,7 @@ pub const Connection = struct {
pub fn getPrivate(self: *const Connection) !*anyopaque { pub fn getPrivate(self: *const Connection) !*anyopaque {
var private: *anyopaque = undefined; var private: *anyopaque = undefined;
try libcurl.curl_easy_getinfo(self.easy, .private, &private); try libcurl.curl_easy_getinfo(self._easy, .private, &private);
return private; return private;
} }
@@ -465,7 +463,7 @@ pub const Connection = struct {
try self.setCookies(cookies); try self.setCookies(cookies);
} }
try libcurl.curl_easy_perform(self.easy); try libcurl.curl_easy_perform(self._easy);
return self.getResponseCode(); return self.getResponseCode();
} }
}; };
@@ -487,11 +485,11 @@ pub const Handles = struct {
} }
pub fn add(self: *Handles, conn: *const Connection) !void { pub fn add(self: *Handles, conn: *const Connection) !void {
try libcurl.curl_multi_add_handle(self.multi, conn.easy); try libcurl.curl_multi_add_handle(self.multi, conn._easy);
} }
pub fn remove(self: *Handles, conn: *const Connection) !void { pub fn remove(self: *Handles, conn: *const Connection) !void {
try libcurl.curl_multi_remove_handle(self.multi, conn.easy); try libcurl.curl_multi_remove_handle(self.multi, conn._easy);
} }
pub fn perform(self: *Handles) !c_int { pub fn perform(self: *Handles) !c_int {
@@ -514,7 +512,7 @@ pub const Handles = struct {
const msg = libcurl.curl_multi_info_read(self.multi, &messages_count) orelse return null; const msg = libcurl.curl_multi_info_read(self.multi, &messages_count) orelse return null;
return switch (msg.data) { return switch (msg.data) {
.done => |err| .{ .done => |err| .{
.conn = .{ .easy = msg.easy_handle }, .conn = .{ ._easy = msg.easy_handle },
.err = err, .err = err,
}, },
else => unreachable, else => unreachable,

4
src/sys/libcurl.zig
View File

@@ -516,6 +516,10 @@ pub fn curl_easy_cleanup(easy: *Curl) void {
c.curl_easy_cleanup(easy); c.curl_easy_cleanup(easy);
} }
pub fn curl_easy_reset(easy: *Curl) void {
c.curl_easy_reset(easy);
}
pub fn curl_easy_perform(easy: *Curl) Error!void { pub fn curl_easy_perform(easy: *Curl) Error!void {
try errorCheck(c.curl_easy_perform(easy)); try errorCheck(c.curl_easy_perform(easy));
} }