diff --git a/src/Net.zig b/src/Net.zig index f3d3558a..c667a902 100644 --- a/src/Net.zig +++ b/src/Net.zig @@ -33,332 +33,6 @@ const assert = @import("lightpanda").assert; pub const ENABLE_DEBUG = false; const IS_DEBUG = builtin.mode == .Debug; -pub fn globalInit() Error!void { - try errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL)); -} - -pub fn globalDeinit() void { - c.curl_global_cleanup(); -} - -pub const Method = enum(u8) { - GET = 0, - PUT = 1, - POST = 2, - DELETE = 3, - HEAD = 4, - OPTIONS = 5, - PATCH = 6, - PROPFIND = 7, -}; - -pub const Header = struct { - name: []const u8, - value: []const u8, -}; - -pub const Headers = struct { - headers: ?*c.curl_slist, - cookies: ?[*c]const u8, - - pub fn init(user_agent: [:0]const u8) !Headers { - const header_list = c.curl_slist_append(null, user_agent); - if (header_list == null) { - return error.OutOfMemory; - } - return .{ .headers = header_list, .cookies = null }; - } - - pub fn deinit(self: *const Headers) void { - if (self.headers) |hdr| { - c.curl_slist_free_all(hdr); - } - } - - pub fn add(self: *Headers, header: [*c]const u8) !void { - // Copies the value - const updated_headers = c.curl_slist_append(self.headers, header); - if (updated_headers == null) return error.OutOfMemory; - self.headers = updated_headers; - } - - pub fn parseHeader(header_str: []const u8) ?Header { - const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null; - - const name = std.mem.trim(u8, header_str[0..colon_pos], " \t"); - const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t"); - - return .{ .name = name, .value = value }; - } - - pub fn iterator(self: *Headers) Iterator { - return .{ - .header = self.headers, - .cookies = self.cookies, - }; - } - - const Iterator = struct { - header: [*c]c.curl_slist, - cookies: ?[*c]const u8, - - pub fn next(self: *Iterator) ?Header { - const h = self.header orelse { - const cookies = self.cookies orelse return null; - self.cookies = null; - return .{ .name = "Cookie", .value = std.mem.span(@as([*:0]const u8, cookies)) }; - }; - - self.header = h.*.next; - return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data)))); - } - }; -}; - -pub const Connection = struct { - easy: *c.CURL, - http_headers: *const Config.HttpHeaders, - - pub fn init( - ca_blob_: ?c.curl_blob, - config: *const Config, - ) !Connection { - const easy = c.curl_easy_init() orelse return error.FailedToInitializeEasy; - errdefer _ = c.curl_easy_cleanup(easy); - - // timeouts - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_TIMEOUT_MS, @as(c_long, @intCast(config.httpTimeout())))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CONNECTTIMEOUT_MS, @as(c_long, @intCast(config.httpConnectTimeout())))); - - // redirect behavior - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_MAXREDIRS, @as(c_long, @intCast(config.httpMaxRedirects())))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_FOLLOWLOCATION, @as(c_long, 2))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_REDIR_PROTOCOLS_STR, "HTTP,HTTPS")); // remove FTP and FTPS from the default - - // proxy - const http_proxy = config.httpProxy(); - if (http_proxy) |proxy| { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY, proxy.ptr)); - } - - // tls - if (ca_blob_) |ca_blob| { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CAINFO_BLOB, ca_blob)); - if (http_proxy != null) { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_CAINFO_BLOB, ca_blob)); - } - } else { - assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{}); - - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYHOST, @as(c_long, 0))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYPEER, @as(c_long, 0))); - - if (http_proxy != null) { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYHOST, @as(c_long, 0))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYPEER, @as(c_long, 0))); - } - } - - // compression, don't remove this. CloudFront will send gzip content - // even if we don't support it, and then it won't be decompressed. - // empty string means: use whatever's available - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_ACCEPT_ENCODING, "")); - - // debug - if (comptime ENABLE_DEBUG) { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_VERBOSE, @as(c_long, 1))); - - // Sometimes the default debug output hides some useful data. You can - // uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to - // get more control over the data (specifically, the `CURLINFO_TEXT` - // can include useful data). - - // try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_DEBUGFUNCTION, debugCallback)); - } - - return .{ - .easy = easy, - .http_headers = &config.http_headers, - }; - } - - pub fn deinit(self: *const Connection) void { - c.curl_easy_cleanup(self.easy); - } - - pub fn setURL(self: *const Connection, url: [:0]const u8) !void { - try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_URL, url.ptr)); - } - - // a libcurl request has 2 methods. The first is the method that - // controls how libcurl behaves. This specifically influences how redirects - // are handled. For example, if you do a POST and get a 301, libcurl will - // change that to a GET. But if you do a POST and get a 308, libcurl will - // keep the POST (and re-send the body). - // The second method is the actual string that's included in the request - // headers. - // These two methods can be different - you can tell curl to behave as though - // you made a GET, but include "POST" in the request header. - // - // Here, we're only concerned about the 2nd method. If we want, we'll set - // the first one based on whether or not we have a body. - // - // It's important that, for each use of this connection, we set the 2nd - // method. Else, if we make a HEAD request and re-use the connection, but - // DON'T reset this, it'll keep making HEAD requests. - // (I don't know if it's as important to reset the 1st method, or if libcurl - // can infer that based on the presence of the body, but we also reset it - // to be safe); - pub fn setMethod(self: *const Connection, method: Method) !void { - const easy = self.easy; - const m: [:0]const u8 = switch (method) { - .GET => "GET", - .POST => "POST", - .PUT => "PUT", - .DELETE => "DELETE", - .HEAD => "HEAD", - .OPTIONS => "OPTIONS", - .PATCH => "PATCH", - .PROPFIND => "PROPFIND", - }; - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, m.ptr)); - } - - pub fn setBody(self: *const Connection, body: []const u8) !void { - const easy = self.easy; - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPPOST, @as(c_long, 1))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(body.len)))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_COPYPOSTFIELDS, body.ptr)); - } - - // These are headers that may not be send to the users for inteception. - pub fn secretHeaders(self: *const Connection, headers: *Headers) !void { - if (self.http_headers.proxy_bearer_header) |hdr| { - try headers.add(hdr); - } - } - - pub fn request(self: *const Connection) !u16 { - const easy = self.easy; - - var header_list = try Headers.init(self.http_headers.user_agent_header); - defer header_list.deinit(); - try self.secretHeaders(&header_list); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); - - // Add cookies. - if (header_list.cookies) |cookies| { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_COOKIE, cookies)); - } - - try errorCheck(c.curl_easy_perform(easy)); - var http_code: c_long = undefined; - try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_RESPONSE_CODE, &http_code)); - if (http_code < 0 or http_code > std.math.maxInt(u16)) { - return 0; - } - return @intCast(http_code); - } -}; - -// TODO: on BSD / Linux, we could just read the PEM file directly. -// This whole rescan + decode is really just needed for MacOS. On Linux -// bundle.rescan does find the .pem file(s) which could be in a few different -// places, so it's still useful, just not efficient. -pub fn loadCerts(allocator: Allocator) !c.curl_blob { - var bundle: std.crypto.Certificate.Bundle = .{}; - try bundle.rescan(allocator); - defer bundle.deinit(allocator); - - const bytes = bundle.bytes.items; - if (bytes.len == 0) { - log.warn(.app, "No system certificates", .{}); - return .{ - .len = 0, - .flags = 0, - .data = bytes.ptr, - }; - } - - const encoder = std.base64.standard.Encoder; - var arr: std.ArrayList(u8) = .empty; - - const encoded_size = encoder.calcSize(bytes.len); - const buffer_size = encoded_size + - (bundle.map.count() * 75) + // start / end per certificate + extra, just in case - (encoded_size / 64) // newline per 64 characters - ; - try arr.ensureTotalCapacity(allocator, buffer_size); - errdefer arr.deinit(allocator); - var writer = arr.writer(allocator); - - var it = bundle.map.valueIterator(); - while (it.next()) |index| { - const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*); - - try writer.writeAll("-----BEGIN CERTIFICATE-----\n"); - var line_writer = LineWriter{ .inner = writer }; - try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]); - try writer.writeAll("\n-----END CERTIFICATE-----\n"); - } - - // Final encoding should not be larger than our initial size estimate - assert(buffer_size > arr.items.len, "Http loadCerts", .{ .estimate = buffer_size, .len = arr.items.len }); - - // Allocate exactly the size needed and copy the data - const result = try allocator.dupe(u8, arr.items); - // Free the original oversized allocation - arr.deinit(allocator); - - return .{ - .len = result.len, - .data = result.ptr, - .flags = 0, - }; -} - -// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is -// what Zig has), with lines wrapped at 64 characters and with a basic header -// and footer -const LineWriter = struct { - col: usize = 0, - inner: std.ArrayList(u8).Writer, - - pub fn writeAll(self: *LineWriter, data: []const u8) !void { - var writer = self.inner; - - var col = self.col; - const len = 64 - col; - - var remain = data; - if (remain.len > len) { - col = 0; - try writer.writeAll(data[0..len]); - try writer.writeByte('\n'); - remain = data[len..]; - } - - while (remain.len > 64) { - try writer.writeAll(remain[0..64]); - try writer.writeByte('\n'); - remain = data[len..]; - } - try writer.writeAll(remain); - self.col = col + remain.len; - } -}; - -fn debugCallback(_: *c.CURL, msg_type: c.curl_infotype, raw: [*c]u8, len: usize, _: *anyopaque) callconv(.c) void { - const data = raw[0..len]; - switch (msg_type) { - c.CURLINFO_TEXT => std.debug.print("libcurl [text]: {s}\n", .{data}), - c.CURLINFO_HEADER_OUT => std.debug.print("libcurl [req-h]: {s}\n", .{data}), - c.CURLINFO_HEADER_IN => std.debug.print("libcurl [res-h]: {s}\n", .{data}), - // c.CURLINFO_DATA_IN => std.debug.print("libcurl [res-b]: {s}\n", .{data}), - else => std.debug.print("libcurl ?? {d}\n", .{msg_type}), - } -} - pub const Error = error{ UnsupportedProtocol, FailedInit, @@ -598,6 +272,532 @@ pub fn errorMCheck(code: c.CURLMcode) ErrorMulti!void { return fromMCode(code); } +pub const Method = enum(u8) { + GET = 0, + PUT = 1, + POST = 2, + DELETE = 3, + HEAD = 4, + OPTIONS = 5, + PATCH = 6, + PROPFIND = 7, +}; + +pub const Header = struct { + name: []const u8, + value: []const u8, +}; + +pub const Headers = struct { + headers: ?*c.curl_slist, + cookies: ?[*c]const u8, + + pub fn init(user_agent: [:0]const u8) !Headers { + const header_list = c.curl_slist_append(null, user_agent); + if (header_list == null) { + return error.OutOfMemory; + } + return .{ .headers = header_list, .cookies = null }; + } + + pub fn deinit(self: *const Headers) void { + if (self.headers) |hdr| { + c.curl_slist_free_all(hdr); + } + } + + pub fn add(self: *Headers, header: [*c]const u8) !void { + // Copies the value + const updated_headers = c.curl_slist_append(self.headers, header); + if (updated_headers == null) { + return error.OutOfMemory; + } + + self.headers = updated_headers; + } + + pub fn parseHeader(header_str: []const u8) ?Header { + const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null; + + const name = std.mem.trim(u8, header_str[0..colon_pos], " \t"); + const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t"); + + return .{ .name = name, .value = value }; + } + + pub fn iterator(self: *Headers) Iterator { + return .{ + .header = self.headers, + .cookies = self.cookies, + }; + } + + const Iterator = struct { + header: [*c]c.curl_slist, + cookies: ?[*c]const u8, + + pub fn next(self: *Iterator) ?Header { + const h = self.header orelse { + const cookies = self.cookies orelse return null; + self.cookies = null; + return .{ .name = "Cookie", .value = std.mem.span(@as([*:0]const u8, cookies)) }; + }; + + self.header = h.*.next; + return parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data)))); + } + }; +}; + +// In normal cases, the header iterator comes from the curl linked list. +// But it's also possible to inject a response, via `transfer.fulfill`. In that +// case, the resposne headers are a list, []const Http.Header. +// This union, is an iterator that exposes the same API for either case. +pub const HeaderIterator = union(enum) { + curl: CurlHeaderIterator, + list: ListHeaderIterator, + + pub fn next(self: *HeaderIterator) ?Header { + switch (self.*) { + inline else => |*it| return it.next(), + } + } + + const CurlHeaderIterator = struct { + conn: *const Connection, + prev: ?*c.curl_header = null, + + pub fn next(self: *CurlHeaderIterator) ?Header { + const h = c.curl_easy_nextheader(self.conn.easy, c.CURLH_HEADER, -1, self.prev) orelse return null; + self.prev = h; + + const header = h.*; + return .{ + .name = std.mem.span(header.name), + .value = std.mem.span(header.value), + }; + } + }; + + const ListHeaderIterator = struct { + index: usize = 0, + list: []const Header, + + pub fn next(self: *ListHeaderIterator) ?Header { + const idx = self.index; + if (idx == self.list.len) { + return null; + } + self.index = idx + 1; + return self.list[idx]; + } + }; +}; + +pub const HeaderValue = struct { + value: []const u8, + amount: usize, +}; + +pub const AuthChallenge = struct { + status: u16, + source: enum { server, proxy }, + scheme: enum { basic, digest }, + realm: []const u8, + + pub fn parse(status: u16, header: []const u8) !AuthChallenge { + var ac: AuthChallenge = .{ + .status = status, + .source = undefined, + .realm = "TODO", // TODO parser and set realm + .scheme = undefined, + }; + + const sep = std.mem.indexOfPos(u8, header, 0, ": ") orelse return error.InvalidHeader; + const hname = header[0..sep]; + const hvalue = header[sep + 2 ..]; + + if (std.ascii.eqlIgnoreCase("WWW-Authenticate", hname)) { + ac.source = .server; + } else if (std.ascii.eqlIgnoreCase("Proxy-Authenticate", hname)) { + ac.source = .proxy; + } else { + return error.InvalidAuthChallenge; + } + + const pos = std.mem.indexOfPos(u8, std.mem.trim(u8, hvalue, std.ascii.whitespace[0..]), 0, " ") orelse hvalue.len; + const _scheme = hvalue[0..pos]; + if (std.ascii.eqlIgnoreCase(_scheme, "basic")) { + ac.scheme = .basic; + } else if (std.ascii.eqlIgnoreCase(_scheme, "digest")) { + ac.scheme = .digest; + } else { + return error.UnknownAuthChallengeScheme; + } + + return ac; + } +}; + +pub const ResponseHead = struct { + pub const MAX_CONTENT_TYPE_LEN = 64; + + status: u16, + url: [*c]const u8, + redirect_count: u32, + _content_type_len: usize = 0, + _content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined, + // this is normally an empty list, but if the response is being injected + // than it'll be populated. It isn't meant to be used directly, but should + // be used through the transfer.responseHeaderIterator() which abstracts + // whether the headers are from a live curl easy handle, or injected. + _injected_headers: []const Header = &.{}, + + pub fn contentType(self: *ResponseHead) ?[]u8 { + if (self._content_type_len == 0) { + return null; + } + return self._content_type[0..self._content_type_len]; + } +}; + +pub fn globalInit() Error!void { + try errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL)); +} + +pub fn globalDeinit() void { + c.curl_global_cleanup(); +} + +pub const Connection = struct { + easy: *c.CURL, + + pub fn init( + ca_blob_: ?c.curl_blob, + config: *const Config, + ) !Connection { + const easy = c.curl_easy_init() orelse return error.FailedToInitializeEasy; + errdefer _ = c.curl_easy_cleanup(easy); + + // timeouts + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_TIMEOUT_MS, @as(c_long, @intCast(config.httpTimeout())))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CONNECTTIMEOUT_MS, @as(c_long, @intCast(config.httpConnectTimeout())))); + + // redirect behavior + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_MAXREDIRS, @as(c_long, @intCast(config.httpMaxRedirects())))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_FOLLOWLOCATION, @as(c_long, 2))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_REDIR_PROTOCOLS_STR, "HTTP,HTTPS")); // remove FTP and FTPS from the default + + // proxy + const http_proxy = config.httpProxy(); + if (http_proxy) |proxy| { + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY, proxy.ptr)); + } + + // tls + if (ca_blob_) |ca_blob| { + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CAINFO_BLOB, ca_blob)); + if (http_proxy != null) { + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_CAINFO_BLOB, ca_blob)); + } + } else { + assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{}); + + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYHOST, @as(c_long, 0))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYPEER, @as(c_long, 0))); + + if (http_proxy != null) { + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYHOST, @as(c_long, 0))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYPEER, @as(c_long, 0))); + } + } + + // compression, don't remove this. CloudFront will send gzip content + // even if we don't support it, and then it won't be decompressed. + // empty string means: use whatever's available + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_ACCEPT_ENCODING, "")); + + // debug + if (comptime ENABLE_DEBUG) { + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_VERBOSE, @as(c_long, 1))); + + // Sometimes the default debug output hides some useful data. You can + // uncomment the following line (BUT KEEP THE LIVE ABOVE AS-IS), to + // get more control over the data (specifically, the `CURLINFO_TEXT` + // can include useful data). + + // try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_DEBUGFUNCTION, debugCallback)); + } + + return .{ + .easy = easy, + }; + } + + pub fn deinit(self: *const Connection) void { + c.curl_easy_cleanup(self.easy); + } + + pub fn setURL(self: *const Connection, url: [:0]const u8) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_URL, url.ptr)); + } + + // a libcurl request has 2 methods. The first is the method that + // controls how libcurl behaves. This specifically influences how redirects + // are handled. For example, if you do a POST and get a 301, libcurl will + // change that to a GET. But if you do a POST and get a 308, libcurl will + // keep the POST (and re-send the body). + // The second method is the actual string that's included in the request + // headers. + // These two methods can be different - you can tell curl to behave as though + // you made a GET, but include "POST" in the request header. + // + // Here, we're only concerned about the 2nd method. If we want, we'll set + // the first one based on whether or not we have a body. + // + // It's important that, for each use of this connection, we set the 2nd + // method. Else, if we make a HEAD request and re-use the connection, but + // DON'T reset this, it'll keep making HEAD requests. + // (I don't know if it's as important to reset the 1st method, or if libcurl + // can infer that based on the presence of the body, but we also reset it + // to be safe); + pub fn setMethod(self: *const Connection, method: Method) !void { + const easy = self.easy; + const m: [:0]const u8 = switch (method) { + .GET => "GET", + .POST => "POST", + .PUT => "PUT", + .DELETE => "DELETE", + .HEAD => "HEAD", + .OPTIONS => "OPTIONS", + .PATCH => "PATCH", + .PROPFIND => "PROPFIND", + }; + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, m.ptr)); + } + + pub fn setBody(self: *const Connection, body: []const u8) !void { + const easy = self.easy; + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPPOST, @as(c_long, 1))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(body.len)))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_COPYPOSTFIELDS, body.ptr)); + } + + pub fn setGetMode(self: *const Connection) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_HTTPGET, @as(c_long, 1))); + } + + pub fn setHeaders(self: *const Connection, headers: *Headers) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_HTTPHEADER, headers.headers)); + } + + pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_COOKIE, cookies)); + } + + pub fn setPrivate(self: *const Connection, ptr: *anyopaque) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_PRIVATE, ptr)); + } + + pub fn setProxyCredentials(self: *const Connection, creds: [:0]const u8) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_PROXYUSERPWD, creds.ptr)); + } + + pub fn setCallbacks( + self: *const Connection, + header_cb: *const fn ([*]const u8, usize, usize, *anyopaque) callconv(.c) usize, + data_cb: *const fn ([*]const u8, usize, usize, *anyopaque) callconv(.c) isize, + ) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_HEADERDATA, self.easy)); + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_HEADERFUNCTION, header_cb)); + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_WRITEDATA, self.easy)); + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_WRITEFUNCTION, data_cb)); + } + + pub fn setProxy(self: *const Connection, proxy: ?[*:0]const u8) !void { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_PROXY, proxy)); + } + + pub fn setTlsVerify(self: *const Connection, verify: bool, use_proxy: bool) !void { + const host_val: c_long = if (verify) 2 else 0; + const peer_val: c_long = if (verify) 1 else 0; + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_SSL_VERIFYHOST, host_val)); + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_SSL_VERIFYPEER, peer_val)); + if (use_proxy) { + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_PROXY_SSL_VERIFYHOST, host_val)); + try errorCheck(c.curl_easy_setopt(self.easy, c.CURLOPT_PROXY_SSL_VERIFYPEER, peer_val)); + } + } + + pub fn getEffectiveUrl(self: *const Connection) ![*c]const u8 { + var url: [*c]u8 = undefined; + try errorCheck(c.curl_easy_getinfo(self.easy, c.CURLINFO_EFFECTIVE_URL, &url)); + return url; + } + + pub fn getResponseCode(self: *const Connection) !u16 { + var status: c_long = undefined; + try errorCheck(c.curl_easy_getinfo(self.easy, c.CURLINFO_RESPONSE_CODE, &status)); + if (status < 0 or status > std.math.maxInt(u16)) { + return 0; + } + return @intCast(status); + } + + pub fn getRedirectCount(self: *const Connection) !u32 { + var count: c_long = undefined; + try errorCheck(c.curl_easy_getinfo(self.easy, c.CURLINFO_REDIRECT_COUNT, &count)); + return @intCast(count); + } + + pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue { + var hdr: [*c]c.curl_header = null; + const result = c.curl_easy_header(self.easy, name, index, c.CURLH_HEADER, -1, &hdr); + if (result == c.CURLE_OK) { + return .{ + .amount = hdr.*.amount, + .value = std.mem.span(hdr.*.value), + }; + } + + if (result == c.CURLE_FAILED_INIT) { + // seems to be what it returns if the header isn't found + return null; + } + log.err(.http, "get response header", .{ + .name = name, + .err = fromCode(result), + }); + return null; + } + + pub fn getPrivate(self: *const Connection) !*anyopaque { + var private: *anyopaque = undefined; + try errorCheck(c.curl_easy_getinfo(self.easy, c.CURLINFO_PRIVATE, &private)); + return private; + } + + // These are headers that may not be send to the users for inteception. + pub fn secretHeaders(_: *const Connection, headers: *Headers, http_headers: *const Config.HttpHeaders) !void { + if (http_headers.proxy_bearer_header) |hdr| { + try headers.add(hdr); + } + } + + pub fn request(self: *const Connection, http_headers: *const Config.HttpHeaders) !u16 { + var header_list = try Headers.init(http_headers.user_agent_header); + defer header_list.deinit(); + try self.secretHeaders(&header_list, http_headers); + try self.setHeaders(&header_list); + + // Add cookies. + if (header_list.cookies) |cookies| { + try self.setCookies(cookies); + } + + try errorCheck(c.curl_easy_perform(self.easy)); + return self.getResponseCode(); + } +}; + +// TODO: on BSD / Linux, we could just read the PEM file directly. +// This whole rescan + decode is really just needed for MacOS. On Linux +// bundle.rescan does find the .pem file(s) which could be in a few different +// places, so it's still useful, just not efficient. +pub fn loadCerts(allocator: Allocator) !c.curl_blob { + var bundle: std.crypto.Certificate.Bundle = .{}; + try bundle.rescan(allocator); + defer bundle.deinit(allocator); + + const bytes = bundle.bytes.items; + if (bytes.len == 0) { + log.warn(.app, "No system certificates", .{}); + return .{ + .len = 0, + .flags = 0, + .data = bytes.ptr, + }; + } + + const encoder = std.base64.standard.Encoder; + var arr: std.ArrayList(u8) = .empty; + + const encoded_size = encoder.calcSize(bytes.len); + const buffer_size = encoded_size + + (bundle.map.count() * 75) + // start / end per certificate + extra, just in case + (encoded_size / 64) // newline per 64 characters + ; + try arr.ensureTotalCapacity(allocator, buffer_size); + errdefer arr.deinit(allocator); + var writer = arr.writer(allocator); + + var it = bundle.map.valueIterator(); + while (it.next()) |index| { + const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*); + + try writer.writeAll("-----BEGIN CERTIFICATE-----\n"); + var line_writer = LineWriter{ .inner = writer }; + try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]); + try writer.writeAll("\n-----END CERTIFICATE-----\n"); + } + + // Final encoding should not be larger than our initial size estimate + assert(buffer_size > arr.items.len, "Http loadCerts", .{ .estimate = buffer_size, .len = arr.items.len }); + + // Allocate exactly the size needed and copy the data + const result = try allocator.dupe(u8, arr.items); + // Free the original oversized allocation + arr.deinit(allocator); + + return .{ + .len = result.len, + .data = result.ptr, + .flags = 0, + }; +} + +// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is +// what Zig has), with lines wrapped at 64 characters and with a basic header +// and footer +const LineWriter = struct { + col: usize = 0, + inner: std.ArrayList(u8).Writer, + + pub fn writeAll(self: *LineWriter, data: []const u8) !void { + var writer = self.inner; + + var col = self.col; + const len = 64 - col; + + var remain = data; + if (remain.len > len) { + col = 0; + try writer.writeAll(data[0..len]); + try writer.writeByte('\n'); + remain = data[len..]; + } + + while (remain.len > 64) { + try writer.writeAll(remain[0..64]); + try writer.writeByte('\n'); + remain = data[len..]; + } + try writer.writeAll(remain); + self.col = col + remain.len; + } +}; + +fn debugCallback(_: *c.CURL, msg_type: c.curl_infotype, raw: [*c]u8, len: usize, _: *anyopaque) callconv(.c) void { + const data = raw[0..len]; + switch (msg_type) { + c.CURLINFO_TEXT => std.debug.print("libcurl [text]: {s}\n", .{data}), + c.CURLINFO_HEADER_OUT => std.debug.print("libcurl [req-h]: {s}\n", .{data}), + c.CURLINFO_HEADER_IN => std.debug.print("libcurl [res-h]: {s}\n", .{data}), + // c.CURLINFO_DATA_IN => std.debug.print("libcurl [res-b]: {s}\n", .{data}), + else => std.debug.print("libcurl ?? {d}\n", .{msg_type}), + } +} + // Zig is in a weird backend transition right now. Need to determine if // SIMD is even available. const backend_supports_vectors = switch (builtin.zig_backend) { @@ -996,6 +1196,7 @@ pub const WsConnection = struct { const CLOSE_PROTOCOL_ERROR = [_]u8{ 136, 2, 3, 234 }; //code: 1002 // "private-use" close codes must be from 4000-49999 const CLOSE_TIMEOUT = [_]u8{ 136, 2, 15, 160 }; // code: 4000 + socket: posix.socket_t, socket_flags: usize, reader: Reader(true), @@ -1279,139 +1480,6 @@ pub const WsConnection = struct { } }; -pub const ResponseHeader = struct { - pub const MAX_CONTENT_TYPE_LEN = 64; - - status: u16, - url: [*c]const u8, - redirect_count: u32, - _content_type_len: usize = 0, - _content_type: [MAX_CONTENT_TYPE_LEN]u8 = undefined, - // this is normally an empty list, but if the response is being injected - // than it'll be populated. It isn't meant to be used directly, but should - // be used through the transfer.responseHeaderIterator() which abstracts - // whether the headers are from a live curl easy handle, or injected. - _injected_headers: []const Header = &.{}, - - pub fn contentType(self: *ResponseHeader) ?[]u8 { - if (self._content_type_len == 0) { - return null; - } - return self._content_type[0..self._content_type_len]; - } -}; - -pub const CurlHeaderValue = struct { - value: []const u8, - amount: usize, -}; - -pub fn getResponseHeader(easy: *c.CURL, name: [:0]const u8, index: usize) ?CurlHeaderValue { - var hdr: [*c]c.curl_header = null; - const result = c.curl_easy_header(easy, name, index, c.CURLH_HEADER, -1, &hdr); - if (result == c.CURLE_OK) { - return .{ - .amount = hdr.*.amount, - .value = std.mem.span(hdr.*.value), - }; - } - - if (result == c.CURLE_FAILED_INIT) { - // seems to be what it returns if the header isn't found - return null; - } - log.err(.http, "get response header", .{ - .name = name, - .err = fromCode(result), - }); - return null; -} - -// In normal cases, the header iterator comes from the curl linked list. -// But it's also possible to inject a response, via `transfer.fulfill`. In that -// case, the resposne headers are a list, []const Http.Header. -// This union, is an iterator that exposes the same API for either case. -pub const HeaderIterator = union(enum) { - curl: CurlHeaderIterator, - list: ListHeaderIterator, - - pub fn next(self: *HeaderIterator) ?Header { - switch (self.*) { - inline else => |*it| return it.next(), - } - } - - const CurlHeaderIterator = struct { - easy: *c.CURL, - prev: ?*c.curl_header = null, - - pub fn next(self: *CurlHeaderIterator) ?Header { - const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null; - self.prev = h; - - const header = h.*; - return .{ - .name = std.mem.span(header.name), - .value = std.mem.span(header.value), - }; - } - }; - - const ListHeaderIterator = struct { - index: usize = 0, - list: []const Header, - - pub fn next(self: *ListHeaderIterator) ?Header { - const idx = self.index; - if (idx == self.list.len) { - return null; - } - self.index = idx + 1; - return self.list[idx]; - } - }; -}; - -pub const AuthChallenge = struct { - status: u16, - source: enum { server, proxy }, - scheme: enum { basic, digest }, - realm: []const u8, - - pub fn parse(status: u16, header: []const u8) !AuthChallenge { - var ac: AuthChallenge = .{ - .status = status, - .source = undefined, - .realm = "TODO", // TODO parser and set realm - .scheme = undefined, - }; - - const sep = std.mem.indexOfPos(u8, header, 0, ": ") orelse return error.InvalidHeader; - const hname = header[0..sep]; - const hvalue = header[sep + 2 ..]; - - if (std.ascii.eqlIgnoreCase("WWW-Authenticate", hname)) { - ac.source = .server; - } else if (std.ascii.eqlIgnoreCase("Proxy-Authenticate", hname)) { - ac.source = .proxy; - } else { - return error.InvalidAuthChallenge; - } - - const pos = std.mem.indexOfPos(u8, std.mem.trim(u8, hvalue, std.ascii.whitespace[0..]), 0, " ") orelse hvalue.len; - const _scheme = hvalue[0..pos]; - if (std.ascii.eqlIgnoreCase(_scheme, "basic")) { - ac.scheme = .basic; - } else if (std.ascii.eqlIgnoreCase(_scheme, "digest")) { - ac.scheme = .digest; - } else { - return error.UnknownAuthChallengeScheme; - } - - return ac; - } -}; - const testing = std.testing; test "mask" { diff --git a/src/http/Client.zig b/src/http/Client.zig index 7de71d85..d3498a01 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -22,7 +22,7 @@ const lp = @import("lightpanda"); const log = @import("../log.zig"); const builtin = @import("builtin"); -const Http = @import("Http.zig"); +const Net = @import("../Net.zig"); const Config = @import("../Config.zig"); const URL = @import("../browser/URL.zig"); const Notification = @import("../Notification.zig"); @@ -30,18 +30,20 @@ const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar; const Robots = @import("../browser/Robots.zig"); const RobotStore = Robots.RobotStore; -const c = Http.c; +const c = Net.c; const posix = std.posix; const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; -const errorCheck = Http.errorCheck; -const errorMCheck = Http.errorMCheck; - const IS_DEBUG = builtin.mode == .Debug; -const Method = Http.Method; +const errorCheck = Net.errorCheck; +const errorMCheck = Net.errorMCheck; + +const Method = Net.Method; +const ResponseHead = Net.ResponseHead; +const HeaderIterator = Net.HeaderIterator; // This is loosely tied to a browser Page. Loading all the , doing // XHR requests, and loading imports all happens through here. Sine the app @@ -186,14 +188,14 @@ pub fn deinit(self: *Client) void { self.allocator.destroy(self); } -pub fn newHeaders(self: *const Client) !Http.Headers { - return Http.Headers.init(self.config.http_headers.user_agent_header); +pub fn newHeaders(self: *const Client) !Net.Headers { + return Net.Headers.init(self.config.http_headers.user_agent_header); } pub fn abort(self: *Client) void { while (self.handles.in_use.first) |node| { const handle: *Handle = @fieldParentPtr("node", node); - var transfer = Transfer.fromEasy(handle.conn.easy) catch |err| { + var transfer = Transfer.fromConnection(&handle.conn) catch |err| { log.err(.http, "get private info", .{ .err = err, .source = "abort" }); continue; }; @@ -354,7 +356,7 @@ fn fetchRobotsThenProcessRequest(self: *Client, robots_url: [:0]const u8, req: R try entry.value_ptr.append(self.allocator, req); } -fn robotsHeaderCallback(transfer: *Http.Transfer) !bool { +fn robotsHeaderCallback(transfer: *Transfer) !bool { const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx)); if (transfer.response_header) |hdr| { @@ -369,7 +371,7 @@ fn robotsHeaderCallback(transfer: *Http.Transfer) !bool { return true; } -fn robotsDataCallback(transfer: *Http.Transfer, data: []const u8) !void { +fn robotsDataCallback(transfer: *Transfer, data: []const u8) !void { const ctx: *RobotsRequestContext = @ptrCast(@alignCast(transfer.ctx)); try ctx.buffer.appendSlice(ctx.client.allocator, data); } @@ -548,7 +550,7 @@ pub fn abortTransfer(self: *Client, transfer: *Transfer) void { } // For an intercepted request -pub fn fulfillTransfer(self: *Client, transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void { +pub fn fulfillTransfer(self: *Client, transfer: *Transfer, status: u16, headers: []const Net.Header, body: ?[]const u8) !void { if (comptime IS_DEBUG) { std.debug.assert(transfer._intercept_state != .not_intercepted); log.debug(.http, "filfull transfer", .{ .intercepted = self.intercepted }); @@ -627,7 +629,7 @@ pub fn changeProxy(self: *Client, proxy: [:0]const u8) !void { try self.ensureNoActiveConnection(); for (self.handles.handles) |*h| { - try errorCheck(c.curl_easy_setopt(h.conn.easy, c.CURLOPT_PROXY, proxy.ptr)); + try h.conn.setProxy(proxy.ptr); } self.use_proxy = true; } @@ -639,7 +641,7 @@ pub fn restoreOriginalProxy(self: *Client) !void { const proxy = if (self.http_proxy) |p| p.ptr else null; for (self.handles.handles) |*h| { - try errorCheck(c.curl_easy_setopt(h.conn.easy, c.CURLOPT_PROXY, proxy)); + try h.conn.setProxy(proxy); } self.use_proxy = proxy != null; } @@ -650,15 +652,7 @@ pub fn enableTlsVerify(self: *const Client) !void { // the command during navigate and Curl seems to accept it... for (self.handles.handles) |*h| { - const easy = h.conn.easy; - - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYHOST, @as(c_long, 2))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYPEER, @as(c_long, 1))); - - if (self.use_proxy) { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYHOST, @as(c_long, 2))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYPEER, @as(c_long, 1))); - } + try h.conn.setTlsVerify(true, self.use_proxy); } } @@ -668,21 +662,12 @@ pub fn disableTlsVerify(self: *const Client) !void { // the command during navigate and Curl seems to accept it... for (self.handles.handles) |*h| { - const easy = h.conn.easy; - - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYHOST, @as(c_long, 0))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYPEER, @as(c_long, 0))); - - if (self.use_proxy) { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYHOST, @as(c_long, 0))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_SSL_VERIFYPEER, @as(c_long, 0))); - } + try h.conn.setTlsVerify(false, self.use_proxy); } } fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) anyerror!void { - const conn = handle.conn; - const easy = conn.easy; + const conn = &handle.conn; const req = &transfer.req; { @@ -694,23 +679,23 @@ fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) anyerror!voi if (req.body) |b| { try conn.setBody(b); } else { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPGET, @as(c_long, 1))); + try conn.setGetMode(); } var header_list = req.headers; - try conn.secretHeaders(&header_list); // Add headers that must be hidden from intercepts - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); + try conn.secretHeaders(&header_list, &self.config.http_headers); // Add headers that must be hidden from intercepts + try conn.setHeaders(&header_list); // Add cookies. if (header_list.cookies) |cookies| { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_COOKIE, cookies)); + try conn.setCookies(cookies); } - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PRIVATE, transfer)); + try conn.setPrivate(transfer); // add credentials if (req.credentials) |creds| { - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXYUSERPWD, creds.ptr)); + try conn.setProxyCredentials(creds); } } @@ -719,11 +704,11 @@ fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) anyerror!voi // fails BEFORE `curl_multi_add_handle` suceeds, the we still need to do // cleanup. But if things fail after `curl_multi_add_handle`, we expect // perfom to pickup the failure and cleanup. - try errorMCheck(c.curl_multi_add_handle(self.multi, easy)); + try errorMCheck(c.curl_multi_add_handle(self.multi, conn.easy)); if (req.start_callback) |cb| { cb(transfer) catch |err| { - try errorMCheck(c.curl_multi_remove_handle(self.multi, easy)); + try errorMCheck(c.curl_multi_remove_handle(self.multi, conn.easy)); transfer.deinit(); return err; }; @@ -786,7 +771,8 @@ fn processMessages(self: *Client) !bool { } const easy = msg.easy_handle.?; - const transfer = try Transfer.fromEasy(easy); + const conn: Net.Connection = .{ .easy = easy }; + const transfer = try Transfer.fromConnection(&conn); // In case of auth challenge // TODO give a way to configure the number of auth retries. @@ -840,7 +826,7 @@ fn processMessages(self: *Client) !bool { // In case of request w/o data, we need to call the header done // callback now. if (!transfer._header_done_called) { - const proceed = transfer.headerDoneCallback(easy) catch |err| { + const proceed = transfer.headerDoneCallback(&conn) catch |err| { log.err(.http, "header_done_callback", .{ .err = err }); requestFailed(transfer, err, true); continue; @@ -951,7 +937,7 @@ const Handles = struct { // wraps a c.CURL (an easy handle) pub const Handle = struct { client: *Client, - conn: Http.Connection, + conn: Net.Connection, node: Handles.HandleList.Node, fn init( @@ -959,16 +945,11 @@ pub const Handle = struct { ca_blob: ?c.curl_blob, config: *const Config, ) !Handle { - const conn = try Http.Connection.init(ca_blob, config); + var conn = try Net.Connection.init(ca_blob, config); errdefer conn.deinit(); - const easy = conn.easy; - // callbacks - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HEADERDATA, easy)); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HEADERFUNCTION, Transfer.headerCallback)); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_WRITEDATA, easy)); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_WRITEFUNCTION, Transfer.dataCallback)); + try conn.setCallbacks(Transfer.headerCallback, Transfer.dataCallback); return .{ .node = .{}, @@ -988,7 +969,7 @@ pub const RequestCookie = struct { is_navigation: bool, origin: [:0]const u8, - pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Http.Headers) !void { + pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Net.Headers) !void { var arr: std.ArrayList(u8) = .{}; try self.jar.forRequest(url, arr.writer(temp), .{ .is_http = self.is_http, @@ -1007,7 +988,7 @@ pub const Request = struct { page_id: u32, method: Method, url: [:0]const u8, - headers: Http.Headers, + headers: Net.Headers, body: ?[]const u8 = null, cookie_jar: ?*CookieJar, resource_type: ResourceType, @@ -1053,7 +1034,7 @@ pub const Request = struct { }; }; -pub const AuthChallenge = @import("../Net.zig").AuthChallenge; +const AuthChallenge = Net.AuthChallenge; pub const Transfer = struct { arena: ArenaAllocator, @@ -1071,7 +1052,7 @@ pub const Transfer = struct { max_response_size: ?usize = null, // We'll store the response header here - response_header: ?ResponseHeader = null, + response_header: ?ResponseHead = null, // track if the header callbacks done have been called. _header_done_called: bool = false, @@ -1128,34 +1109,28 @@ pub const Transfer = struct { self.client.transfer_pool.destroy(self); } - fn buildResponseHeader(self: *Transfer, easy: *c.CURL) !void { + fn buildResponseHeader(self: *Transfer, conn: *const Net.Connection) !void { if (comptime IS_DEBUG) { std.debug.assert(self.response_header == null); } - var url: [*c]u8 = undefined; - try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_EFFECTIVE_URL, &url)); + const url = try conn.getEffectiveUrl(); - var status: c_long = undefined; - if (self._auth_challenge) |_| { - status = 407; - } else { - try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_RESPONSE_CODE, &status)); - } - - var redirect_count: c_long = undefined; - try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_REDIRECT_COUNT, &redirect_count)); + const status: u16 = if (self._auth_challenge != null) + 407 + else + try conn.getResponseCode(); self.response_header = .{ .url = url, - .status = @intCast(status), - .redirect_count = @intCast(redirect_count), + .status = status, + .redirect_count = try conn.getRedirectCount(), }; - if (getResponseHeader(easy, "content-type", 0)) |ct| { + if (conn.getResponseHeader("content-type", 0)) |ct| { var hdr = &self.response_header.?; const value = ct.value; - const len = @min(value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN); + const len = @min(value.len, ResponseHead.MAX_CONTENT_TYPE_LEN); hdr._content_type_len = len; @memcpy(hdr._content_type[0..len], value[0..len]); } @@ -1182,7 +1157,7 @@ pub const Transfer = struct { self.req.credentials = userpwd; } - pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const Http.Header) !void { + pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const Net.Header) !void { self.req.headers.deinit(); var buf: std.ArrayList(u8) = .empty; @@ -1261,7 +1236,7 @@ pub const Transfer = struct { // redirectionCookies manages cookies during redirections handled by Curl. // It sets the cookies from the current response to the cookie jar. // It also immediately sets cookies for the following request. - fn redirectionCookies(transfer: *Transfer, easy: *c.CURL) !void { + fn redirectionCookies(transfer: *Transfer, conn: *const Net.Connection) !void { const req = &transfer.req; const arena = transfer.arena.allocator(); @@ -1269,7 +1244,7 @@ pub const Transfer = struct { if (req.cookie_jar) |jar| { var i: usize = 0; while (true) { - const ct = getResponseHeader(easy, "set-cookie", i); + const ct = conn.getResponseHeader("set-cookie", i); if (ct == null) break; try jar.populateFromResponse(transfer.url, ct.?.value); i += 1; @@ -1278,12 +1253,11 @@ pub const Transfer = struct { } // set cookies for the following redirection's request. - const location = getResponseHeader(easy, "location", 0) orelse { + const location = conn.getResponseHeader("location", 0) orelse { return error.LocationNotFound; }; - var base_url: [*c]u8 = undefined; - try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_EFFECTIVE_URL, &base_url)); + const base_url = try conn.getEffectiveUrl(); const url = try URL.resolve(arena, std.mem.span(base_url), location.value, .{}); transfer.url = url; @@ -1297,23 +1271,23 @@ pub const Transfer = struct { .is_navigation = req.resource_type == .document, }); try cookies.append(arena, 0); //null terminate - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_COOKIE, @as([*c]const u8, @ptrCast(cookies.items.ptr)))); + try conn.setCookies(@ptrCast(cookies.items.ptr)); } } // headerDoneCallback is called once the headers have been read. // It can be called either on dataCallback or once the request for those // w/o body. - fn headerDoneCallback(transfer: *Transfer, easy: *c.CURL) !bool { + fn headerDoneCallback(transfer: *Transfer, conn: *const Net.Connection) !bool { lp.assert(transfer._header_done_called == false, "Transfer.headerDoneCallback", .{}); defer transfer._header_done_called = true; - try transfer.buildResponseHeader(easy); + try transfer.buildResponseHeader(conn); - if (getResponseHeader(easy, "content-type", 0)) |ct| { + if (conn.getResponseHeader("content-type", 0)) |ct| { var hdr = &transfer.response_header.?; const value = ct.value; - const len = @min(value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN); + const len = @min(value.len, ResponseHead.MAX_CONTENT_TYPE_LEN); hdr._content_type_len = len; @memcpy(hdr._content_type[0..len], value[0..len]); } @@ -1321,7 +1295,7 @@ pub const Transfer = struct { if (transfer.req.cookie_jar) |jar| { var i: usize = 0; while (true) { - const ct = getResponseHeader(easy, "set-cookie", i); + const ct = conn.getResponseHeader("set-cookie", i); if (ct == null) break; jar.populateFromResponse(transfer.url, ct.?.value) catch |err| { log.err(.http, "set cookie", .{ .err = err, .req = transfer }); @@ -1359,8 +1333,8 @@ pub const Transfer = struct { std.debug.assert(header_count == 1); } - const easy: *c.CURL = @ptrCast(@alignCast(data)); - var transfer = fromEasy(easy) catch |err| { + const conn: Net.Connection = .{ .easy = @ptrCast(@alignCast(data)) }; + var transfer = fromConnection(&conn) catch |err| { log.err(.http, "get private info", .{ .err = err, .source = "header callback" }); return 0; }; @@ -1463,7 +1437,7 @@ pub const Transfer = struct { if (transfer._redirecting) { // parse and set cookies for the redirection. - redirectionCookies(transfer, easy) catch |err| { + redirectionCookies(transfer, &conn) catch |err| { if (comptime IS_DEBUG) { log.debug(.http, "redirection cookies", .{ .err = err }); } @@ -1481,8 +1455,8 @@ pub const Transfer = struct { std.debug.assert(chunk_count == 1); } - const easy: *c.CURL = @ptrCast(@alignCast(data)); - var transfer = fromEasy(easy) catch |err| { + const conn: Net.Connection = .{ .easy = @ptrCast(@alignCast(data)) }; + var transfer = fromConnection(&conn) catch |err| { log.err(.http, "get private info", .{ .err = err, .source = "body callback" }); return c.CURL_WRITEFUNC_ERROR; }; @@ -1492,7 +1466,7 @@ pub const Transfer = struct { } if (!transfer._header_done_called) { - const proceed = transfer.headerDoneCallback(easy) catch |err| { + const proceed = transfer.headerDoneCallback(&conn) catch |err| { log.err(.http, "header_done_callback", .{ .err = err, .req = transfer }); return c.CURL_WRITEFUNC_ERROR; }; @@ -1532,7 +1506,7 @@ pub const Transfer = struct { if (self._handle) |handle| { // If we have a handle, than this is a real curl request and we // iterate through the header that curl maintains. - return .{ .curl = .{ .easy = handle.conn.easy } }; + return .{ .curl = .{ .conn = &handle.conn } }; } // If there's no handle, it either means this is being called before // the request is even being made (which would be a bug in the code) @@ -1541,14 +1515,18 @@ pub const Transfer = struct { return .{ .list = .{ .list = self.response_header.?._injected_headers } }; } - // pub because Page.printWaitAnalysis uses it - pub fn fromEasy(easy: *c.CURL) !*Transfer { - var private: *anyopaque = undefined; - try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_PRIVATE, &private)); + pub fn fromConnection(conn: *const Net.Connection) !*Transfer { + const private = try conn.getPrivate(); return @ptrCast(@alignCast(private)); } - pub fn fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void { + // pub because Page.printWaitAnalysis uses it + pub fn fromEasy(easy: *c.CURL) !*Transfer { + const conn: Net.Connection = .{ .easy = easy }; + return fromConnection(&conn); + } + + pub fn fulfill(transfer: *Transfer, status: u16, headers: []const Net.Header, body: ?[]const u8) !void { if (transfer._handle != null) { // should never happen, should have been intercepted/paused, and then // either continued, aborted or fulfilled once. @@ -1562,7 +1540,7 @@ pub const Transfer = struct { }; } - fn _fulfill(transfer: *Transfer, status: u16, headers: []const Http.Header, body: ?[]const u8) !void { + fn _fulfill(transfer: *Transfer, status: u16, headers: []const Net.Header, body: ?[]const u8) !void { const req = &transfer.req; if (req.start_callback) |cb| { try cb(transfer); @@ -1576,7 +1554,7 @@ pub const Transfer = struct { }; for (headers) |hdr| { if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { - const len = @min(hdr.value.len, ResponseHeader.MAX_CONTENT_TYPE_LEN); + const len = @min(hdr.value.len, ResponseHead.MAX_CONTENT_TYPE_LEN); @memcpy(transfer.response_header.?._content_type[0..len], hdr.value[0..len]); transfer.response_header.?._content_type_len = len; break; @@ -1607,7 +1585,7 @@ pub const Transfer = struct { if (self._handle) |handle| { // If we have a handle, than this is a normal request. We can get the // header value from the easy handle. - const cl = getResponseHeader(handle.conn.easy, "content-length", 0) orelse return null; + const cl = handle.conn.getResponseHeader("content-length", 0) orelse return null; return cl.value; } @@ -1625,11 +1603,3 @@ pub const Transfer = struct { return null; } }; - -pub const ResponseHeader = @import("../Net.zig").ResponseHeader; - -const HeaderIterator = Net.HeaderIterator; - -const Net = @import("../Net.zig"); -const CurlHeaderValue = Net.CurlHeaderValue; -const getResponseHeader = Net.getResponseHeader; diff --git a/src/http/Http.zig b/src/http/Http.zig index 6e629dd7..01ac6dd7 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -19,9 +19,9 @@ const std = @import("std"); const Net = @import("../Net.zig"); -pub const c = Net.c; +const c = Net.c; -pub const ENABLE_DEBUG = Net.ENABLE_DEBUG; +const ENABLE_DEBUG = Net.ENABLE_DEBUG; pub const Client = @import("Client.zig"); pub const Transfer = Client.Transfer; @@ -29,11 +29,6 @@ pub const Method = Net.Method; pub const Header = Net.Header; pub const Headers = Net.Headers; -pub const Connection = Net.Connection; - -pub const errorCheck = Net.errorCheck; -pub const errorMCheck = Net.errorMCheck; - const Config = @import("../Config.zig"); const RobotStore = @import("../browser/Robots.zig").RobotStore; @@ -91,6 +86,6 @@ pub fn createClient(self: *Http, allocator: Allocator) !*Client { return Client.init(allocator, self.ca_blob, self.robot_store, self.config); } -pub fn newConnection(self: *Http) !Connection { - return Connection.init(self.ca_blob, self.config); +pub fn newConnection(self: *Http) !Net.Connection { + return Net.Connection.init(self.ca_blob, self.config); } diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index d05929fb..8d27f135 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -21,6 +21,7 @@ pub const LightPanda = struct { mutex: std.Thread.Mutex, cond: Thread.Condition, connection: Http.Connection, + config: *const Config, pending: std.DoublyLinkedList, mem_pool: std.heap.MemoryPool(LightPandaEvent), @@ -40,6 +41,7 @@ pub const LightPanda = struct { .running = true, .allocator = allocator, .connection = connection, + .config = app.config, .mem_pool = std.heap.MemoryPool(LightPandaEvent).init(allocator), }; } @@ -109,7 +111,7 @@ pub const LightPanda = struct { } try self.connection.setBody(aw.written()); - const status = try self.connection.request(); + const status = try self.connection.request(&self.config.http_headers); if (status != 200) { log.warn(.telemetry, "server error", .{ .status = status });