mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-02-04 06:23:45 +00:00
Introduce common network thread
This commit is contained in:
@@ -27,14 +27,13 @@ const Platform = @import("browser/js/Platform.zig");
|
|||||||
const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
|
const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
|
||||||
|
|
||||||
pub const Http = @import("http/Http.zig");
|
pub const Http = @import("http/Http.zig");
|
||||||
pub const Network = Http.Network;
|
|
||||||
pub const ArenaPool = @import("ArenaPool.zig");
|
pub const ArenaPool = @import("ArenaPool.zig");
|
||||||
pub const Notification = @import("Notification.zig");
|
pub const Notification = @import("Notification.zig");
|
||||||
|
|
||||||
const App = @This();
|
const App = @This();
|
||||||
|
|
||||||
config: *const Config,
|
config: *const Config,
|
||||||
network: Network,
|
http: Http,
|
||||||
platform: Platform,
|
platform: Platform,
|
||||||
snapshot: Snapshot,
|
snapshot: Snapshot,
|
||||||
telemetry: Telemetry,
|
telemetry: Telemetry,
|
||||||
@@ -49,8 +48,8 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
|
|||||||
|
|
||||||
app.config = config;
|
app.config = config;
|
||||||
|
|
||||||
app.network = try Network.init(allocator, config);
|
app.http = try Http.init(allocator, config);
|
||||||
errdefer app.network.deinit();
|
errdefer app.http.deinit();
|
||||||
|
|
||||||
app.notification = try Notification.init(allocator, null);
|
app.notification = try Notification.init(allocator, null);
|
||||||
errdefer app.notification.deinit();
|
errdefer app.notification.deinit();
|
||||||
@@ -88,7 +87,7 @@ pub fn deinit(self: *App, allocator: Allocator) void {
|
|||||||
self.snapshot.deinit();
|
self.snapshot.deinit();
|
||||||
self.platform.deinit();
|
self.platform.deinit();
|
||||||
self.arena_pool.deinit();
|
self.arena_pool.deinit();
|
||||||
self.network.deinit();
|
self.http.deinit();
|
||||||
|
|
||||||
allocator.destroy(self);
|
allocator.destroy(self);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,8 @@ const log = @import("log.zig");
|
|||||||
const App = @import("App.zig");
|
const App = @import("App.zig");
|
||||||
const Config = @import("Config.zig");
|
const Config = @import("Config.zig");
|
||||||
const CDP = @import("cdp/cdp.zig").CDP;
|
const CDP = @import("cdp/cdp.zig").CDP;
|
||||||
const Http = App.Http;
|
const Http = @import("http/Http.zig");
|
||||||
|
const HttpClient = @import("http/Client.zig");
|
||||||
const ThreadPool = @import("ThreadPool.zig");
|
const ThreadPool = @import("ThreadPool.zig");
|
||||||
const LimitedAllocator = @import("LimitedAllocator.zig");
|
const LimitedAllocator = @import("LimitedAllocator.zig");
|
||||||
|
|
||||||
@@ -166,7 +167,7 @@ pub const Client = struct {
|
|||||||
|
|
||||||
allocator: Allocator,
|
allocator: Allocator,
|
||||||
app: *App,
|
app: *App,
|
||||||
http: Http,
|
http: *HttpClient,
|
||||||
json_version_response: []const u8,
|
json_version_response: []const u8,
|
||||||
reader: Reader(true),
|
reader: Reader(true),
|
||||||
socket: posix.socket_t,
|
socket: posix.socket_t,
|
||||||
@@ -205,7 +206,7 @@ pub const Client = struct {
|
|||||||
var reader = try Reader(true).init(allocator);
|
var reader = try Reader(true).init(allocator);
|
||||||
errdefer reader.deinit();
|
errdefer reader.deinit();
|
||||||
|
|
||||||
var http = try app.network.createHttp(allocator);
|
const http = try app.http.createClient(allocator);
|
||||||
errdefer http.deinit();
|
errdefer http.deinit();
|
||||||
|
|
||||||
return .{
|
return .{
|
||||||
@@ -233,25 +234,29 @@ pub const Client = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn run(self: *Client) void {
|
fn run(self: *Client) void {
|
||||||
var http = &self.http;
|
const http = self.http;
|
||||||
http.addCDPClient(.{
|
http.cdp_client = .{
|
||||||
.socket = self.socket,
|
.socket = self.socket,
|
||||||
.ctx = self,
|
.ctx = self,
|
||||||
.blocking_read_start = Client.blockingReadStart,
|
.blocking_read_start = Client.blockingReadStart,
|
||||||
.blocking_read = Client.blockingRead,
|
.blocking_read = Client.blockingRead,
|
||||||
.blocking_read_end = Client.blockingReadStop,
|
.blocking_read_end = Client.blockingReadStop,
|
||||||
});
|
};
|
||||||
defer http.removeCDPClient();
|
defer http.cdp_client = null;
|
||||||
|
|
||||||
self.httpLoop(http) catch |err| {
|
self.httpLoop(http) catch |err| {
|
||||||
log.err(.app, "CDP client loop", .{ .err = err });
|
log.err(.app, "CDP client loop", .{ .err = err });
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
fn httpLoop(self: *Client, http: anytype) !void {
|
fn httpLoop(self: *Client, http: *HttpClient) !void {
|
||||||
lp.assert(self.mode == .http, "Client.httpLoop invalid mode", .{});
|
lp.assert(self.mode == .http, "Client.httpLoop invalid mode", .{});
|
||||||
while (true) {
|
while (true) {
|
||||||
if (http.poll(self.timeout_ms) != .cdp_socket) {
|
const status = http.tick(self.timeout_ms) catch |err| {
|
||||||
|
log.err(.app, "http tick", .{ .err = err });
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if (status != .cdp_socket) {
|
||||||
log.info(.app, "CDP timeout", .{});
|
log.info(.app, "CDP timeout", .{});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -268,7 +273,7 @@ pub const Client = struct {
|
|||||||
return self.cdpLoop(http);
|
return self.cdpLoop(http);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cdpLoop(self: *Client, http: anytype) !void {
|
fn cdpLoop(self: *Client, http: *HttpClient) !void {
|
||||||
var cdp = &self.mode.cdp;
|
var cdp = &self.mode.cdp;
|
||||||
var last_message = timestamp(.monotonic);
|
var last_message = timestamp(.monotonic);
|
||||||
var ms_remaining = self.timeout_ms;
|
var ms_remaining = self.timeout_ms;
|
||||||
@@ -283,7 +288,11 @@ pub const Client = struct {
|
|||||||
ms_remaining = self.timeout_ms;
|
ms_remaining = self.timeout_ms;
|
||||||
},
|
},
|
||||||
.no_page => {
|
.no_page => {
|
||||||
if (http.poll(ms_remaining) != .cdp_socket) {
|
const status = http.tick(ms_remaining) catch |err| {
|
||||||
|
log.err(.app, "http tick", .{ .err = err });
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if (status != .cdp_socket) {
|
||||||
log.info(.app, "CDP timeout", .{});
|
log.info(.app, "CDP timeout", .{});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -518,7 +527,7 @@ pub const Client = struct {
|
|||||||
break :blk res;
|
break :blk res;
|
||||||
};
|
};
|
||||||
|
|
||||||
self.mode = .{ .cdp = try CDP.init(self.allocator, self.app, &self.http, self) };
|
self.mode = .{ .cdp = try CDP.init(self.allocator, self.app, self.http, self) };
|
||||||
return self.send(response);
|
return self.send(response);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,10 +24,10 @@ const ArenaAllocator = std.heap.ArenaAllocator;
|
|||||||
const js = @import("js/js.zig");
|
const js = @import("js/js.zig");
|
||||||
const log = @import("../log.zig");
|
const log = @import("../log.zig");
|
||||||
const App = @import("../App.zig");
|
const App = @import("../App.zig");
|
||||||
|
const Http = @import("../http/Http.zig");
|
||||||
|
const HttpClient = @import("../http/Client.zig");
|
||||||
|
|
||||||
const ArenaPool = App.ArenaPool;
|
const ArenaPool = App.ArenaPool;
|
||||||
const Http = App.Http;
|
|
||||||
const HttpClient = Http.Client;
|
|
||||||
const Notification = App.Notification;
|
const Notification = App.Notification;
|
||||||
|
|
||||||
const IS_DEBUG = @import("builtin").mode == .Debug;
|
const IS_DEBUG = @import("builtin").mode == .Debug;
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ const IS_DEBUG = builtin.mode == .Debug;
|
|||||||
const log = @import("../log.zig");
|
const log = @import("../log.zig");
|
||||||
|
|
||||||
const App = @import("../App.zig");
|
const App = @import("../App.zig");
|
||||||
|
const Http = @import("../http/Http.zig");
|
||||||
|
const Client = @import("../http/Client.zig");
|
||||||
const String = @import("../string.zig").String;
|
const String = @import("../string.zig").String;
|
||||||
|
|
||||||
const Mime = @import("Mime.zig");
|
const Mime = @import("Mime.zig");
|
||||||
@@ -59,7 +61,6 @@ const PageTransitionEvent = @import("webapi/event/PageTransitionEvent.zig");
|
|||||||
const NavigationKind = @import("webapi/navigation/root.zig").NavigationKind;
|
const NavigationKind = @import("webapi/navigation/root.zig").NavigationKind;
|
||||||
const KeyboardEvent = @import("webapi/event/KeyboardEvent.zig");
|
const KeyboardEvent = @import("webapi/event/KeyboardEvent.zig");
|
||||||
|
|
||||||
const Http = App.Http;
|
|
||||||
const ArenaPool = App.ArenaPool;
|
const ArenaPool = App.ArenaPool;
|
||||||
|
|
||||||
const timestamp = @import("../datetime.zig").timestamp;
|
const timestamp = @import("../datetime.zig").timestamp;
|
||||||
@@ -966,7 +967,7 @@ fn printWaitAnalysis(self: *Page) void {
|
|||||||
std.debug.print("\nactive requests: {d}\n", .{self._session.browser.http_client.active});
|
std.debug.print("\nactive requests: {d}\n", .{self._session.browser.http_client.active});
|
||||||
var n_ = self._session.browser.http_client.handles.in_use.first;
|
var n_ = self._session.browser.http_client.handles.in_use.first;
|
||||||
while (n_) |n| {
|
while (n_) |n| {
|
||||||
const handle: *Http.Client.Handle = @fieldParentPtr("node", n);
|
const handle: *Client.Handle = @fieldParentPtr("node", n);
|
||||||
const transfer = Http.Transfer.fromEasy(handle.conn.easy) catch |err| {
|
const transfer = Http.Transfer.fromEasy(handle.conn.easy) catch |err| {
|
||||||
std.debug.print(" - failed to load transfer: {any}\n", .{err});
|
std.debug.print(" - failed to load transfer: {any}\n", .{err});
|
||||||
break;
|
break;
|
||||||
@@ -3079,7 +3080,7 @@ const RequestCookieOpts = struct {
|
|||||||
is_http: bool = true,
|
is_http: bool = true,
|
||||||
is_navigation: bool = false,
|
is_navigation: bool = false,
|
||||||
};
|
};
|
||||||
pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) Http.Client.RequestCookie {
|
pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) Client.RequestCookie {
|
||||||
return .{
|
return .{
|
||||||
.jar = &self._session.cookie_jar,
|
.jar = &self._session.cookie_jar,
|
||||||
.origin = self.url,
|
.origin = self.url,
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ const URL = @import("URL.zig");
|
|||||||
const Page = @import("Page.zig");
|
const Page = @import("Page.zig");
|
||||||
const Browser = @import("Browser.zig");
|
const Browser = @import("Browser.zig");
|
||||||
const Http = @import("../http/Http.zig");
|
const Http = @import("../http/Http.zig");
|
||||||
|
const Client = @import("../http/Client.zig");
|
||||||
|
|
||||||
const Element = @import("webapi/Element.zig");
|
const Element = @import("webapi/Element.zig");
|
||||||
|
|
||||||
@@ -59,7 +60,7 @@ ready_scripts: std.DoublyLinkedList,
|
|||||||
|
|
||||||
shutdown: bool = false,
|
shutdown: bool = false,
|
||||||
|
|
||||||
client: *Http.Client,
|
client: *Client,
|
||||||
allocator: Allocator,
|
allocator: Allocator,
|
||||||
buffer_pool: BufferPool,
|
buffer_pool: BufferPool,
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ _pad: bool = false,
|
|||||||
pub const init: Navigator = .{};
|
pub const init: Navigator = .{};
|
||||||
|
|
||||||
pub fn getUserAgent(_: *const Navigator, page: *Page) []const u8 {
|
pub fn getUserAgent(_: *const Navigator, page: *Page) []const u8 {
|
||||||
return page._session.browser.app.network.user_agent;
|
return page._session.browser.app.http.user_agent;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn getAppName(_: *const Navigator) []const u8 {
|
pub fn getAppName(_: *const Navigator) []const u8 {
|
||||||
|
|||||||
@@ -26,9 +26,9 @@ const log = @import("../log.zig");
|
|||||||
const js = @import("../browser/js/js.zig");
|
const js = @import("../browser/js/js.zig");
|
||||||
|
|
||||||
const App = @import("../App.zig");
|
const App = @import("../App.zig");
|
||||||
const Http = App.Http;
|
|
||||||
const Browser = @import("../browser/Browser.zig");
|
const Browser = @import("../browser/Browser.zig");
|
||||||
const Session = @import("../browser/Session.zig");
|
const Session = @import("../browser/Session.zig");
|
||||||
|
const HttpClient = @import("../http/Client.zig");
|
||||||
const Page = @import("../browser/Page.zig");
|
const Page = @import("../browser/Page.zig");
|
||||||
const Incrementing = @import("../id.zig").Incrementing;
|
const Incrementing = @import("../id.zig").Incrementing;
|
||||||
const Notification = @import("../Notification.zig");
|
const Notification = @import("../Notification.zig");
|
||||||
@@ -79,8 +79,8 @@ pub fn CDPT(comptime TypeProvider: type) type {
|
|||||||
|
|
||||||
const Self = @This();
|
const Self = @This();
|
||||||
|
|
||||||
pub fn init(allocator: Allocator, app: *App, http: *Http, client: TypeProvider.Client) !Self {
|
pub fn init(allocator: Allocator, app: *App, http: *HttpClient, client: TypeProvider.Client) !Self {
|
||||||
const browser = try Browser.init(allocator, app, http.client);
|
const browser = try Browser.init(allocator, app, http);
|
||||||
errdefer browser.deinit();
|
errdefer browser.deinit();
|
||||||
|
|
||||||
return .{
|
return .{
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ const TestContext = struct {
|
|||||||
self.client = Client.init(self.arena.allocator());
|
self.client = Client.init(self.arena.allocator());
|
||||||
// Don't use the arena here. We want to detect leaks in CDP.
|
// Don't use the arena here. We want to detect leaks in CDP.
|
||||||
// The arena is only for test-specific stuff
|
// The arena is only for test-specific stuff
|
||||||
self.cdp_ = TestCDP.init(std.testing.allocator, base.test_app, &base.test_http, &self.client.?) catch unreachable;
|
self.cdp_ = TestCDP.init(std.testing.allocator, base.test_app, base.test_http, &self.client.?) catch unreachable;
|
||||||
}
|
}
|
||||||
return &self.cdp_.?;
|
return &self.cdp_.?;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,71 +17,77 @@
|
|||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
const lp = @import("lightpanda");
|
const Allocator = std.mem.Allocator;
|
||||||
const Config = @import("../Config.zig");
|
const ArenaAllocator = std.heap.ArenaAllocator;
|
||||||
|
|
||||||
pub const c = @cImport({
|
pub const c = @cImport({
|
||||||
@cInclude("curl/curl.h");
|
@cInclude("curl/curl.h");
|
||||||
});
|
});
|
||||||
|
|
||||||
pub const ENABLE_DEBUG = false;
|
const Client = @import("Client.zig");
|
||||||
pub const Client = @import("Client.zig");
|
const lp = @import("lightpanda");
|
||||||
pub const Transfer = Client.Transfer;
|
const Config = @import("../Config.zig");
|
||||||
|
|
||||||
const log = @import("../log.zig");
|
const log = @import("../log.zig");
|
||||||
const errors = @import("errors.zig");
|
const errors = @import("errors.zig");
|
||||||
|
pub const Transfer = Client.Transfer;
|
||||||
|
|
||||||
const Allocator = std.mem.Allocator;
|
pub const ENABLE_DEBUG = false;
|
||||||
const ArenaAllocator = std.heap.ArenaAllocator;
|
|
||||||
|
|
||||||
// Client.zig does the bulk of the work and is loosely tied to a browser Page.
|
|
||||||
// But we still need something above Client.zig for the "utility" http stuff
|
|
||||||
// we need to do, like telemetry. The most important thing we want from this
|
|
||||||
// is to be able to share the ca_blob, which can be quite large - loading it
|
|
||||||
// once for all http connections is a win.
|
|
||||||
const Http = @This();
|
const Http = @This();
|
||||||
|
|
||||||
pub const Network = @import("Network.zig");
|
allocator: Allocator,
|
||||||
|
config: *const Config,
|
||||||
|
ca_blob: ?c.curl_blob,
|
||||||
|
user_agent: [:0]const u8,
|
||||||
|
proxy_bearer_header: ?[:0]const u8,
|
||||||
|
|
||||||
network: *Network,
|
pub fn init(allocator: Allocator, config: *const Config) !Http {
|
||||||
client: *Client,
|
try errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL));
|
||||||
|
errdefer c.curl_global_cleanup();
|
||||||
|
|
||||||
pub fn init(allocator: Allocator, network: *Network) !Http {
|
const user_agent = try config.userAgent(allocator);
|
||||||
var client = try Client.init(allocator, network.ca_blob, network.config);
|
errdefer allocator.free(user_agent);
|
||||||
errdefer client.deinit();
|
|
||||||
|
var proxy_bearer_header: ?[:0]const u8 = null;
|
||||||
|
if (config.proxyBearerToken()) |bt| {
|
||||||
|
proxy_bearer_header = try std.fmt.allocPrintSentinel(allocator, "Proxy-Authorization: Bearer {s}", .{bt}, 0);
|
||||||
|
}
|
||||||
|
errdefer if (proxy_bearer_header) |h| allocator.free(h);
|
||||||
|
|
||||||
|
var ca_blob: ?c.curl_blob = null;
|
||||||
|
if (config.tlsVerifyHost()) {
|
||||||
|
ca_blob = try loadCerts(allocator);
|
||||||
|
}
|
||||||
|
|
||||||
return .{
|
return .{
|
||||||
.network = network,
|
.allocator = allocator,
|
||||||
.client = client,
|
.config = config,
|
||||||
|
.ca_blob = ca_blob,
|
||||||
|
.user_agent = user_agent,
|
||||||
|
.proxy_bearer_header = proxy_bearer_header,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn deinit(self: *Http) void {
|
pub fn deinit(self: *Http) void {
|
||||||
self.client.deinit();
|
if (self.ca_blob) |ca_blob| {
|
||||||
|
const data: [*]u8 = @ptrCast(ca_blob.data);
|
||||||
|
self.allocator.free(data[0..ca_blob.len]);
|
||||||
|
}
|
||||||
|
if (self.proxy_bearer_header) |h| self.allocator.free(h);
|
||||||
|
self.allocator.free(self.user_agent);
|
||||||
|
c.curl_global_cleanup();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn poll(self: *Http, timeout_ms: u32) Client.PerformStatus {
|
pub fn createClient(self: *Http, allocator: Allocator) !*Client {
|
||||||
return self.client.tick(timeout_ms) catch |err| {
|
return Client.init(allocator, self.ca_blob, self.config);
|
||||||
log.err(.app, "http poll", .{ .err = err });
|
|
||||||
return .normal;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn addCDPClient(self: *Http, cdp_client: Client.CDPClient) void {
|
|
||||||
lp.assert(self.client.cdp_client == null, "Http addCDPClient existing", .{});
|
|
||||||
self.client.cdp_client = cdp_client;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn removeCDPClient(self: *Http) void {
|
|
||||||
self.client.cdp_client = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn newConnection(self: *Http) !Connection {
|
pub fn newConnection(self: *Http) !Connection {
|
||||||
return Connection.init(self.network.ca_blob, self.network.config, self.network.user_agent, self.network.proxy_bearer_header);
|
return Connection.init(self.ca_blob, self.config, self.user_agent, self.proxy_bearer_header);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn newHeaders(self: *const Http) Headers {
|
pub fn newHeaders(self: *const Http) !Headers {
|
||||||
return Headers.init(self.network.user_agent);
|
return Headers.init(self.user_agent);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub const Connection = struct {
|
pub const Connection = struct {
|
||||||
@@ -343,3 +349,90 @@ pub fn debugCallback(_: *c.CURL, msg_type: c.curl_infotype, raw: [*c]u8, len: us
|
|||||||
else => std.debug.print("libcurl ?? {d}\n", .{msg_type}),
|
else => std.debug.print("libcurl ?? {d}\n", .{msg_type}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: on BSD / Linux, we could just read the PEM file directly.
|
||||||
|
// This whole rescan + decode is really just needed for MacOS. On Linux
|
||||||
|
// bundle.rescan does find the .pem file(s) which could be in a few different
|
||||||
|
// places, so it's still useful, just not efficient.
|
||||||
|
fn loadCerts(allocator: Allocator) !c.curl_blob {
|
||||||
|
var bundle: std.crypto.Certificate.Bundle = .{};
|
||||||
|
try bundle.rescan(allocator);
|
||||||
|
defer bundle.deinit(allocator);
|
||||||
|
|
||||||
|
const bytes = bundle.bytes.items;
|
||||||
|
if (bytes.len == 0) {
|
||||||
|
log.warn(.app, "No system certificates", .{});
|
||||||
|
return .{
|
||||||
|
.len = 0,
|
||||||
|
.flags = 0,
|
||||||
|
.data = bytes.ptr,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const encoder = std.base64.standard.Encoder;
|
||||||
|
var arr: std.ArrayListUnmanaged(u8) = .empty;
|
||||||
|
|
||||||
|
const encoded_size = encoder.calcSize(bytes.len);
|
||||||
|
const buffer_size = encoded_size +
|
||||||
|
(bundle.map.count() * 75) + // start / end per certificate + extra, just in case
|
||||||
|
(encoded_size / 64) // newline per 64 characters
|
||||||
|
;
|
||||||
|
try arr.ensureTotalCapacity(allocator, buffer_size);
|
||||||
|
errdefer arr.deinit(allocator);
|
||||||
|
var writer = arr.writer(allocator);
|
||||||
|
|
||||||
|
var it = bundle.map.valueIterator();
|
||||||
|
while (it.next()) |index| {
|
||||||
|
const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*);
|
||||||
|
|
||||||
|
try writer.writeAll("-----BEGIN CERTIFICATE-----\n");
|
||||||
|
var line_writer = LineWriter{ .inner = writer };
|
||||||
|
try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]);
|
||||||
|
try writer.writeAll("\n-----END CERTIFICATE-----\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Final encoding should not be larger than our initial size estimate
|
||||||
|
lp.assert(buffer_size > arr.items.len, "Http loadCerts", .{ .estimate = buffer_size, .len = arr.items.len });
|
||||||
|
|
||||||
|
// Allocate exactly the size needed and copy the data
|
||||||
|
const result = try allocator.dupe(u8, arr.items);
|
||||||
|
// Free the original oversized allocation
|
||||||
|
arr.deinit(allocator);
|
||||||
|
|
||||||
|
return .{
|
||||||
|
.len = result.len,
|
||||||
|
.data = result.ptr,
|
||||||
|
.flags = 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is
|
||||||
|
// what Zig has), with lines wrapped at 64 characters and with a basic header
|
||||||
|
// and footer
|
||||||
|
const LineWriter = struct {
|
||||||
|
col: usize = 0,
|
||||||
|
inner: std.ArrayListUnmanaged(u8).Writer,
|
||||||
|
|
||||||
|
pub fn writeAll(self: *LineWriter, data: []const u8) !void {
|
||||||
|
var writer = self.inner;
|
||||||
|
|
||||||
|
var col = self.col;
|
||||||
|
const len = 64 - col;
|
||||||
|
|
||||||
|
var remain = data;
|
||||||
|
if (remain.len > len) {
|
||||||
|
col = 0;
|
||||||
|
try writer.writeAll(data[0..len]);
|
||||||
|
try writer.writeByte('\n');
|
||||||
|
remain = data[len..];
|
||||||
|
}
|
||||||
|
|
||||||
|
while (remain.len > 64) {
|
||||||
|
try writer.writeAll(remain[0..64]);
|
||||||
|
try writer.writeByte('\n');
|
||||||
|
remain = data[len..];
|
||||||
|
}
|
||||||
|
try writer.writeAll(remain);
|
||||||
|
self.col = col + remain.len;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|||||||
@@ -1,163 +0,0 @@
|
|||||||
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
|
||||||
//
|
|
||||||
// Francis Bouvier <francis@lightpanda.io>
|
|
||||||
// Pierre Tachoire <pierre@lightpanda.io>
|
|
||||||
//
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
|
||||||
// it under the terms of the GNU Affero General Public License as
|
|
||||||
// published by the Free Software Foundation, either version 3 of the
|
|
||||||
// License, or (at your option) any later version.
|
|
||||||
//
|
|
||||||
// This program is distributed in the hope that it will be useful,
|
|
||||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
// GNU Affero General Public License for more details.
|
|
||||||
//
|
|
||||||
// You should have received a copy of the GNU Affero General Public License
|
|
||||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
const std = @import("std");
|
|
||||||
const lp = @import("lightpanda");
|
|
||||||
const Allocator = std.mem.Allocator;
|
|
||||||
|
|
||||||
const log = @import("../log.zig");
|
|
||||||
const Config = @import("../Config.zig");
|
|
||||||
const Http = @import("Http.zig");
|
|
||||||
|
|
||||||
pub const c = Http.c;
|
|
||||||
|
|
||||||
const Network = @This();
|
|
||||||
|
|
||||||
allocator: Allocator,
|
|
||||||
config: *const Config,
|
|
||||||
ca_blob: ?c.curl_blob,
|
|
||||||
user_agent: [:0]const u8,
|
|
||||||
proxy_bearer_header: ?[:0]const u8,
|
|
||||||
|
|
||||||
pub fn init(allocator: Allocator, config: *const Config) !Network {
|
|
||||||
try Http.errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL));
|
|
||||||
errdefer c.curl_global_cleanup();
|
|
||||||
|
|
||||||
const user_agent = try config.userAgent(allocator);
|
|
||||||
errdefer allocator.free(user_agent);
|
|
||||||
|
|
||||||
var proxy_bearer_header: ?[:0]const u8 = null;
|
|
||||||
if (config.proxyBearerToken()) |bt| {
|
|
||||||
proxy_bearer_header = try std.fmt.allocPrintSentinel(allocator, "Proxy-Authorization: Bearer {s}", .{bt}, 0);
|
|
||||||
}
|
|
||||||
errdefer if (proxy_bearer_header) |h| allocator.free(h);
|
|
||||||
|
|
||||||
var ca_blob: ?c.curl_blob = null;
|
|
||||||
if (config.tlsVerifyHost()) {
|
|
||||||
ca_blob = try loadCerts(allocator);
|
|
||||||
}
|
|
||||||
|
|
||||||
return .{
|
|
||||||
.allocator = allocator,
|
|
||||||
.config = config,
|
|
||||||
.ca_blob = ca_blob,
|
|
||||||
.user_agent = user_agent,
|
|
||||||
.proxy_bearer_header = proxy_bearer_header,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn deinit(self: *Network) void {
|
|
||||||
if (self.ca_blob) |ca_blob| {
|
|
||||||
const data: [*]u8 = @ptrCast(ca_blob.data);
|
|
||||||
self.allocator.free(data[0..ca_blob.len]);
|
|
||||||
}
|
|
||||||
if (self.proxy_bearer_header) |h| self.allocator.free(h);
|
|
||||||
self.allocator.free(self.user_agent);
|
|
||||||
c.curl_global_cleanup();
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn createHttp(self: *Network, allocator: Allocator) !Http {
|
|
||||||
return Http.init(allocator, self);
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: on BSD / Linux, we could just read the PEM file directly.
|
|
||||||
// This whole rescan + decode is really just needed for MacOS. On Linux
|
|
||||||
// bundle.rescan does find the .pem file(s) which could be in a few different
|
|
||||||
// places, so it's still useful, just not efficient.
|
|
||||||
fn loadCerts(allocator: Allocator) !c.curl_blob {
|
|
||||||
var bundle: std.crypto.Certificate.Bundle = .{};
|
|
||||||
try bundle.rescan(allocator);
|
|
||||||
defer bundle.deinit(allocator);
|
|
||||||
|
|
||||||
const bytes = bundle.bytes.items;
|
|
||||||
if (bytes.len == 0) {
|
|
||||||
log.warn(.app, "No system certificates", .{});
|
|
||||||
return .{
|
|
||||||
.len = 0,
|
|
||||||
.flags = 0,
|
|
||||||
.data = bytes.ptr,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const encoder = std.base64.standard.Encoder;
|
|
||||||
var arr: std.ArrayListUnmanaged(u8) = .empty;
|
|
||||||
|
|
||||||
const encoded_size = encoder.calcSize(bytes.len);
|
|
||||||
const buffer_size = encoded_size +
|
|
||||||
(bundle.map.count() * 75) + // start / end per certificate + extra, just in case
|
|
||||||
(encoded_size / 64) // newline per 64 characters
|
|
||||||
;
|
|
||||||
try arr.ensureTotalCapacity(allocator, buffer_size);
|
|
||||||
errdefer arr.deinit(allocator);
|
|
||||||
var writer = arr.writer(allocator);
|
|
||||||
|
|
||||||
var it = bundle.map.valueIterator();
|
|
||||||
while (it.next()) |index| {
|
|
||||||
const cert = try std.crypto.Certificate.der.Element.parse(bytes, index.*);
|
|
||||||
|
|
||||||
try writer.writeAll("-----BEGIN CERTIFICATE-----\n");
|
|
||||||
var line_writer = LineWriter{ .inner = writer };
|
|
||||||
try encoder.encodeWriter(&line_writer, bytes[index.*..cert.slice.end]);
|
|
||||||
try writer.writeAll("\n-----END CERTIFICATE-----\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Final encoding should not be larger than our initial size estimate
|
|
||||||
lp.assert(buffer_size > arr.items.len, "Network loadCerts", .{ .estimate = buffer_size, .len = arr.items.len });
|
|
||||||
|
|
||||||
// Allocate exactly the size needed and copy the data
|
|
||||||
const result = try allocator.dupe(u8, arr.items);
|
|
||||||
// Free the original oversized allocation
|
|
||||||
arr.deinit(allocator);
|
|
||||||
|
|
||||||
return .{
|
|
||||||
.len = result.len,
|
|
||||||
.data = result.ptr,
|
|
||||||
.flags = 0,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is
|
|
||||||
// what Zig has), with lines wrapped at 64 characters and with a basic header
|
|
||||||
// and footer
|
|
||||||
const LineWriter = struct {
|
|
||||||
col: usize = 0,
|
|
||||||
inner: std.ArrayListUnmanaged(u8).Writer,
|
|
||||||
|
|
||||||
pub fn writeAll(self: *LineWriter, data: []const u8) !void {
|
|
||||||
var writer = self.inner;
|
|
||||||
|
|
||||||
var col = self.col;
|
|
||||||
const len = 64 - col;
|
|
||||||
|
|
||||||
var remain = data;
|
|
||||||
if (remain.len > len) {
|
|
||||||
col = 0;
|
|
||||||
try writer.writeAll(data[0..len]);
|
|
||||||
try writer.writeByte('\n');
|
|
||||||
remain = data[len..];
|
|
||||||
}
|
|
||||||
|
|
||||||
while (remain.len > 64) {
|
|
||||||
try writer.writeAll(remain[0..64]);
|
|
||||||
try writer.writeByte('\n');
|
|
||||||
remain = data[len..];
|
|
||||||
}
|
|
||||||
try writer.writeAll(remain);
|
|
||||||
self.col = col + remain.len;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
@@ -39,10 +39,10 @@ pub const FetchOpts = struct {
|
|||||||
writer: ?*std.Io.Writer = null,
|
writer: ?*std.Io.Writer = null,
|
||||||
};
|
};
|
||||||
pub fn fetch(allocator: std.mem.Allocator, app: *App, url: [:0]const u8, opts: FetchOpts) !void {
|
pub fn fetch(allocator: std.mem.Allocator, app: *App, url: [:0]const u8, opts: FetchOpts) !void {
|
||||||
var http = try app.network.createHttp(allocator);
|
const http = try app.http.createClient(allocator);
|
||||||
defer http.deinit();
|
defer http.deinit();
|
||||||
|
|
||||||
var browser = try Browser.init(allocator, app, http.client);
|
var browser = try Browser.init(allocator, app, http);
|
||||||
defer browser.deinit();
|
defer browser.deinit();
|
||||||
|
|
||||||
var session = try browser.newSession();
|
var session = try browser.newSession();
|
||||||
|
|||||||
@@ -47,10 +47,10 @@ pub fn main() !void {
|
|||||||
var test_arena = std.heap.ArenaAllocator.init(allocator);
|
var test_arena = std.heap.ArenaAllocator.init(allocator);
|
||||||
defer test_arena.deinit();
|
defer test_arena.deinit();
|
||||||
|
|
||||||
var http = try app.network.createHttp(allocator);
|
const http = try app.http.createClient(allocator);
|
||||||
defer http.deinit();
|
defer http.deinit();
|
||||||
|
|
||||||
var browser = try lp.Browser.init(allocator, app, http.client);
|
var browser = try lp.Browser.init(allocator, app, http);
|
||||||
defer browser.deinit();
|
defer browser.deinit();
|
||||||
|
|
||||||
const session = try browser.newSession();
|
const session = try browser.newSession();
|
||||||
|
|||||||
@@ -70,10 +70,10 @@ pub fn main() !void {
|
|||||||
var app = try lp.App.init(allocator, &config);
|
var app = try lp.App.init(allocator, &config);
|
||||||
defer app.deinit(allocator);
|
defer app.deinit(allocator);
|
||||||
|
|
||||||
var http = try app.network.createHttp(allocator);
|
const http = try app.http.createClient(allocator);
|
||||||
defer http.deinit();
|
defer http.deinit();
|
||||||
|
|
||||||
var browser = try lp.Browser.init(allocator, app, http.client);
|
var browser = try lp.Browser.init(allocator, app, http);
|
||||||
defer browser.deinit();
|
defer browser.deinit();
|
||||||
|
|
||||||
// An arena for running each tests. Is reset after every test.
|
// An arena for running each tests. Is reset after every test.
|
||||||
|
|||||||
@@ -20,16 +20,12 @@ pub const LightPanda = struct {
|
|||||||
allocator: Allocator,
|
allocator: Allocator,
|
||||||
mutex: std.Thread.Mutex,
|
mutex: std.Thread.Mutex,
|
||||||
cond: Thread.Condition,
|
cond: Thread.Condition,
|
||||||
http: Http,
|
|
||||||
connection: Http.Connection,
|
connection: Http.Connection,
|
||||||
pending: std.DoublyLinkedList,
|
pending: std.DoublyLinkedList,
|
||||||
mem_pool: std.heap.MemoryPool(LightPandaEvent),
|
mem_pool: std.heap.MemoryPool(LightPandaEvent),
|
||||||
|
|
||||||
pub fn init(allocator: Allocator, app: *App) !LightPanda {
|
pub fn init(allocator: Allocator, app: *App) !LightPanda {
|
||||||
var http = try app.network.createHttp(allocator);
|
const connection = try app.http.newConnection();
|
||||||
errdefer http.deinit();
|
|
||||||
|
|
||||||
const connection = try http.newConnection();
|
|
||||||
errdefer connection.deinit();
|
errdefer connection.deinit();
|
||||||
|
|
||||||
try connection.setURL(URL);
|
try connection.setURL(URL);
|
||||||
@@ -42,7 +38,6 @@ pub const LightPanda = struct {
|
|||||||
.thread = null,
|
.thread = null,
|
||||||
.running = true,
|
.running = true,
|
||||||
.allocator = allocator,
|
.allocator = allocator,
|
||||||
.http = http,
|
|
||||||
.connection = connection,
|
.connection = connection,
|
||||||
.mem_pool = std.heap.MemoryPool(LightPandaEvent).init(allocator),
|
.mem_pool = std.heap.MemoryPool(LightPandaEvent).init(allocator),
|
||||||
};
|
};
|
||||||
@@ -58,7 +53,6 @@ pub const LightPanda = struct {
|
|||||||
}
|
}
|
||||||
self.mem_pool.deinit();
|
self.mem_pool.deinit();
|
||||||
self.connection.deinit();
|
self.connection.deinit();
|
||||||
self.http.deinit();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn send(self: *LightPanda, iid: ?[]const u8, run_mode: Config.RunMode, raw_event: telemetry.Event) !void {
|
pub fn send(self: *LightPanda, iid: ?[]const u8, run_mode: Config.RunMode, raw_event: telemetry.Event) !void {
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ pub fn reset() void {
|
|||||||
const App = @import("App.zig");
|
const App = @import("App.zig");
|
||||||
const js = @import("browser/js/js.zig");
|
const js = @import("browser/js/js.zig");
|
||||||
const Config = @import("Config.zig");
|
const Config = @import("Config.zig");
|
||||||
|
const Client = @import("http/Client.zig");
|
||||||
const Page = @import("browser/Page.zig");
|
const Page = @import("browser/Page.zig");
|
||||||
const Browser = @import("browser/Browser.zig");
|
const Browser = @import("browser/Browser.zig");
|
||||||
const Session = @import("browser/Session.zig");
|
const Session = @import("browser/Session.zig");
|
||||||
@@ -333,7 +334,7 @@ fn isJsonValue(a: std.json.Value, b: std.json.Value) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub var test_app: *App = undefined;
|
pub var test_app: *App = undefined;
|
||||||
pub var test_http: App.Http = undefined;
|
pub var test_http: *Client = undefined;
|
||||||
pub var test_browser: Browser = undefined;
|
pub var test_browser: Browser = undefined;
|
||||||
pub var test_session: *Session = undefined;
|
pub var test_session: *Session = undefined;
|
||||||
|
|
||||||
@@ -468,10 +469,10 @@ test "tests:beforeAll" {
|
|||||||
test_app = try App.init(@import("root").tracking_allocator, &test_config);
|
test_app = try App.init(@import("root").tracking_allocator, &test_config);
|
||||||
errdefer test_app.deinit(@import("root").tracking_allocator);
|
errdefer test_app.deinit(@import("root").tracking_allocator);
|
||||||
|
|
||||||
test_http = try test_app.network.createHttp(@import("root").tracking_allocator);
|
test_http = try test_app.http.createClient(@import("root").tracking_allocator);
|
||||||
errdefer test_http.deinit();
|
errdefer test_http.deinit();
|
||||||
|
|
||||||
test_browser = try Browser.init(@import("root").tracking_allocator, test_app, test_http.client);
|
test_browser = try Browser.init(@import("root").tracking_allocator, test_app, test_http);
|
||||||
errdefer test_browser.deinit();
|
errdefer test_browser.deinit();
|
||||||
|
|
||||||
test_session = try test_browser.newSession();
|
test_session = try test_browser.newSession();
|
||||||
|
|||||||
Reference in New Issue
Block a user