From 0764a44e1d9ccedbc69b2d24122b6d26292b01c3 Mon Sep 17 00:00:00 2001 From: Nikolay Govorov Date: Mon, 26 Jan 2026 03:22:45 +0000 Subject: [PATCH] Centralizes configuration, eliminates unnecessary copying of config --- build.zig.zon | 4 +- src/App.zig | 41 +- src/Config.zig | 708 +++++++++++++++++++++++++++++++ src/browser/webapi/Navigator.zig | 2 +- src/http/Client.zig | 47 +- src/http/Http.zig | 80 ++-- src/lightpanda.zig | 1 + src/main.zig | 627 +-------------------------- src/main_legacy_test.zig | 16 +- src/main_wpt.zig | 15 +- src/telemetry/lightpanda.zig | 5 +- src/telemetry/telemetry.zig | 13 +- src/testing.zig | 19 +- 13 files changed, 836 insertions(+), 742 deletions(-) create mode 100644 src/Config.zig diff --git a/build.zig.zon b/build.zig.zon index 92506e1f..f66a040e 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -6,8 +6,8 @@ .minimum_zig_version = "0.15.2", .dependencies = .{ .v8 = .{ - .url = "https://github.com/lightpanda-io/zig-v8-fork/archive/v0.2.5.tar.gz", - .hash = "v8-0.0.0-xddH641NBAC3MqKV44YCkwvnUenhQyGlgJI8OScx0tlP", + .url = "https://github.com/lightpanda-io/zig-v8-fork/archive/v0.2.5.tar.gz", + .hash = "v8-0.0.0-xddH641NBAC3MqKV44YCkwvnUenhQyGlgJI8OScx0tlP", }, //.v8 = .{ .path = "../zig-v8-fork" }, .@"boringssl-zig" = .{ diff --git a/src/App.zig b/src/App.zig index fca528cd..376b79b6 100644 --- a/src/App.zig +++ b/src/App.zig @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2025 Lightpanda (Selecy SAS) +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) // // Francis Bouvier // Pierre Tachoire @@ -21,6 +21,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const log = @import("log.zig"); +const Config = @import("Config.zig"); const Snapshot = @import("browser/js/Snapshot.zig"); const Platform = @import("browser/js/Platform.zig"); const Telemetry = @import("telemetry/telemetry.zig").Telemetry; @@ -29,12 +30,10 @@ pub const Http = @import("http/Http.zig"); pub const ArenaPool = @import("ArenaPool.zig"); pub const Notification = @import("Notification.zig"); -// Container for global state / objects that various parts of the system -// might need. const App = @This(); http: Http, -config: Config, +config: *const Config, platform: Platform, snapshot: Snapshot, telemetry: Telemetry, @@ -44,26 +43,7 @@ app_dir_path: ?[]const u8, notification: *Notification, shutdown: bool = false, -pub const RunMode = enum { - help, - fetch, - serve, - version, -}; - -pub const Config = struct { - run_mode: RunMode, - tls_verify_host: bool = true, - http_proxy: ?[:0]const u8 = null, - proxy_bearer_token: ?[:0]const u8 = null, - http_timeout_ms: ?u31 = null, - http_connect_timeout_ms: ?u31 = null, - http_max_host_open: ?u8 = null, - http_max_concurrent: ?u8 = null, - user_agent: [:0]const u8, -}; - -pub fn init(allocator: Allocator, config: Config) !*App { +pub fn init(allocator: Allocator, config: *const Config) !*App { const app = try allocator.create(App); errdefer allocator.destroy(app); @@ -73,16 +53,7 @@ pub fn init(allocator: Allocator, config: Config) !*App { app.notification = try Notification.init(allocator, null); errdefer app.notification.deinit(); - app.http = try Http.init(allocator, .{ - .max_host_open = config.http_max_host_open orelse 4, - .max_concurrent = config.http_max_concurrent orelse 10, - .timeout_ms = config.http_timeout_ms orelse 5000, - .connect_timeout_ms = config.http_connect_timeout_ms orelse 0, - .http_proxy = config.http_proxy, - .tls_verify_host = config.tls_verify_host, - .proxy_bearer_token = config.proxy_bearer_token, - .user_agent = config.user_agent, - }); + app.http = try Http.init(allocator, config); errdefer app.http.deinit(); app.platform = try Platform.init(); @@ -93,7 +64,7 @@ pub fn init(allocator: Allocator, config: Config) !*App { app.app_dir_path = getAndMakeAppDir(allocator); - app.telemetry = try Telemetry.init(app, config.run_mode); + app.telemetry = try Telemetry.init(app, config.mode); errdefer app.telemetry.deinit(); try app.telemetry.register(app.notification); diff --git a/src/Config.zig b/src/Config.zig new file mode 100644 index 00000000..ed987214 --- /dev/null +++ b/src/Config.zig @@ -0,0 +1,708 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const builtin = @import("builtin"); +const Allocator = std.mem.Allocator; + +const log = @import("log.zig"); +const dump = @import("browser/dump.zig"); + +pub const RunMode = enum { + help, + fetch, + serve, + version, +}; + +mode: Mode, +exec_name: []const u8, + +const Config = @This(); + +pub fn tlsVerifyHost(self: *const Config) bool { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.tls_verify_host, + else => unreachable, + }; +} + +pub fn httpProxy(self: *const Config) ?[:0]const u8 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_proxy, + else => unreachable, + }; +} + +pub fn proxyBearerToken(self: *const Config) ?[:0]const u8 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.proxy_bearer_token, + else => unreachable, + }; +} + +pub fn httpMaxConcurrent(self: *const Config) u8 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_max_concurrent orelse 10, + else => unreachable, + }; +} + +pub fn httpMaxHostOpen(self: *const Config) u8 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_max_host_open orelse 4, + else => unreachable, + }; +} + +pub fn httpConnectTimeout(self: *const Config) u31 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_connect_timeout orelse 0, + else => unreachable, + }; +} + +pub fn httpTimeout(self: *const Config) u31 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_timeout orelse 5000, + else => unreachable, + }; +} + +pub fn httpMaxRedirects(_: *const Config) u8 { + return 10; +} + +pub fn logLevel(self: *const Config) ?log.Level { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.log_level, + else => unreachable, + }; +} + +pub fn logFormat(self: *const Config) ?log.Format { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.log_format, + else => unreachable, + }; +} + +pub fn logFilterScopes(self: *const Config) ?[]const log.Scope { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.log_filter_scopes, + else => unreachable, + }; +} + +pub fn userAgentSuffix(self: *const Config) ?[]const u8 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.user_agent_suffix, + else => unreachable, + }; +} + +pub fn userAgent(self: *const Config, allocator: Allocator) ![:0]const u8 { + const base = "User-Agent: Lightpanda/1.0"; + if (self.userAgentSuffix()) |suffix| { + return try std.fmt.allocPrintSentinel(allocator, "{s} {s}", .{ base, suffix }, 0); + } + return base; +} + +pub const Mode = union(RunMode) { + help: bool, // false when being printed because of an error + fetch: Fetch, + serve: Serve, + version: void, +}; + +pub const Serve = struct { + host: []const u8 = "127.0.0.1", + port: u16 = 9222, + timeout: u31 = 10, + max_connections: u16 = 16, + max_tabs_per_connection: u16 = 8, + max_memory_per_tab: u64 = 512 * 1024 * 1024, + max_pending_connections: u16 = 128, + common: Common = .{}, +}; + +pub const Fetch = struct { + url: [:0]const u8, + dump: bool = false, + common: Common = .{}, + withbase: bool = false, + strip: dump.Opts.Strip = .{}, +}; + +pub const Common = struct { + proxy_bearer_token: ?[:0]const u8 = null, + http_proxy: ?[:0]const u8 = null, + http_max_concurrent: ?u8 = null, + http_max_host_open: ?u8 = null, + http_timeout: ?u31 = null, + http_connect_timeout: ?u31 = null, + tls_verify_host: bool = true, + log_level: ?log.Level = null, + log_format: ?log.Format = null, + log_filter_scopes: ?[]log.Scope = null, + user_agent_suffix: ?[]const u8 = null, +}; + +pub fn printUsageAndExit(self: *const Config, success: bool) void { + // MAX_HELP_LEN| + const common_options = + \\ + \\--insecure_disable_tls_host_verification + \\ Disables host verification on all HTTP requests. This is an + \\ advanced option which should only be set if you understand + \\ and accept the risk of disabling host verification. + \\ + \\--http_proxy The HTTP proxy to use for all HTTP requests. + \\ A username:password can be included for basic authentication. + \\ Defaults to none. + \\ + \\--proxy_bearer_token + \\ The to send for bearer authentication with the proxy + \\ Proxy-Authorization: Bearer + \\ + \\--http_max_concurrent + \\ The maximum number of concurrent HTTP requests. + \\ Defaults to 10. + \\ + \\--http_max_host_open + \\ The maximum number of open connection to a given host:port. + \\ Defaults to 4. + \\ + \\--http_connect_timeout + \\ The time, in milliseconds, for establishing an HTTP connection + \\ before timing out. 0 means it never times out. + \\ Defaults to 0. + \\ + \\--http_timeout + \\ The maximum time, in milliseconds, the transfer is allowed + \\ to complete. 0 means it never times out. + \\ Defaults to 10000. + \\ + \\--log_level The log level: debug, info, warn, error or fatal. + \\ Defaults to + ++ (if (builtin.mode == .Debug) " info." else "warn.") ++ + \\ + \\ + \\--log_format The log format: pretty or logfmt. + \\ Defaults to + ++ (if (builtin.mode == .Debug) " pretty." else " logfmt.") ++ + \\ + \\ + \\--log_filter_scopes + \\ Filter out too verbose logs per scope: + \\ http, unknown_prop, event, ... + \\ + \\--user_agent_suffix + \\ Suffix to append to the Lightpanda/X.Y User-Agent + \\ + ; + + // MAX_HELP_LEN| + const usage = + \\usage: {s} command [options] [URL] + \\ + \\Command can be either 'fetch', 'serve' or 'help' + \\ + \\fetch command + \\Fetches the specified URL + \\Example: {s} fetch --dump https://lightpanda.io/ + \\ + \\Options: + \\--dump Dumps document to stdout. + \\ Defaults to false. + \\ + \\--strip_mode Comma separated list of tag groups to remove from dump + \\ the dump. e.g. --strip_mode js,css + \\ - "js" script and link[as=script, rel=preload] + \\ - "ui" includes img, picture, video, css and svg + \\ - "css" includes style and link[rel=stylesheet] + \\ - "full" includes js, ui and css + \\ + \\--with_base Add a tag in dump. Defaults to false. + \\ + ++ common_options ++ + \\ + \\serve command + \\Starts a websocket CDP server + \\Example: {s} serve --host 127.0.0.1 --port 9222 + \\ + \\Options: + \\--host Host of the CDP server + \\ Defaults to "127.0.0.1" + \\ + \\--port Port of the CDP server + \\ Defaults to 9222 + \\ + \\--timeout Inactivity timeout in seconds before disconnecting clients + \\ Defaults to 10 (seconds). Limited to 604800 (1 week). + \\ + \\--max_connections + \\ Maximum number of simultaneous CDP connections. + \\ Defaults to 16. + \\ + \\--max_tabs Maximum number of tabs per CDP connection. + \\ Defaults to 8. + \\ + \\--max_tab_memory + \\ Maximum memory per tab in bytes. + \\ Defaults to 536870912 (512 MB). + \\ + \\--max_pending_connections + \\ Maximum pending connections in the accept queue. + \\ Defaults to 128. + \\ + ++ common_options ++ + \\ + \\version command + \\Displays the version of {s} + \\ + \\help command + \\Displays this message + \\ + ; + std.debug.print(usage, .{ self.exec_name, self.exec_name, self.exec_name, self.exec_name }); + if (success) { + return std.process.cleanExit(); + } + std.process.exit(1); +} + +pub fn parseArgs(allocator: Allocator) !Config { + var args = try std.process.argsWithAllocator(allocator); + defer args.deinit(); + + const exec_name = std.fs.path.basename(args.next().?); + + var config = Config{ + .mode = .{ .help = false }, + .exec_name = try allocator.dupe(u8, exec_name), + }; + + const mode_string = args.next() orelse ""; + const mode = std.meta.stringToEnum(RunMode, mode_string) orelse blk: { + const inferred_mode = inferMode(mode_string) orelse return config; + // "command" wasn't a command but an option. We can't reset args, but + // we can create a new one. Not great, but this fallback is temporary + // as we transition to this command mode approach. + args.deinit(); + + args = try std.process.argsWithAllocator(allocator); + // skip the exec_name + _ = args.skip(); + + break :blk inferred_mode; + }; + + config.mode = switch (mode) { + .help => .{ .help = true }, + .serve => .{ .serve = parseServeArgs(allocator, &args) catch return config }, + .fetch => .{ .fetch = parseFetchArgs(allocator, &args) catch return config }, + .version => .{ .version = {} }, + }; + return config; +} + +fn inferMode(opt: []const u8) ?RunMode { + if (opt.len == 0) { + return .serve; + } + + if (std.mem.startsWith(u8, opt, "--") == false) { + return .fetch; + } + + if (std.mem.eql(u8, opt, "--dump")) { + return .fetch; + } + + if (std.mem.eql(u8, opt, "--noscript")) { + return .fetch; + } + + if (std.mem.eql(u8, opt, "--strip_mode")) { + return .fetch; + } + + if (std.mem.eql(u8, opt, "--with_base")) { + return .fetch; + } + + if (std.mem.eql(u8, opt, "--host")) { + return .serve; + } + + if (std.mem.eql(u8, opt, "--port")) { + return .serve; + } + + if (std.mem.eql(u8, opt, "--timeout")) { + return .serve; + } + + return null; +} + +fn parseServeArgs( + allocator: Allocator, + args: *std.process.ArgIterator, +) !Serve { + var serve: Serve = .{}; + + while (args.next()) |opt| { + if (std.mem.eql(u8, "--host", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--host" }); + return error.InvalidArgument; + }; + serve.host = try allocator.dupe(u8, str); + continue; + } + + if (std.mem.eql(u8, "--port", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--port" }); + return error.InvalidArgument; + }; + + serve.port = std.fmt.parseInt(u16, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--port", .err = err }); + return error.InvalidArgument; + }; + continue; + } + + if (std.mem.eql(u8, "--timeout", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--timeout" }); + return error.InvalidArgument; + }; + + serve.timeout = std.fmt.parseInt(u31, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--timeout", .err = err }); + return error.InvalidArgument; + }; + continue; + } + + if (std.mem.eql(u8, "--max_connections", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--max_connections" }); + return error.InvalidArgument; + }; + + serve.max_connections = std.fmt.parseInt(u16, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--max_connections", .err = err }); + return error.InvalidArgument; + }; + continue; + } + + if (std.mem.eql(u8, "--max_tabs", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--max_tabs" }); + return error.InvalidArgument; + }; + + serve.max_tabs_per_connection = std.fmt.parseInt(u16, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--max_tabs", .err = err }); + return error.InvalidArgument; + }; + continue; + } + + if (std.mem.eql(u8, "--max_tab_memory", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--max_tab_memory" }); + return error.InvalidArgument; + }; + + serve.max_memory_per_tab = std.fmt.parseInt(u64, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--max_tab_memory", .err = err }); + return error.InvalidArgument; + }; + continue; + } + + if (std.mem.eql(u8, "--max_pending_connections", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--max_pending_connections" }); + return error.InvalidArgument; + }; + + serve.max_pending_connections = std.fmt.parseInt(u16, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--max_pending_connections", .err = err }); + return error.InvalidArgument; + }; + continue; + } + + if (try parseCommonArg(allocator, opt, args, &serve.common)) { + continue; + } + + log.fatal(.app, "unknown argument", .{ .mode = "serve", .arg = opt }); + return error.UnkownOption; + } + + return serve; +} + +fn parseFetchArgs( + allocator: Allocator, + args: *std.process.ArgIterator, +) !Fetch { + var fetch_dump: bool = false; + var withbase: bool = false; + var url: ?[:0]const u8 = null; + var common: Common = .{}; + var strip: dump.Opts.Strip = .{}; + + while (args.next()) |opt| { + if (std.mem.eql(u8, "--dump", opt)) { + fetch_dump = true; + continue; + } + + if (std.mem.eql(u8, "--noscript", opt)) { + log.warn(.app, "deprecation warning", .{ + .feature = "--noscript argument", + .hint = "use '--strip_mode js' instead", + }); + strip.js = true; + continue; + } + + if (std.mem.eql(u8, "--with_base", opt)) { + withbase = true; + continue; + } + + if (std.mem.eql(u8, "--strip_mode", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--strip_mode" }); + return error.InvalidArgument; + }; + + var it = std.mem.splitScalar(u8, str, ','); + while (it.next()) |part| { + const trimmed = std.mem.trim(u8, part, &std.ascii.whitespace); + if (std.mem.eql(u8, trimmed, "js")) { + strip.js = true; + } else if (std.mem.eql(u8, trimmed, "ui")) { + strip.ui = true; + } else if (std.mem.eql(u8, trimmed, "css")) { + strip.css = true; + } else if (std.mem.eql(u8, trimmed, "full")) { + strip.js = true; + strip.ui = true; + strip.css = true; + } else { + log.fatal(.app, "invalid option choice", .{ .arg = "--strip_mode", .value = trimmed }); + } + } + continue; + } + + if (try parseCommonArg(allocator, opt, args, &common)) { + continue; + } + + if (std.mem.startsWith(u8, opt, "--")) { + log.fatal(.app, "unknown argument", .{ .mode = "fetch", .arg = opt }); + return error.UnkownOption; + } + + if (url != null) { + log.fatal(.app, "duplicate fetch url", .{ .help = "only 1 URL can be specified" }); + return error.TooManyURLs; + } + url = try allocator.dupeZ(u8, opt); + } + + if (url == null) { + log.fatal(.app, "missing fetch url", .{ .help = "URL to fetch must be provided" }); + return error.MissingURL; + } + + return .{ + .url = url.?, + .dump = fetch_dump, + .strip = strip, + .common = common, + .withbase = withbase, + }; +} + +fn parseCommonArg( + allocator: Allocator, + opt: []const u8, + args: *std.process.ArgIterator, + common: *Common, +) !bool { + if (std.mem.eql(u8, "--insecure_disable_tls_host_verification", opt)) { + common.tls_verify_host = false; + return true; + } + + if (std.mem.eql(u8, "--http_proxy", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_proxy" }); + return error.InvalidArgument; + }; + common.http_proxy = try allocator.dupeZ(u8, str); + return true; + } + + if (std.mem.eql(u8, "--proxy_bearer_token", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--proxy_bearer_token" }); + return error.InvalidArgument; + }; + common.proxy_bearer_token = try allocator.dupeZ(u8, str); + return true; + } + + if (std.mem.eql(u8, "--http_max_concurrent", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_max_concurrent" }); + return error.InvalidArgument; + }; + + common.http_max_concurrent = std.fmt.parseInt(u8, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_concurrent", .err = err }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--http_max_host_open", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_max_host_open" }); + return error.InvalidArgument; + }; + + common.http_max_host_open = std.fmt.parseInt(u8, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_host_open", .err = err }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--http_connect_timeout", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_connect_timeout" }); + return error.InvalidArgument; + }; + + common.http_connect_timeout = std.fmt.parseInt(u31, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_connect_timeout", .err = err }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--http_timeout", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_timeout" }); + return error.InvalidArgument; + }; + + common.http_timeout = std.fmt.parseInt(u31, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_timeout", .err = err }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--log_level", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--log_level" }); + return error.InvalidArgument; + }; + + common.log_level = std.meta.stringToEnum(log.Level, str) orelse blk: { + if (std.mem.eql(u8, str, "error")) { + break :blk .err; + } + log.fatal(.app, "invalid option choice", .{ .arg = "--log_level", .value = str }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--log_format", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--log_format" }); + return error.InvalidArgument; + }; + + common.log_format = std.meta.stringToEnum(log.Format, str) orelse { + log.fatal(.app, "invalid option choice", .{ .arg = "--log_format", .value = str }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--log_filter_scopes", opt)) { + if (builtin.mode != .Debug) { + log.fatal(.app, "experimental", .{ .help = "log scope filtering is only available in debug builds" }); + return false; + } + + const str = args.next() orelse { + // disables the default filters + common.log_filter_scopes = &.{}; + return true; + }; + + var arr: std.ArrayListUnmanaged(log.Scope) = .empty; + + var it = std.mem.splitScalar(u8, str, ','); + while (it.next()) |part| { + try arr.append(allocator, std.meta.stringToEnum(log.Scope, part) orelse { + log.fatal(.app, "invalid option choice", .{ .arg = "--log_filter_scopes", .value = part }); + return false; + }); + } + common.log_filter_scopes = arr.items; + return true; + } + + if (std.mem.eql(u8, "--user_agent_suffix", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--user_agent_suffix" }); + return error.InvalidArgument; + }; + for (str) |c| { + if (!std.ascii.isPrint(c)) { + log.fatal(.app, "not printable character", .{ .arg = "--user_agent_suffix" }); + return error.InvalidArgument; + } + } + common.user_agent_suffix = try allocator.dupe(u8, str); + return true; + } + + return false; +} diff --git a/src/browser/webapi/Navigator.zig b/src/browser/webapi/Navigator.zig index 451c7521..c095ff65 100644 --- a/src/browser/webapi/Navigator.zig +++ b/src/browser/webapi/Navigator.zig @@ -27,7 +27,7 @@ _pad: bool = false, pub const init: Navigator = .{}; pub fn getUserAgent(_: *const Navigator, page: *Page) []const u8 { - return page._session.browser.app.config.user_agent; + return page._session.browser.app.http.user_agent; } pub fn getAppName(_: *const Navigator) []const u8 { diff --git a/src/http/Client.zig b/src/http/Client.zig index 9d497597..c1edd1e7 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -21,6 +21,7 @@ const log = @import("../log.zig"); const builtin = @import("builtin"); const Http = @import("Http.zig"); +const Config = @import("../Config.zig"); const URL = @import("../browser/URL.zig"); const Notification = @import("../Notification.zig"); const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar; @@ -124,7 +125,7 @@ pub const CDPClient = struct { const TransferQueue = std.DoublyLinkedList; -pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Client { +pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, config: *const Config) !*Client { var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator); errdefer transfer_pool.deinit(); @@ -134,11 +135,19 @@ pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Clie const multi = c.curl_multi_init() orelse return error.FailedToInitializeMulti; errdefer _ = c.curl_multi_cleanup(multi); - try errorMCheck(c.curl_multi_setopt(multi, c.CURLMOPT_MAX_HOST_CONNECTIONS, @as(c_long, opts.max_host_open))); + try errorMCheck(c.curl_multi_setopt(multi, c.CURLMOPT_MAX_HOST_CONNECTIONS, @as(c_long, config.httpMaxHostOpen()))); - var handles = try Handles.init(allocator, client, ca_blob, &opts); + const user_agent = try config.userAgent(allocator); + var proxy_bearer_header: ?[:0]const u8 = null; + if (config.proxyBearerToken()) |bt| { + proxy_bearer_header = try std.fmt.allocPrintSentinel(allocator, "Proxy-Authorization: Bearer {s}", .{bt}, 0); + } + + var handles = try Handles.init(allocator, client, ca_blob, config, user_agent, proxy_bearer_header); errdefer handles.deinit(allocator); + const http_proxy = config.httpProxy(); + client.* = .{ .queue = .{}, .active = 0, @@ -146,9 +155,9 @@ pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Clie .multi = multi, .handles = handles, .allocator = allocator, - .http_proxy = opts.http_proxy, - .use_proxy = opts.http_proxy != null, - .user_agent = opts.user_agent, + .http_proxy = http_proxy, + .use_proxy = http_proxy != null, + .user_agent = user_agent, .transfer_pool = transfer_pool, }; @@ -657,16 +666,23 @@ const Handles = struct { const HandleList = std.DoublyLinkedList; - // pointer to opts is not stable, don't hold a reference to it! - fn init(allocator: Allocator, client: *Client, ca_blob: ?c.curl_blob, opts: *const Http.Opts) !Handles { - const count = if (opts.max_concurrent == 0) 1 else opts.max_concurrent; + fn init( + allocator: Allocator, + client: *Client, + ca_blob: ?c.curl_blob, + config: *const Config, + user_agent: [:0]const u8, + proxy_bearer_header: ?[:0]const u8, + ) !Handles { + const count: usize = config.httpMaxConcurrent(); + if (count == 0) return error.InvalidMaxConcurrent; const handles = try allocator.alloc(Handle, count); errdefer allocator.free(handles); var available: HandleList = .{}; for (0..count) |i| { - handles[i] = try Handle.init(client, ca_blob, opts); + handles[i] = try Handle.init(client, ca_blob, config, user_agent, proxy_bearer_header); available.append(&handles[i].node); } @@ -713,9 +729,14 @@ pub const Handle = struct { conn: Http.Connection, node: Handles.HandleList.Node, - // pointer to opts is not stable, don't hold a reference to it! - fn init(client: *Client, ca_blob: ?c.curl_blob, opts: *const Http.Opts) !Handle { - const conn = try Http.Connection.init(ca_blob, opts); + fn init( + client: *Client, + ca_blob: ?c.curl_blob, + config: *const Config, + user_agent: [:0]const u8, + proxy_bearer_header: ?[:0]const u8, + ) !Handle { + const conn = try Http.Connection.init(ca_blob, config, user_agent, proxy_bearer_header); errdefer conn.deinit(); const easy = conn.easy; diff --git a/src/http/Http.zig b/src/http/Http.zig index f54fd6e4..b619b93f 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2025 Lightpanda (Selecy SAS) +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) // // Francis Bouvier // Pierre Tachoire @@ -18,6 +18,7 @@ const std = @import("std"); const lp = @import("lightpanda"); +const Config = @import("../Config.zig"); pub const c = @cImport({ @cInclude("curl/curl.h"); @@ -40,12 +41,14 @@ const ArenaAllocator = std.heap.ArenaAllocator; // once for all http connections is a win. const Http = @This(); -opts: Opts, +config: *const Config, client: *Client, ca_blob: ?c.curl_blob, arena: ArenaAllocator, +user_agent: [:0]const u8, +proxy_bearer_header: ?[:0]const u8, -pub fn init(allocator: Allocator, opts: Opts) !Http { +pub fn init(allocator: Allocator, config: *const Config) !Http { try errorCheck(c.curl_global_init(c.CURL_GLOBAL_SSL)); errdefer c.curl_global_cleanup(); @@ -56,24 +59,28 @@ pub fn init(allocator: Allocator, opts: Opts) !Http { var arena = ArenaAllocator.init(allocator); errdefer arena.deinit(); - var adjusted_opts = opts; - if (opts.proxy_bearer_token) |bt| { - adjusted_opts.proxy_bearer_token = try std.fmt.allocPrintSentinel(arena.allocator(), "Proxy-Authorization: Bearer {s}", .{bt}, 0); + const user_agent = try config.userAgent(arena.allocator()); + + var proxy_bearer_header: ?[:0]const u8 = null; + if (config.proxyBearerToken()) |bt| { + proxy_bearer_header = try std.fmt.allocPrintSentinel(arena.allocator(), "Proxy-Authorization: Bearer {s}", .{bt}, 0); } var ca_blob: ?c.curl_blob = null; - if (opts.tls_verify_host) { + if (config.tlsVerifyHost()) { ca_blob = try loadCerts(allocator, arena.allocator()); } - var client = try Client.init(allocator, ca_blob, adjusted_opts); + var client = try Client.init(allocator, ca_blob, config); errdefer client.deinit(); return .{ .arena = arena, .client = client, .ca_blob = ca_blob, - .opts = adjusted_opts, + .config = config, + .user_agent = user_agent, + .proxy_bearer_header = proxy_bearer_header, }; } @@ -100,59 +107,60 @@ pub fn removeCDPClient(self: *Http) void { } pub fn newConnection(self: *Http) !Connection { - return Connection.init(self.ca_blob, &self.opts); + return Connection.init(self.ca_blob, self.config, self.user_agent, self.proxy_bearer_header); } pub fn newHeaders(self: *const Http) Headers { - return Headers.init(self.opts.user_agent); + return Headers.init(self.user_agent); } pub const Connection = struct { easy: *c.CURL, - opts: Connection.Opts, + user_agent: [:0]const u8, + proxy_bearer_header: ?[:0]const u8, - const Opts = struct { - proxy_bearer_token: ?[:0]const u8, + pub fn init( + ca_blob_: ?c.curl_blob, + config: *const Config, user_agent: [:0]const u8, - }; - - // pointer to opts is not stable, don't hold a reference to it! - pub fn init(ca_blob_: ?c.curl_blob, opts: *const Http.Opts) !Connection { + proxy_bearer_header: ?[:0]const u8, + ) !Connection { const easy = c.curl_easy_init() orelse return error.FailedToInitializeEasy; errdefer _ = c.curl_easy_cleanup(easy); // timeouts - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_TIMEOUT_MS, @as(c_long, @intCast(opts.timeout_ms)))); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CONNECTTIMEOUT_MS, @as(c_long, @intCast(opts.connect_timeout_ms)))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_TIMEOUT_MS, @as(c_long, @intCast(config.httpTimeout())))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CONNECTTIMEOUT_MS, @as(c_long, @intCast(config.httpConnectTimeout())))); // redirect behavior - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_MAXREDIRS, @as(c_long, @intCast(opts.max_redirects)))); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_MAXREDIRS, @as(c_long, @intCast(config.httpMaxRedirects())))); try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_FOLLOWLOCATION, @as(c_long, 2))); try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_REDIR_PROTOCOLS_STR, "HTTP,HTTPS")); // remove FTP and FTPS from the default // proxy - if (opts.http_proxy) |proxy| { + const http_proxy = config.httpProxy(); + if (http_proxy) |proxy| { try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY, proxy.ptr)); } // tls if (ca_blob_) |ca_blob| { try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CAINFO_BLOB, ca_blob)); - if (opts.http_proxy != null) { + if (http_proxy != null) { // Note, this can be difference for the proxy and for the main // request. Might be something worth exposting as command // line arguments at some point. try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PROXY_CAINFO_BLOB, ca_blob)); } } else { - lp.assert(opts.tls_verify_host == false, "Http.init tls_verify_host", .{}); + lp.assert(config.tlsVerifyHost() == false, "Http.init tls_verify_host", .{}); // Verify peer checks that the cert is signed by a CA, verify host makes sure the // cert contains the server name. try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYHOST, @as(c_long, 0))); try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_SSL_VERIFYPEER, @as(c_long, 0))); - if (opts.http_proxy != null) { + if (http_proxy != null) { // Note, this can be difference for the proxy and for the main // request. Might be something worth exposting as command // line arguments at some point. @@ -180,10 +188,8 @@ pub const Connection = struct { return .{ .easy = easy, - .opts = .{ - .user_agent = opts.user_agent, - .proxy_bearer_token = opts.proxy_bearer_token, - }, + .user_agent = user_agent, + .proxy_bearer_header = proxy_bearer_header, }; } @@ -237,7 +243,7 @@ pub const Connection = struct { // These are headers that may not be send to the users for inteception. pub fn secretHeaders(self: *const Connection, headers: *Headers) !void { - if (self.opts.proxy_bearer_token) |hdr| { + if (self.proxy_bearer_header) |hdr| { try headers.add(hdr); } } @@ -245,7 +251,7 @@ pub const Connection = struct { pub fn request(self: *const Connection) !u16 { const easy = self.easy; - var header_list = try Headers.init(self.opts.user_agent); + var header_list = try Headers.init(self.user_agent); defer header_list.deinit(); try self.secretHeaders(&header_list); try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); @@ -347,18 +353,6 @@ pub fn errorMCheck(code: c.CURLMcode) errors.Multi!void { return errors.fromMCode(code); } -pub const Opts = struct { - timeout_ms: u31, - max_host_open: u8, - max_concurrent: u8, - connect_timeout_ms: u31, - max_redirects: u8 = 10, - tls_verify_host: bool = true, - http_proxy: ?[:0]const u8 = null, - proxy_bearer_token: ?[:0]const u8 = null, - user_agent: [:0]const u8, -}; - pub const Method = enum(u8) { GET = 0, PUT = 1, diff --git a/src/lightpanda.zig b/src/lightpanda.zig index ade06ceb..b3ca22c2 100644 --- a/src/lightpanda.zig +++ b/src/lightpanda.zig @@ -19,6 +19,7 @@ const std = @import("std"); pub const App = @import("App.zig"); pub const Server = @import("Server.zig"); +pub const Config = @import("Config.zig"); pub const Page = @import("browser/Page.zig"); pub const Browser = @import("browser/Browser.zig"); pub const Session = @import("browser/Session.zig"); diff --git a/src/main.zig b/src/main.zig index d0d83b56..9b849dc4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2025 Lightpanda (Selecy SAS) +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) // // Francis Bouvier // Pierre Tachoire @@ -23,6 +23,7 @@ const Allocator = std.mem.Allocator; const log = lp.log; const App = lp.App; +const Config = lp.Config; const SigHandler = @import("Sighandler.zig"); pub const panic = lp.crash_handler.panic; @@ -52,7 +53,7 @@ pub fn main() !void { } fn run(allocator: Allocator, main_arena: Allocator, sighandler: *SigHandler) !void { - const args = try parseArgs(main_arena); + const args = try Config.parseArgs(main_arena); switch (args.mode) { .help => { @@ -76,26 +77,8 @@ fn run(allocator: Allocator, main_arena: Allocator, sighandler: *SigHandler) !vo log.opts.filter_scopes = lfs; } - const user_agent = blk: { - const USER_AGENT = "User-Agent: Lightpanda/1.0"; - if (args.userAgentSuffix()) |suffix| { - break :blk try std.fmt.allocPrintSentinel(main_arena, "{s} {s}", .{ USER_AGENT, suffix }, 0); - } - break :blk USER_AGENT; - }; - // _app is global to handle graceful shutdown. - var app = try App.init(allocator, .{ - .run_mode = args.mode, - .http_proxy = args.httpProxy(), - .proxy_bearer_token = args.proxyBearerToken(), - .tls_verify_host = args.tlsVerifyHost(), - .http_timeout_ms = args.httpTimeout(), - .http_connect_timeout_ms = args.httpConnectTiemout(), - .http_max_host_open = args.httpMaxHostOpen(), - .http_max_concurrent = args.httpMaxConcurrent(), - .user_agent = user_agent, - }); + var app = try App.init(allocator, &args); defer app.deinit(); app.telemetry.record(.{ .run = {} }); @@ -147,605 +130,3 @@ fn run(allocator: Allocator, main_arena: Allocator, sighandler: *SigHandler) !vo else => unreachable, } } - -const Command = struct { - mode: Mode, - exec_name: []const u8, - - fn tlsVerifyHost(self: *const Command) bool { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.tls_verify_host, - else => unreachable, - }; - } - - fn httpProxy(self: *const Command) ?[:0]const u8 { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.http_proxy, - else => unreachable, - }; - } - - fn proxyBearerToken(self: *const Command) ?[:0]const u8 { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.proxy_bearer_token, - else => unreachable, - }; - } - - fn httpMaxConcurrent(self: *const Command) ?u8 { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.http_max_concurrent, - else => unreachable, - }; - } - - fn httpMaxHostOpen(self: *const Command) ?u8 { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.http_max_host_open, - else => unreachable, - }; - } - - fn httpConnectTiemout(self: *const Command) ?u31 { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.http_connect_timeout, - else => unreachable, - }; - } - - fn httpTimeout(self: *const Command) ?u31 { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.http_timeout, - else => unreachable, - }; - } - - fn logLevel(self: *const Command) ?log.Level { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.log_level, - else => unreachable, - }; - } - - fn logFormat(self: *const Command) ?log.Format { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.log_format, - else => unreachable, - }; - } - - fn logFilterScopes(self: *const Command) ?[]const log.Scope { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.log_filter_scopes, - else => unreachable, - }; - } - - fn userAgentSuffix(self: *const Command) ?[]const u8 { - return switch (self.mode) { - inline .serve, .fetch => |opts| opts.common.user_agent_suffix, - else => unreachable, - }; - } - - const Mode = union(App.RunMode) { - help: bool, // false when being printed because of an error - fetch: Fetch, - serve: Serve, - version: void, - }; - - const Serve = struct { - host: []const u8, - port: u16, - timeout: u31, - common: Common, - }; - - const Fetch = struct { - url: [:0]const u8, - dump: bool = false, - common: Common, - withbase: bool = false, - strip: lp.dump.Opts.Strip = .{}, - }; - - const Common = struct { - proxy_bearer_token: ?[:0]const u8 = null, - http_proxy: ?[:0]const u8 = null, - http_max_concurrent: ?u8 = null, - http_max_host_open: ?u8 = null, - http_timeout: ?u31 = null, - http_connect_timeout: ?u31 = null, - tls_verify_host: bool = true, - log_level: ?log.Level = null, - log_format: ?log.Format = null, - log_filter_scopes: ?[]log.Scope = null, - user_agent_suffix: ?[]const u8 = null, - }; - - fn printUsageAndExit(self: *const Command, success: bool) void { - // MAX_HELP_LEN| - const common_options = - \\ - \\--insecure_disable_tls_host_verification - \\ Disables host verification on all HTTP requests. This is an - \\ advanced option which should only be set if you understand - \\ and accept the risk of disabling host verification. - \\ - \\--http_proxy The HTTP proxy to use for all HTTP requests. - \\ A username:password can be included for basic authentication. - \\ Defaults to none. - \\ - \\--proxy_bearer_token - \\ The to send for bearer authentication with the proxy - \\ Proxy-Authorization: Bearer - \\ - \\--http_max_concurrent - \\ The maximum number of concurrent HTTP requests. - \\ Defaults to 10. - \\ - \\--http_max_host_open - \\ The maximum number of open connection to a given host:port. - \\ Defaults to 4. - \\ - \\--http_connect_timeout - \\ The time, in milliseconds, for establishing an HTTP connection - \\ before timing out. 0 means it never times out. - \\ Defaults to 0. - \\ - \\--http_timeout - \\ The maximum time, in milliseconds, the transfer is allowed - \\ to complete. 0 means it never times out. - \\ Defaults to 10000. - \\ - \\--log_level The log level: debug, info, warn, error or fatal. - \\ Defaults to - ++ (if (builtin.mode == .Debug) " info." else "warn.") ++ - \\ - \\ - \\--log_format The log format: pretty or logfmt. - \\ Defaults to - ++ (if (builtin.mode == .Debug) " pretty." else " logfmt.") ++ - \\ - \\ - \\--log_filter_scopes - \\ Filter out too verbose logs per scope: - \\ http, unknown_prop, event, ... - \\ - \\--user_agent_suffix - \\ Suffix to append to the Lightpanda/X.Y User-Agent - \\ - ; - - // MAX_HELP_LEN| - const usage = - \\usage: {s} command [options] [URL] - \\ - \\Command can be either 'fetch', 'serve' or 'help' - \\ - \\fetch command - \\Fetches the specified URL - \\Example: {s} fetch --dump https://lightpanda.io/ - \\ - \\Options: - \\--dump Dumps document to stdout. - \\ Defaults to false. - \\ - \\--strip_mode Comma separated list of tag groups to remove from dump - \\ the dump. e.g. --strip_mode js,css - \\ - "js" script and link[as=script, rel=preload] - \\ - "ui" includes img, picture, video, css and svg - \\ - "css" includes style and link[rel=stylesheet] - \\ - "full" includes js, ui and css - \\ - \\--with_base Add a tag in dump. Defaults to false. - \\ - ++ common_options ++ - \\ - \\serve command - \\Starts a websocket CDP server - \\Example: {s} serve --host 127.0.0.1 --port 9222 - \\ - \\Options: - \\--host Host of the CDP server - \\ Defaults to "127.0.0.1" - \\ - \\--port Port of the CDP server - \\ Defaults to 9222 - \\ - \\--timeout Inactivity timeout in seconds before disconnecting clients - \\ Defaults to 10 (seconds). Limited to 604800 (1 week). - \\ - ++ common_options ++ - \\ - \\version command - \\Displays the version of {s} - \\ - \\help command - \\Displays this message - \\ - ; - std.debug.print(usage, .{ self.exec_name, self.exec_name, self.exec_name, self.exec_name }); - if (success) { - return std.process.cleanExit(); - } - std.process.exit(1); - } -}; - -fn parseArgs(allocator: Allocator) !Command { - var args = try std.process.argsWithAllocator(allocator); - defer args.deinit(); - - const exec_name = std.fs.path.basename(args.next().?); - - var cmd = Command{ - .mode = .{ .help = false }, - .exec_name = try allocator.dupe(u8, exec_name), - }; - - const mode_string = args.next() orelse ""; - const mode = std.meta.stringToEnum(App.RunMode, mode_string) orelse blk: { - const inferred_mode = inferMode(mode_string) orelse return cmd; - // "command" wasn't a command but an option. We can't reset args, but - // we can create a new one. Not great, but this fallback is temporary - // as we transition to this command mode approach. - args.deinit(); - - args = try std.process.argsWithAllocator(allocator); - // skip the exec_name - _ = args.skip(); - - break :blk inferred_mode; - }; - - cmd.mode = switch (mode) { - .help => .{ .help = true }, - .serve => .{ .serve = parseServeArgs(allocator, &args) catch return cmd }, - .fetch => .{ .fetch = parseFetchArgs(allocator, &args) catch return cmd }, - .version => .{ .version = {} }, - }; - return cmd; -} - -fn inferMode(opt: []const u8) ?App.RunMode { - if (opt.len == 0) { - return .serve; - } - - if (std.mem.startsWith(u8, opt, "--") == false) { - return .fetch; - } - - if (std.mem.eql(u8, opt, "--dump")) { - return .fetch; - } - - if (std.mem.eql(u8, opt, "--noscript")) { - return .fetch; - } - - if (std.mem.eql(u8, opt, "--strip_mode")) { - return .fetch; - } - - if (std.mem.eql(u8, opt, "--with_base")) { - return .fetch; - } - - if (std.mem.eql(u8, opt, "--host")) { - return .serve; - } - - if (std.mem.eql(u8, opt, "--port")) { - return .serve; - } - - if (std.mem.eql(u8, opt, "--timeout")) { - return .serve; - } - - return null; -} - -fn parseServeArgs( - allocator: Allocator, - args: *std.process.ArgIterator, -) !Command.Serve { - var host: []const u8 = "127.0.0.1"; - var port: u16 = 9222; - var timeout: u31 = 10; - var common: Command.Common = .{}; - - while (args.next()) |opt| { - if (std.mem.eql(u8, "--host", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--host" }); - return error.InvalidArgument; - }; - host = try allocator.dupe(u8, str); - continue; - } - - if (std.mem.eql(u8, "--port", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--port" }); - return error.InvalidArgument; - }; - - port = std.fmt.parseInt(u16, str, 10) catch |err| { - log.fatal(.app, "invalid argument value", .{ .arg = "--port", .err = err }); - return error.InvalidArgument; - }; - continue; - } - - if (std.mem.eql(u8, "--timeout", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--timeout" }); - return error.InvalidArgument; - }; - - timeout = std.fmt.parseInt(u31, str, 10) catch |err| { - log.fatal(.app, "invalid argument value", .{ .arg = "--timeout", .err = err }); - return error.InvalidArgument; - }; - continue; - } - - if (try parseCommonArg(allocator, opt, args, &common)) { - continue; - } - - log.fatal(.app, "unknown argument", .{ .mode = "serve", .arg = opt }); - return error.UnkownOption; - } - - return .{ - .host = host, - .port = port, - .common = common, - .timeout = timeout, - }; -} - -fn parseFetchArgs( - allocator: Allocator, - args: *std.process.ArgIterator, -) !Command.Fetch { - var dump: bool = false; - var withbase: bool = false; - var url: ?[:0]const u8 = null; - var common: Command.Common = .{}; - var strip: lp.dump.Opts.Strip = .{}; - - while (args.next()) |opt| { - if (std.mem.eql(u8, "--dump", opt)) { - dump = true; - continue; - } - - if (std.mem.eql(u8, "--noscript", opt)) { - log.warn(.app, "deprecation warning", .{ - .feature = "--noscript argument", - .hint = "use '--strip_mode js' instead", - }); - strip.js = true; - continue; - } - - if (std.mem.eql(u8, "--with_base", opt)) { - withbase = true; - continue; - } - - if (std.mem.eql(u8, "--strip_mode", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--strip_mode" }); - return error.InvalidArgument; - }; - - var it = std.mem.splitScalar(u8, str, ','); - while (it.next()) |part| { - const trimmed = std.mem.trim(u8, part, &std.ascii.whitespace); - if (std.mem.eql(u8, trimmed, "js")) { - strip.js = true; - } else if (std.mem.eql(u8, trimmed, "ui")) { - strip.ui = true; - } else if (std.mem.eql(u8, trimmed, "css")) { - strip.css = true; - } else if (std.mem.eql(u8, trimmed, "full")) { - strip.js = true; - strip.ui = true; - strip.css = true; - } else { - log.fatal(.app, "invalid option choice", .{ .arg = "--strip_mode", .value = trimmed }); - } - } - continue; - } - - if (try parseCommonArg(allocator, opt, args, &common)) { - continue; - } - - if (std.mem.startsWith(u8, opt, "--")) { - log.fatal(.app, "unknown argument", .{ .mode = "fetch", .arg = opt }); - return error.UnkownOption; - } - - if (url != null) { - log.fatal(.app, "duplicate fetch url", .{ .help = "only 1 URL can be specified" }); - return error.TooManyURLs; - } - url = try allocator.dupeZ(u8, opt); - } - - if (url == null) { - log.fatal(.app, "missing fetch url", .{ .help = "URL to fetch must be provided" }); - return error.MissingURL; - } - - return .{ - .url = url.?, - .dump = dump, - .strip = strip, - .common = common, - .withbase = withbase, - }; -} - -fn parseCommonArg( - allocator: Allocator, - opt: []const u8, - args: *std.process.ArgIterator, - common: *Command.Common, -) !bool { - if (std.mem.eql(u8, "--insecure_disable_tls_host_verification", opt)) { - common.tls_verify_host = false; - return true; - } - - if (std.mem.eql(u8, "--http_proxy", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--http_proxy" }); - return error.InvalidArgument; - }; - common.http_proxy = try allocator.dupeZ(u8, str); - return true; - } - - if (std.mem.eql(u8, "--proxy_bearer_token", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--proxy_bearer_token" }); - return error.InvalidArgument; - }; - common.proxy_bearer_token = try allocator.dupeZ(u8, str); - return true; - } - - if (std.mem.eql(u8, "--http_max_concurrent", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--http_max_concurrent" }); - return error.InvalidArgument; - }; - - common.http_max_concurrent = std.fmt.parseInt(u8, str, 10) catch |err| { - log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_concurrent", .err = err }); - return error.InvalidArgument; - }; - return true; - } - - if (std.mem.eql(u8, "--http_max_host_open", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--http_max_host_open" }); - return error.InvalidArgument; - }; - - common.http_max_host_open = std.fmt.parseInt(u8, str, 10) catch |err| { - log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_host_open", .err = err }); - return error.InvalidArgument; - }; - return true; - } - - if (std.mem.eql(u8, "--http_connect_timeout", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--http_connect_timeout" }); - return error.InvalidArgument; - }; - - common.http_connect_timeout = std.fmt.parseInt(u31, str, 10) catch |err| { - log.fatal(.app, "invalid argument value", .{ .arg = "--http_connect_timeout", .err = err }); - return error.InvalidArgument; - }; - return true; - } - - if (std.mem.eql(u8, "--http_timeout", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--http_timeout" }); - return error.InvalidArgument; - }; - - common.http_timeout = std.fmt.parseInt(u31, str, 10) catch |err| { - log.fatal(.app, "invalid argument value", .{ .arg = "--http_timeout", .err = err }); - return error.InvalidArgument; - }; - return true; - } - - if (std.mem.eql(u8, "--log_level", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--log_level" }); - return error.InvalidArgument; - }; - - common.log_level = std.meta.stringToEnum(log.Level, str) orelse blk: { - if (std.mem.eql(u8, str, "error")) { - break :blk .err; - } - log.fatal(.app, "invalid option choice", .{ .arg = "--log_level", .value = str }); - return error.InvalidArgument; - }; - return true; - } - - if (std.mem.eql(u8, "--log_format", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--log_format" }); - return error.InvalidArgument; - }; - - common.log_format = std.meta.stringToEnum(log.Format, str) orelse { - log.fatal(.app, "invalid option choice", .{ .arg = "--log_format", .value = str }); - return error.InvalidArgument; - }; - return true; - } - - if (std.mem.eql(u8, "--log_filter_scopes", opt)) { - if (builtin.mode != .Debug) { - log.fatal(.app, "experimental", .{ .help = "log scope filtering is only available in debug builds" }); - return false; - } - - const str = args.next() orelse { - // disables the default filters - common.log_filter_scopes = &.{}; - return true; - }; - - var arr: std.ArrayListUnmanaged(log.Scope) = .empty; - - var it = std.mem.splitScalar(u8, str, ','); - while (it.next()) |part| { - try arr.append(allocator, std.meta.stringToEnum(log.Scope, part) orelse { - log.fatal(.app, "invalid option choice", .{ .arg = "--log_filter_scopes", .value = part }); - return false; - }); - } - common.log_filter_scopes = arr.items; - return true; - } - - if (std.mem.eql(u8, "--user_agent_suffix", opt)) { - const str = args.next() orelse { - log.fatal(.app, "missing argument value", .{ .arg = "--user_agent_suffix" }); - return error.InvalidArgument; - }; - for (str) |c| { - if (!std.ascii.isPrint(c)) { - log.fatal(.app, "not printable character", .{ .arg = "--user_agent_suffix" }); - return error.InvalidArgument; - } - } - common.user_agent_suffix = try allocator.dupe(u8, str); - return true; - } - - return false; -} diff --git a/src/main_legacy_test.zig b/src/main_legacy_test.zig index 78b85472..5309f98b 100644 --- a/src/main_legacy_test.zig +++ b/src/main_legacy_test.zig @@ -32,12 +32,16 @@ pub fn main() !void { wg.wait(); } lp.log.opts.level = .warn; - - var app = try lp.App.init(allocator, .{ - .run_mode = .serve, - .tls_verify_host = false, - .user_agent = "User-Agent: Lightpanda/1.0 internal-tester", - }); + const config = lp.Config{ + .mode = .{ .serve = .{ + .common = .{ + .tls_verify_host = false, + .user_agent_suffix = "internal-tester", + }, + } }, + .exec_name = "legacy-test", + }; + var app = try lp.App.init(allocator, &config); defer app.deinit(); var test_arena = std.heap.ArenaAllocator.init(allocator); diff --git a/src/main_wpt.zig b/src/main_wpt.zig index a70aacc6..ce381479 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -58,11 +58,16 @@ pub fn main() !void { defer writer.deinit(); lp.log.opts.level = .warn; - var app = try lp.App.init(allocator, .{ - .run_mode = .serve, - .tls_verify_host = false, - .user_agent = "User-Agent: Lightpanda/1.0 internal-tester", - }); + const config = lp.Config{ + .mode = .{ .serve = .{ + .common = .{ + .tls_verify_host = false, + .user_agent_suffix = "internal-tester", + }, + } }, + .exec_name = "lightpanda-wpt", + }; + var app = try lp.App.init(allocator, &config); defer app.deinit(); var browser = try lp.Browser.init(app); diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index cd87bf8e..d05929fb 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -8,6 +8,7 @@ const Allocator = std.mem.Allocator; const log = @import("../log.zig"); const App = @import("../App.zig"); const Http = @import("../http/Http.zig"); +const Config = @import("../Config.zig"); const telemetry = @import("telemetry.zig"); const URL = "https://telemetry.lightpanda.io"; @@ -55,7 +56,7 @@ pub const LightPanda = struct { self.connection.deinit(); } - pub fn send(self: *LightPanda, iid: ?[]const u8, run_mode: App.RunMode, raw_event: telemetry.Event) !void { + pub fn send(self: *LightPanda, iid: ?[]const u8, run_mode: Config.RunMode, raw_event: telemetry.Event) !void { const event = try self.mem_pool.create(); event.* = .{ .iid = iid, @@ -130,7 +131,7 @@ pub const LightPanda = struct { const LightPandaEvent = struct { iid: ?[]const u8, - mode: App.RunMode, + mode: Config.RunMode, event: telemetry.Event, node: std.DoublyLinkedList.Node, diff --git a/src/telemetry/telemetry.zig b/src/telemetry/telemetry.zig index cffa8768..ff0d241d 100644 --- a/src/telemetry/telemetry.zig +++ b/src/telemetry/telemetry.zig @@ -5,6 +5,7 @@ const Allocator = std.mem.Allocator; const log = @import("../log.zig"); const App = @import("../App.zig"); +const Config = @import("../Config.zig"); const Notification = @import("../Notification.zig"); const uuidv4 = @import("../id.zig").uuidv4; @@ -29,11 +30,11 @@ fn TelemetryT(comptime P: type) type { disabled: bool, - run_mode: App.RunMode, + run_mode: Config.RunMode, const Self = @This(); - pub fn init(app: *App, run_mode: App.RunMode) !Self { + pub fn init(app: *App, run_mode: Config.RunMode) !Self { const disabled = isDisabled(); if (builtin.mode != .Debug and builtin.is_test == false) { log.info(.telemetry, "telemetry status", .{ .disabled = disabled }); @@ -145,7 +146,7 @@ const NoopProvider = struct { return .{}; } fn deinit(_: NoopProvider) void {} - pub fn send(_: NoopProvider, _: ?[]const u8, _: App.RunMode, _: Event) !void {} + pub fn send(_: NoopProvider, _: ?[]const u8, _: Config.RunMode, _: Event) !void {} }; extern fn setenv(name: [*:0]u8, value: [*:0]u8, override: c_int) c_int; @@ -161,7 +162,7 @@ test "telemetry: disabled by environment" { return .{}; } fn deinit(_: @This()) void {} - pub fn send(_: @This(), _: ?[]const u8, _: App.RunMode, _: Event) !void { + pub fn send(_: @This(), _: ?[]const u8, _: Config.RunMode, _: Event) !void { unreachable; } }; @@ -206,7 +207,7 @@ test "telemetry: sends event to provider" { const MockProvider = struct { iid: ?[]const u8, - run_mode: ?App.RunMode, + run_mode: ?Config.RunMode, allocator: Allocator, events: std.ArrayListUnmanaged(Event), @@ -221,7 +222,7 @@ const MockProvider = struct { fn deinit(self: *MockProvider) void { self.events.deinit(self.allocator); } - pub fn send(self: *MockProvider, iid: ?[]const u8, run_mode: App.RunMode, events: Event) !void { + pub fn send(self: *MockProvider, iid: ?[]const u8, run_mode: Config.RunMode, events: Event) !void { if (self.iid == null) { try testing.expectEqual(null, self.run_mode); self.iid = iid.?; diff --git a/src/testing.zig b/src/testing.zig index 3a71a0ef..db808498 100644 --- a/src/testing.zig +++ b/src/testing.zig @@ -38,9 +38,10 @@ pub fn reset() void { const App = @import("App.zig"); const js = @import("browser/js/js.zig"); +const Config = @import("Config.zig"); +const Page = @import("browser/Page.zig"); const Browser = @import("browser/Browser.zig"); const Session = @import("browser/Session.zig"); -const Page = @import("browser/Page.zig"); // Merged std.testing.expectEqual and std.testing.expectString // can be useful when testing fields of an anytype an you don't know @@ -449,15 +450,21 @@ const Server = @import("Server.zig"); var test_cdp_server: ?Server = null; var test_http_server: ?TestHTTPServer = null; +const test_config = Config{ + .mode = .{ .serve = .{ + .common = .{ + .tls_verify_host = false, + .user_agent_suffix = "internal-tester", + }, + } }, + .exec_name = "test", +}; + test "tests:beforeAll" { log.opts.level = .warn; log.opts.format = .pretty; - test_app = try App.init(@import("root").tracking_allocator, .{ - .run_mode = .serve, - .tls_verify_host = false, - .user_agent = "User-Agent: Lightpanda/1.0 internal-tester", - }); + test_app = try App.init(@import("root").tracking_allocator, &test_config); errdefer test_app.deinit(); test_browser = try Browser.init(test_app);