public/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-02-04 06:23:45 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

add obey_robots option to args

Browse Source
This commit is contained in:
Muki Kiboigo
2026-01-31 19:29:34 -08:00
parent 04f9793b7e
commit 1a17924620
1 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/main.zig
View File

@@ -87,6 +87,7 @@ fn run(allocator: Allocator, main_arena: Allocator, sighandler: *SigHandler) !vo
// _app is global to handle graceful shutdown. // _app is global to handle graceful shutdown.
var app = try App.init(allocator, .{ var app = try App.init(allocator, .{
.run_mode = args.mode, .run_mode = args.mode,
.obey_robots = args.obeyRobots(),
.http_proxy = args.httpProxy(), .http_proxy = args.httpProxy(),
.proxy_bearer_token = args.proxyBearerToken(), .proxy_bearer_token = args.proxyBearerToken(),
.tls_verify_host = args.tlsVerifyHost(), .tls_verify_host = args.tlsVerifyHost(),
@@ -159,6 +160,13 @@ const Command = struct {
}; };
} }
fn obeyRobots(self: *const Command) bool {
return switch (self.mode) {
inline .serve, .fetch => |opts| opts.common.obey_robots,
else => unreachable,
};
}
fn httpProxy(self: *const Command) ?[:0]const u8 { fn httpProxy(self: *const Command) ?[:0]const u8 {
return switch (self.mode) { return switch (self.mode) {
inline .serve, .fetch => |opts| opts.common.http_proxy, inline .serve, .fetch => |opts| opts.common.http_proxy,
@@ -252,6 +260,7 @@ const Command = struct {
}; };
const Common = struct { const Common = struct {
obey_robots: bool = false,
proxy_bearer_token: ?[:0]const u8 = null, proxy_bearer_token: ?[:0]const u8 = null,
http_proxy: ?[:0]const u8 = null, http_proxy: ?[:0]const u8 = null,
http_max_concurrent: ?u8 = null, http_max_concurrent: ?u8 = null,
@@ -273,6 +282,10 @@ const Command = struct {
\\ Disables host verification on all HTTP requests. This is an \\ Disables host verification on all HTTP requests. This is an
\\ advanced option which should only be set if you understand \\ advanced option which should only be set if you understand
\\ and accept the risk of disabling host verification. \\ and accept the risk of disabling host verification.
\\--obey_robots
\\ Fetches and obeys the robots.txt (if available) of the web pages
\\ we make requests towards.
\\ Defaults to false.
\\ \\
\\--http_proxy The HTTP proxy to use for all HTTP requests. \\--http_proxy The HTTP proxy to use for all HTTP requests.
\\ A username:password can be included for basic authentication. \\ A username:password can be included for basic authentication.
@@ -608,6 +621,11 @@ fn parseCommonArg(
return true; return true;
} }
if (std.mem.eql(u8, "--obey_robots", opt)) {
common.obey_robots = true;
return true;
}
if (std.mem.eql(u8, "--http_proxy", opt)) { if (std.mem.eql(u8, "--http_proxy", opt)) {
const str = args.next() orelse { const str = args.next() orelse {
log.fatal(.app, "missing argument value", .{ .arg = "--http_proxy" }); log.fatal(.app, "missing argument value", .{ .arg = "--http_proxy" });