add obey_robots option to args

src/main.zig

@@ -87,6 +87,7 @@ fn run(allocator: Allocator, main_arena: Allocator, sighandler: *SigHandler) !vo
     // _app is global to handle graceful shutdown.
     var app = try App.init(allocator, .{
         .run_mode = args.mode,
+        .obey_robots = args.obeyRobots(),
         .http_proxy = args.httpProxy(),
         .proxy_bearer_token = args.proxyBearerToken(),
         .tls_verify_host = args.tlsVerifyHost(),
@@ -159,6 +160,13 @@ const Command = struct {
         };
     }
 
+    fn obeyRobots(self: *const Command) bool {
+        return switch (self.mode) {
+            inline .serve, .fetch => |opts| opts.common.obey_robots,
+            else => unreachable,
+        };
+    }
+
     fn httpProxy(self: *const Command) ?[:0]const u8 {
         return switch (self.mode) {
             inline .serve, .fetch => |opts| opts.common.http_proxy,
@@ -252,6 +260,7 @@ const Command = struct {
     };
 
     const Common = struct {
+        obey_robots: bool = false,
         proxy_bearer_token: ?[:0]const u8 = null,
         http_proxy: ?[:0]const u8 = null,
         http_max_concurrent: ?u8 = null,
@@ -273,6 +282,10 @@ const Command = struct {
             \\                Disables host verification on all HTTP requests. This is an
             \\                advanced option which should only be set if you understand
             \\                and accept the risk of disabling host verification.
+            \\--obey_robots
+            \\                Fetches and obeys the robots.txt (if available) of the web pages
+            \\                we make requests towards.
+            \\                Defaults to false.
             \\
             \\--http_proxy    The HTTP proxy to use for all HTTP requests.
             \\                A username:password can be included for basic authentication.
@@ -608,6 +621,11 @@ fn parseCommonArg(
         return true;
     }
 
+    if (std.mem.eql(u8, "--obey_robots", opt)) {
+        common.obey_robots = true;
+        return true;
+    }
+
     if (std.mem.eql(u8, "--http_proxy", opt)) {
         const str = args.next() orelse {
             log.fatal(.app, "missing argument value", .{ .arg = "--http_proxy" });