From c97a32e24b4875fc57a3c85aa299d7f4a2edd7e9 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 24 Jun 2025 15:10:20 +0800 Subject: [PATCH 1/7] Initial work on CONNECT proxy. Cannot currently connect to the proxy over TLS (though, once connected, it can connect to the actual site over TLS). No support for authentication. --- src/app.zig | 9 +- src/http/client.zig | 226 ++++++++++++++++++++++++++++++++++++++------ src/main.zig | 136 ++++++++++++++++---------- 3 files changed, 291 insertions(+), 80 deletions(-) diff --git a/src/app.zig b/src/app.zig index 803c55f0..30489755 100644 --- a/src/app.zig +++ b/src/app.zig @@ -3,7 +3,8 @@ const Allocator = std.mem.Allocator; const log = @import("log.zig"); const Loop = @import("runtime/loop.zig").Loop; -const HttpClient = @import("http/client.zig").Client; +const http = @import("http/client.zig"); + const Telemetry = @import("telemetry/telemetry.zig").Telemetry; const Notification = @import("notification.zig").Notification; @@ -14,7 +15,7 @@ pub const App = struct { config: Config, allocator: Allocator, telemetry: Telemetry, - http_client: HttpClient, + http_client: http.Client, app_dir_path: ?[]const u8, notification: *Notification, @@ -29,6 +30,7 @@ pub const App = struct { run_mode: RunMode, tls_verify_host: bool = true, http_proxy: ?std.Uri = null, + proxy_type: ?http.ProxyType = null, }; pub fn init(allocator: Allocator, config: Config) !*App { @@ -52,9 +54,10 @@ pub const App = struct { .telemetry = undefined, .app_dir_path = app_dir_path, .notification = notification, - .http_client = try HttpClient.init(allocator, .{ + .http_client = try http.Client.init(allocator, .{ .max_concurrent = 3, .http_proxy = config.http_proxy, + .proxy_type = config.proxy_type, .tls_verify_host = config.tls_verify_host, }), .config = config, diff --git a/src/http/client.zig b/src/http/client.zig index 72fc466f..859e2827 100644 --- a/src/http/client.zig +++ b/src/http/client.zig @@ -41,6 +41,11 @@ const BUFFER_LEN = 32 * 1024; const MAX_HEADER_LINE_LEN = 4096; +pub const ProxyType = enum { + simple, + connect, +}; + // Thread-safe. Holds our root certificate, connection pool and state pool // Used to create Requests. pub const Client = struct { @@ -48,6 +53,7 @@ pub const Client = struct { allocator: Allocator, state_pool: StatePool, http_proxy: ?Uri, + proxy_type: ?ProxyType, root_ca: tls.config.CertBundle, tls_verify_host: bool = true, connection_manager: ConnectionManager, @@ -56,6 +62,7 @@ pub const Client = struct { const Opts = struct { max_concurrent: usize = 3, http_proxy: ?std.Uri = null, + proxy_type: ?ProxyType = null, tls_verify_host: bool = true, max_idle_connection: usize = 10, }; @@ -76,6 +83,7 @@ pub const Client = struct { .allocator = allocator, .state_pool = state_pool, .http_proxy = opts.http_proxy, + .proxy_type = if (opts.http_proxy == null) null else (opts.proxy_type orelse .connect), .tls_verify_host = opts.tls_verify_host, .connection_manager = connection_manager, .request_pool = std.heap.MemoryPool(Request).init(allocator), @@ -186,6 +194,16 @@ pub const Client = struct { pub fn freeSlotCount(self: *Client) usize { return self.state_pool.freeSlotCount(); } + + fn isConnectProxy(self: *const Client) bool { + const proxy_type = self.proxy_type orelse return false; + return proxy_type == .connect; + } + + fn isSimpleProxy(self: *const Client) bool { + const proxy_type = self.proxy_type orelse return false; + return proxy_type == .simple; + } }; const RequestOpts = struct { @@ -330,6 +348,7 @@ pub const Request = struct { _keepalive: bool, // extracted from request_uri + _request_port: u16, _request_host: []const u8, // extracted from connect_uri @@ -420,6 +439,7 @@ pub const Request = struct { ._connect_host = decomposed.connect_host, ._connect_port = decomposed.connect_port, ._request_host = decomposed.request_host, + ._request_port = decomposed.request_port, ._state = state, ._client = client, ._aborter = null, @@ -455,6 +475,7 @@ pub const Request = struct { connect_port: u16, connect_host: []const u8, connect_uri: *const std.Uri, + request_port: u16, request_host: []const u8, }; fn decomposeURL(client: *const Client, uri: *const Uri) !DecomposedURL { @@ -470,8 +491,10 @@ pub const Request = struct { connect_host = proxy.host.?.percent_encoded; } + const is_connect_proxy = client.isConnectProxy(); + var secure: bool = undefined; - const scheme = connect_uri.scheme; + const scheme = if (is_connect_proxy) uri.scheme else connect_uri.scheme; if (std.ascii.eqlIgnoreCase(scheme, "https")) { secure = true; } else if (std.ascii.eqlIgnoreCase(scheme, "http")) { @@ -479,13 +502,15 @@ pub const Request = struct { } else { return error.UnsupportedUriScheme; } - const connect_port: u16 = connect_uri.port orelse if (secure) 443 else 80; + const request_port: u16 = uri.port orelse if (secure) 443 else 80; + const connect_port: u16 = connect_uri.port orelse (if (is_connect_proxy) 80 else request_port); return .{ .secure = secure, .connect_port = connect_port, .connect_host = connect_host, .connect_uri = connect_uri, + .request_port = request_port, .request_host = request_host, }; } @@ -595,13 +620,18 @@ pub const Request = struct { }; self._connection = connection; + const is_connect_proxy = self._client.isConnectProxy(); + if (is_connect_proxy) { + try SyncHandler.connect(self); + } + if (self._secure) { self._connection.?.tls = .{ .blocking = try tls.client(std.net.Stream{ .handle = socket }, .{ - .host = self._connect_host, + .host = if (is_connect_proxy) self._request_host else self._connect_host, .root_ca = self._client.root_ca, .insecure_skip_verify = self._tls_verify_host == false, - // .key_log_callback = tls.config.key_log.callback, + .key_log_callback = tls.config.key_log.callback, }), }; } @@ -682,7 +712,7 @@ pub const Request = struct { if (self._secure) { connection.tls = .{ .nonblocking = try tls.nb.Client().init(self._client.allocator, .{ - .host = self._connect_host, + .host = if (self._client.isConnectProxy()) self._request_host else self._connect_host, .root_ca = self._client.root_ca, .insecure_skip_verify = self._tls_verify_host == false, // .key_log_callback = tls.config.key_log.callback, @@ -831,7 +861,7 @@ pub const Request = struct { } fn buildHeader(self: *Request) ![]const u8 { - const proxied = self.connect_uri != self.request_uri; + const proxied = self._client.isSimpleProxy(); const buf = self._state.header_buf; var fbs = std.io.fixedBufferStream(buf); @@ -851,6 +881,16 @@ pub const Request = struct { return buf[0..fbs.pos]; } + fn buildConnectHeader(self: *Request) ![]const u8 { + const buf = self._state.header_buf; + var fbs = std.io.fixedBufferStream(buf); + var writer = fbs.writer(); + + try writer.print("CONNECT {s}:{d} HTTP/1.1\r\n", .{ self._request_host, self._request_port }); + try writer.print("Host: {s}:{d}\r\n\r\n", .{ self._request_host, self._request_port }); + return buf[0..fbs.pos]; + } + fn requestStarting(self: *Request) void { const notification = self.notification orelse return; if (self._notified_start) { @@ -895,6 +935,15 @@ pub const Request = struct { .headers = response.headers.items, }); } + + fn shouldProxyConnect(self: *const Request) bool { + // if the connection comes from a keepalive pool, than we already + // made a CONNECT request + if (self._connection_from_keepalive) { + return false; + } + return self._client.isConnectProxy(); + } }; // Handles asynchronous requests @@ -958,6 +1007,7 @@ fn AsyncHandler(comptime H: type, comptime L: type) type { const SendQueue = std.DoublyLinkedList([]const u8); const SendState = enum { + connect, handshake, header, body, @@ -986,7 +1036,19 @@ fn AsyncHandler(comptime H: type, comptime L: type) type { if (self.shutdown) { return self.maybeShutdown(); } + result catch |err| return self.handleError("Connection failed", err); + + if (self.request.shouldProxyConnect()) { + self.state = .connect; + const header = self.request.buildConnectHeader() catch |err| { + return self.handleError("Failed to build CONNECT header", err); + }; + self.send(header); + self.receive(); + return; + } + self.conn.connected() catch |err| { self.handleError("connected handler error", err); }; @@ -1056,6 +1118,12 @@ fn AsyncHandler(comptime H: type, comptime L: type) type { return; } + if (self.state == .connect) { + // We're in a proxy CONNECT flow. There's nothing for us to + // do except for wait for the response. + return; + } + self.conn.sent() catch |err| { self.handleError("send handling", err); }; @@ -1099,7 +1167,27 @@ fn AsyncHandler(comptime H: type, comptime L: type) type { return self.handleError("Connection closed", error.ConnectionResetByPeer); } - const status = self.conn.received(self.read_buf[0 .. self.read_pos + n]) catch |err| { + const data = self.read_buf[0 .. self.read_pos + n]; + + if (self.state == .connect) { + const success = self.reader.connectResponse(data) catch |err| { + return self.handleError("Invalid CONNECT response", err); + }; + + if (!success) { + self.receive(); + } else { + // CONNECT was successful, resume our normal flow + self.state = .handshake; + self.reader = self.request.newReader(); + self.conn.connected() catch |err| { + self.handleError("connected handler error", err); + }; + } + return; + } + + const status = self.conn.received(data) catch |err| { if (err == error.TlsAlertCloseNotify and self.state == .handshake and self.maybeRetryRequest()) { return; } @@ -1438,7 +1526,7 @@ fn AsyncHandler(comptime H: type, comptime L: type) type { const handler = self.handler; switch (self.protocol) { .plain => switch (handler.state) { - .handshake => unreachable, + .handshake, .connect => unreachable, .header => { handler.state = .body; if (handler.request.body) |body| { @@ -1455,6 +1543,7 @@ fn AsyncHandler(comptime H: type, comptime L: type) type { return; } switch (handler.state) { + .connect => unreachable, .handshake => return self.sendSecureHeader(tls_client), .header => { handler.state = .body; @@ -1589,6 +1678,37 @@ const SyncHandler = struct { } } + // Unfortunately, this is called from the Request doSendSync since we need + // to do this before setting up our TLS connection. + fn connect(request: *Request) !void { + const socket = request._connection.?.socket; + + const header = try request.buildConnectHeader(); + try Conn.writeAll(socket, header); + + var pos: usize = 0; + var reader = request.newReader(); + var read_buf = request._state.read_buf; + + while (true) { + // we would never 'maybeRetryOrErr' on a CONNECT request, because + // we only send CONNECT requests on newly established connections + // and maybeRetryOrErr is only for connections that might have been + // closed while being kept-alive + const n = try posix.read(socket, read_buf[pos..]); + if (n == 0) { + return error.ConnectionResetByPeer; + } + pos += n; + if (try reader.connectResponse(read_buf[0..pos])) { + // returns true if we have a successful connect response + return; + } + + // we don't have enough data yet. + } + } + fn maybeRetryOrErr(self: *SyncHandler, err: anyerror) !Response { var request = self.request; @@ -1828,6 +1948,26 @@ const Reader = struct { return .{ .use_get = use_get, .location = location }; } + fn connectResponse(self: *Reader, data: []u8) !bool { + const result = try self.process(data); + if (self.header_done == false) { + return false; + } + + if (result.done == false) { + // CONNECT responses should not have a body. If the header is + // done, then the entire response should be done. + return error.InvalidConnectResponse; + } + + const status = self.response.status; + if (status < 200 or status > 299) { + return error.InvalidConnectResponseStatus; + } + + return true; + } + fn process(self: *Reader, data: []u8) ProcessError!Result { if (self.body_reader) |*br| { const ok, const result = try br.process(data); @@ -2790,14 +2930,14 @@ test "HttpClient Reader: fuzz" { } test "HttpClient: invalid url" { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("http:///"); try testing.expectError(error.UriMissingHost, client.request(.GET, &uri)); } test "HttpClient: sync connect error" { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("HTTP://127.0.0.1:9920"); @@ -2809,7 +2949,7 @@ test "HttpClient: sync connect error" { test "HttpClient: sync no body" { for (0..2) |i| { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("http://127.0.0.1:9582/http_client/simple"); @@ -2831,7 +2971,7 @@ test "HttpClient: sync no body" { test "HttpClient: sync tls no body" { for (0..1) |_| { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("https://127.0.0.1:9581/http_client/simple"); @@ -2850,7 +2990,33 @@ test "HttpClient: sync tls no body" { test "HttpClient: sync with body" { for (0..2) |i| { - var client = try testClient(); + var client = try testClient(.{}); + defer client.deinit(); + + const uri = try Uri.parse("http://127.0.0.1:9582/http_client/echo"); + var req = try client.request(.GET, &uri); + defer req.deinit(); + + var res = try req.sendSync(.{}); + + if (i == 0) { + try testing.expectEqual("over 9000!", try res.peek()); + } + try testing.expectEqual("over 9000!", try res.next()); + try testing.expectEqual(201, res.header.status); + try testing.expectEqual(5, res.header.count()); + try testing.expectEqual("Close", res.header.get("connection")); + try testing.expectEqual("10", res.header.get("content-length")); + try testing.expectEqual("127.0.0.1", res.header.get("_host")); + try testing.expectEqual("Lightpanda/1.0", res.header.get("_user-agent")); + try testing.expectEqual("*/*", res.header.get("_accept")); + } +} + +test "HttpClient: sync with body proxy CONNECT" { + for (0..2) |i| { + const proxy_uri = try Uri.parse("http://127.0.0.1:9582/"); + var client = try testClient(.{ .proxy_type = .connect, .http_proxy = proxy_uri }); defer client.deinit(); const uri = try Uri.parse("http://127.0.0.1:9582/http_client/echo"); @@ -2875,7 +3041,7 @@ test "HttpClient: sync with body" { test "HttpClient: sync with gzip body" { for (0..2) |i| { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("http://127.0.0.1:9582/http_client/gzip"); @@ -2897,7 +3063,7 @@ test "HttpClient: sync tls with body" { defer arr.deinit(testing.allocator); try arr.ensureTotalCapacity(testing.allocator, 20); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); for (0..5) |_| { defer arr.clearRetainingCapacity(); @@ -2927,7 +3093,7 @@ test "HttpClient: sync redirect from TLS to Plaintext" { for (0..5) |_| { defer arr.clearRetainingCapacity(); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("https://127.0.0.1:9581/http_client/redirect/insecure"); @@ -2957,7 +3123,7 @@ test "HttpClient: sync redirect plaintext to TLS" { for (0..5) |_| { defer arr.clearRetainingCapacity(); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("http://127.0.0.1:9582/http_client/redirect/secure"); @@ -2978,7 +3144,7 @@ test "HttpClient: sync redirect plaintext to TLS" { } test "HttpClient: sync GET redirect" { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); const uri = try Uri.parse("http://127.0.0.1:9582/http_client/redirect"); @@ -3024,7 +3190,7 @@ test "HttpClient: async connect error" { }; var reset: Thread.ResetEvent = .{}; - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = Handler{ @@ -3056,7 +3222,7 @@ test "HttpClient: async connect error" { test "HttpClient: async no body" { defer testing.reset(); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = try CaptureHandler.init(); @@ -3075,7 +3241,7 @@ test "HttpClient: async no body" { test "HttpClient: async with body" { defer testing.reset(); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = try CaptureHandler.init(); @@ -3100,7 +3266,7 @@ test "HttpClient: async with body" { test "HttpClient: async with gzip body" { defer testing.reset(); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = try CaptureHandler.init(); @@ -3123,7 +3289,7 @@ test "HttpClient: async with gzip body" { test "HttpClient: async redirect" { defer testing.reset(); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = try CaptureHandler.init(); @@ -3153,7 +3319,7 @@ test "HttpClient: async redirect" { test "HttpClient: async tls no body" { defer testing.reset(); - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); for (0..5) |_| { var handler = try CaptureHandler.init(); @@ -3178,7 +3344,7 @@ test "HttpClient: async tls no body" { test "HttpClient: async tls with body" { defer testing.reset(); for (0..5) |_| { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = try CaptureHandler.init(); @@ -3202,7 +3368,7 @@ test "HttpClient: async tls with body" { test "HttpClient: async redirect from TLS to Plaintext" { defer testing.reset(); for (0..1) |_| { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = try CaptureHandler.init(); @@ -3228,7 +3394,7 @@ test "HttpClient: async redirect from TLS to Plaintext" { test "HttpClient: async redirect plaintext to TLS" { defer testing.reset(); for (0..5) |_| { - var client = try testClient(); + var client = try testClient(.{}); defer client.deinit(); var handler = try CaptureHandler.init(); @@ -3441,6 +3607,8 @@ fn testReader(state: *State, res: *TestResponse, data: []const u8) !void { return error.NeverDone; } -fn testClient() !Client { - return try Client.init(testing.allocator, .{ .max_concurrent = 1 }); +fn testClient(opts: Client.Opts) !Client { + var o = opts; + o.max_concurrent = 1; + return try Client.init(testing.allocator, o); } diff --git a/src/main.zig b/src/main.zig index ef68d139..41a97373 100644 --- a/src/main.zig +++ b/src/main.zig @@ -23,6 +23,7 @@ const Allocator = std.mem.Allocator; const log = @import("log.zig"); const server = @import("server.zig"); const App = @import("app.zig").App; +const http = @import("http/client.zig"); const Platform = @import("runtime/js.zig").Platform; const Browser = @import("browser/browser.zig").Browser; @@ -83,6 +84,7 @@ fn run(alloc: Allocator) !void { var app = try App.init(alloc, .{ .run_mode = args.mode, .http_proxy = args.httpProxy(), + .proxy_type = args.proxyType(), .tls_verify_host = args.tlsVerifyHost(), }); defer app.deinit(); @@ -155,6 +157,13 @@ const Command = struct { }; } + fn proxyType(self: *const Command) ?http.ProxyType { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.proxy_type, + else => unreachable, + }; + } + fn logLevel(self: *const Command) ?log.Level { return switch (self.mode) { inline .serve, .fetch => |opts| opts.common.log_level, @@ -198,6 +207,7 @@ const Command = struct { const Common = struct { http_proxy: ?std.Uri = null, + proxy_type: ?http.ProxyType = null, tls_verify_host: bool = true, log_level: ?log.Level = null, log_format: ?log.Format = null, @@ -216,6 +226,13 @@ const Command = struct { \\--http_proxy The HTTP proxy to use for all HTTP requests. \\ Defaults to none. \\ + \\--proxy_type The type of proxy: connect, simple. + \\ 'connect' creates a tunnel through the proxy via + \\ and initial CONNECT request. + \\ 'simple' sends the full URL in the request target + \\ and expects the proxy to MITM the request. + \\ Defaults to connect when --http_proxy is set. + \\ \\--log_level The log level: debug, info, warn, error or fatal. \\ Defaults to ++ (if (builtin.mode == .Debug) " info." else "warn.") ++ @@ -456,6 +473,22 @@ fn parseCommonArg( return error.InvalidArgument; }; common.http_proxy = try std.Uri.parse(try allocator.dupe(u8, str)); + if (common.http_proxy.?.host == null) { + log.fatal(.app, "invalid http proxy", .{ .arg = "--http_proxy", .hint = "missing scheme?" }); + return error.InvalidArgument; + } + return true; + } + + if (std.mem.eql(u8, "--proxy_type", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--proxy_type" }); + return error.InvalidArgument; + }; + common.proxy_type = std.meta.stringToEnum(http.ProxyType, str) orelse { + log.fatal(.app, "invalid option choice", .{ .arg = "--proxy_type", .value = str }); + return error.InvalidArgument; + }; return true; } @@ -573,58 +606,65 @@ fn serveHTTP(address: std.net.Address) !void { var conn = try listener.accept(); defer conn.stream.close(); var http_server = std.http.Server.init(conn, &read_buffer); - - var request = http_server.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => continue :ACCEPT, - else => { - std.debug.print("Test HTTP Server error: {}\n", .{err}); - return err; - }, - }; - - const path = request.head.target; - if (std.mem.eql(u8, path, "/loader")) { - try request.respond("Hello!", .{ - .extra_headers = &.{.{ .name = "Connection", .value = "close" }}, - }); - } else if (std.mem.eql(u8, path, "/http_client/simple")) { - try request.respond("", .{ - .extra_headers = &.{.{ .name = "Connection", .value = "close" }}, - }); - } else if (std.mem.eql(u8, path, "/http_client/redirect")) { - try request.respond("", .{ - .status = .moved_permanently, - .extra_headers = &.{ - .{ .name = "Connection", .value = "close" }, - .{ .name = "LOCATION", .value = "../http_client/echo" }, + REQUEST: while (true) { + var request = http_server.receiveHead() catch |err| switch (err) { + error.HttpConnectionClosing => continue :ACCEPT, + else => { + std.debug.print("Test HTTP Server error: {}\n", .{err}); + return err; }, - }); - } else if (std.mem.eql(u8, path, "/http_client/redirect/secure")) { - try request.respond("", .{ - .status = .moved_permanently, - .extra_headers = &.{ .{ .name = "Connection", .value = "close" }, .{ .name = "LOCATION", .value = "https://127.0.0.1:9581/http_client/body" } }, - }); - } else if (std.mem.eql(u8, path, "/http_client/gzip")) { - const body = &.{ 0x1f, 0x8b, 0x08, 0x08, 0x01, 0xc6, 0x19, 0x68, 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x00, 0x73, 0x54, 0xc8, 0x4b, 0x2d, 0x57, 0x48, 0x2a, 0xca, 0x2f, 0x2f, 0x4e, 0x2d, 0x52, 0x48, 0x2a, 0xcd, 0xcc, 0x29, 0x51, 0x48, 0xcb, 0x2f, 0x52, 0xc8, 0x4d, 0x4c, 0xce, 0xc8, 0xcc, 0x4b, 0x2d, 0xe6, 0x02, 0x00, 0xe7, 0xc3, 0x4b, 0x27, 0x21, 0x00, 0x00, 0x00 }; - try request.respond(body, .{ - .extra_headers = &.{ .{ .name = "Connection", .value = "close" }, .{ .name = "Content-Encoding", .value = "gzip" } }, - }); - } else if (std.mem.eql(u8, path, "/http_client/echo")) { - var headers: std.ArrayListUnmanaged(std.http.Header) = .{}; + }; - var it = request.iterateHeaders(); - while (it.next()) |hdr| { - try headers.append(aa, .{ - .name = try std.fmt.allocPrint(aa, "_{s}", .{hdr.name}), - .value = hdr.value, + if (request.head.method == .CONNECT) { + try request.respond("", .{ .status = .ok }); + continue :REQUEST; + } + + const path = request.head.target; + if (std.mem.eql(u8, path, "/loader")) { + try request.respond("Hello!", .{ + .extra_headers = &.{.{ .name = "Connection", .value = "close" }}, + }); + } else if (std.mem.eql(u8, path, "/http_client/simple")) { + try request.respond("", .{ + .extra_headers = &.{.{ .name = "Connection", .value = "close" }}, + }); + } else if (std.mem.eql(u8, path, "/http_client/redirect")) { + try request.respond("", .{ + .status = .moved_permanently, + .extra_headers = &.{ + .{ .name = "Connection", .value = "close" }, + .{ .name = "LOCATION", .value = "../http_client/echo" }, + }, + }); + } else if (std.mem.eql(u8, path, "/http_client/redirect/secure")) { + try request.respond("", .{ + .status = .moved_permanently, + .extra_headers = &.{ .{ .name = "Connection", .value = "close" }, .{ .name = "LOCATION", .value = "https://127.0.0.1:9581/http_client/body" } }, + }); + } else if (std.mem.eql(u8, path, "/http_client/gzip")) { + const body = &.{ 0x1f, 0x8b, 0x08, 0x08, 0x01, 0xc6, 0x19, 0x68, 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x00, 0x73, 0x54, 0xc8, 0x4b, 0x2d, 0x57, 0x48, 0x2a, 0xca, 0x2f, 0x2f, 0x4e, 0x2d, 0x52, 0x48, 0x2a, 0xcd, 0xcc, 0x29, 0x51, 0x48, 0xcb, 0x2f, 0x52, 0xc8, 0x4d, 0x4c, 0xce, 0xc8, 0xcc, 0x4b, 0x2d, 0xe6, 0x02, 0x00, 0xe7, 0xc3, 0x4b, 0x27, 0x21, 0x00, 0x00, 0x00 }; + try request.respond(body, .{ + .extra_headers = &.{ .{ .name = "Connection", .value = "close" }, .{ .name = "Content-Encoding", .value = "gzip" } }, + }); + } else if (std.mem.eql(u8, path, "/http_client/echo")) { + var headers: std.ArrayListUnmanaged(std.http.Header) = .{}; + + var it = request.iterateHeaders(); + while (it.next()) |hdr| { + try headers.append(aa, .{ + .name = try std.fmt.allocPrint(aa, "_{s}", .{hdr.name}), + .value = hdr.value, + }); + } + try headers.append(aa, .{ .name = "Connection", .value = "Close" }); + + try request.respond("over 9000!", .{ + .status = .created, + .extra_headers = headers.items, }); } - try headers.append(aa, .{ .name = "Connection", .value = "Close" }); - - try request.respond("over 9000!", .{ - .status = .created, - .extra_headers = headers.items, - }); + continue :ACCEPT; } } } From 4560f31010fdb6881db3828860d40cd0645eefd7 Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Tue, 24 Jun 2025 16:37:11 +0200 Subject: [PATCH 2/7] basic/bearer proxy authentication --- src/app.zig | 2 ++ src/http/client.zig | 54 ++++++++++++++++++++++++++++++++++++++++++--- src/main.zig | 42 +++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/app.zig b/src/app.zig index 30489755..841aaf7f 100644 --- a/src/app.zig +++ b/src/app.zig @@ -31,6 +31,7 @@ pub const App = struct { tls_verify_host: bool = true, http_proxy: ?std.Uri = null, proxy_type: ?http.ProxyType = null, + proxy_auth: ?http.ProxyAuth = null, }; pub fn init(allocator: Allocator, config: Config) !*App { @@ -58,6 +59,7 @@ pub const App = struct { .max_concurrent = 3, .http_proxy = config.http_proxy, .proxy_type = config.proxy_type, + .proxy_auth = config.proxy_auth, .tls_verify_host = config.tls_verify_host, }), .config = config, diff --git a/src/http/client.zig b/src/http/client.zig index 859e2827..bc210553 100644 --- a/src/http/client.zig +++ b/src/http/client.zig @@ -46,6 +46,34 @@ pub const ProxyType = enum { connect, }; +pub const ProxyAuth = union(enum) { + basic: struct { user_pass: []const u8 }, + bearer: struct { token: []const u8 }, + + pub fn header_value(self: ProxyAuth, allocator: Allocator) ![]const u8 { + switch (self) { + .basic => |*auth| { + if (std.mem.indexOfScalar(u8, auth.user_pass, ':') == null) return error.InvalidProxyAuth; + + const prefix = "Basic "; + var encoder = std.base64.standard.Encoder; + const size = encoder.calcSize(auth.user_pass.len); + var buffer = try allocator.alloc(u8, size + prefix.len); + std.mem.copyForwards(u8, buffer, prefix); + _ = std.base64.standard.Encoder.encode(buffer[prefix.len..], auth.user_pass); + return buffer; + }, + .bearer => |*auth| { + const prefix = "Bearer "; + var buffer = try allocator.alloc(u8, auth.token.len + prefix.len); + std.mem.copyForwards(u8, buffer, prefix); + std.mem.copyForwards(u8, buffer[prefix.len..], auth.token); + return buffer; + }, + } + } +}; + // Thread-safe. Holds our root certificate, connection pool and state pool // Used to create Requests. pub const Client = struct { @@ -54,6 +82,7 @@ pub const Client = struct { state_pool: StatePool, http_proxy: ?Uri, proxy_type: ?ProxyType, + proxy_auth: ?[]const u8, // Basic or Bearer root_ca: tls.config.CertBundle, tls_verify_host: bool = true, connection_manager: ConnectionManager, @@ -63,6 +92,7 @@ pub const Client = struct { max_concurrent: usize = 3, http_proxy: ?std.Uri = null, proxy_type: ?ProxyType = null, + proxy_auth: ?ProxyAuth = null, tls_verify_host: bool = true, max_idle_connection: usize = 10, }; @@ -71,10 +101,10 @@ pub const Client = struct { var root_ca: tls.config.CertBundle = if (builtin.is_test) .{} else try tls.config.CertBundle.fromSystem(allocator); errdefer root_ca.deinit(allocator); - const state_pool = try StatePool.init(allocator, opts.max_concurrent); + var state_pool = try StatePool.init(allocator, opts.max_concurrent); errdefer state_pool.deinit(allocator); - const connection_manager = ConnectionManager.init(allocator, opts.max_idle_connection); + var connection_manager = ConnectionManager.init(allocator, opts.max_idle_connection); errdefer connection_manager.deinit(); return .{ @@ -84,6 +114,7 @@ pub const Client = struct { .state_pool = state_pool, .http_proxy = opts.http_proxy, .proxy_type = if (opts.http_proxy == null) null else (opts.proxy_type orelse .connect), + .proxy_auth = if (opts.proxy_auth) |*auth| try auth.header_value(allocator) else null, .tls_verify_host = opts.tls_verify_host, .connection_manager = connection_manager, .request_pool = std.heap.MemoryPool(Request).init(allocator), @@ -98,6 +129,10 @@ pub const Client = struct { self.state_pool.deinit(allocator); self.connection_manager.deinit(); self.request_pool.deinit(); + + if (self.proxy_auth) |auth| { + allocator.free(auth); + } } pub fn request(self: *Client, method: Request.Method, uri: *const Uri) !*Request { @@ -763,6 +798,13 @@ pub const Request = struct { try self.headers.append(arena, .{ .name = "User-Agent", .value = "Lightpanda/1.0" }); try self.headers.append(arena, .{ .name = "Accept", .value = "*/*" }); + + if (self._client.isSimpleProxy()) { + if (self._client.proxy_auth) |proxy_auth| { + try self.headers.append(arena, .{ .name = "Proxy-Authorization", .value = proxy_auth }); + } + } + self.requestStarting(); } @@ -887,7 +929,13 @@ pub const Request = struct { var writer = fbs.writer(); try writer.print("CONNECT {s}:{d} HTTP/1.1\r\n", .{ self._request_host, self._request_port }); - try writer.print("Host: {s}:{d}\r\n\r\n", .{ self._request_host, self._request_port }); + try writer.print("Host: {s}:{d}\r\n", .{ self._request_host, self._request_port }); + + if (self._client.proxy_auth) |proxy_auth| { + try writer.print("Proxy-Authorization: {s}\r\n", .{proxy_auth}); + } + + _ = try writer.write("\r\n"); return buf[0..fbs.pos]; } diff --git a/src/main.zig b/src/main.zig index 41a97373..327be9ca 100644 --- a/src/main.zig +++ b/src/main.zig @@ -85,6 +85,7 @@ fn run(alloc: Allocator) !void { .run_mode = args.mode, .http_proxy = args.httpProxy(), .proxy_type = args.proxyType(), + .proxy_auth = args.proxyAuth(), .tls_verify_host = args.tlsVerifyHost(), }); defer app.deinit(); @@ -164,6 +165,13 @@ const Command = struct { }; } + fn proxyAuth(self: *const Command) ?http.ProxyAuth { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.proxy_auth, + else => unreachable, + }; + } + fn logLevel(self: *const Command) ?log.Level { return switch (self.mode) { inline .serve, .fetch => |opts| opts.common.log_level, @@ -208,6 +216,7 @@ const Command = struct { const Common = struct { http_proxy: ?std.Uri = null, proxy_type: ?http.ProxyType = null, + proxy_auth: ?http.ProxyAuth = null, tls_verify_host: bool = true, log_level: ?log.Level = null, log_format: ?log.Format = null, @@ -233,6 +242,14 @@ const Command = struct { \\ and expects the proxy to MITM the request. \\ Defaults to connect when --http_proxy is set. \\ + \\--proxy_bearer_token + \\ The token to send for bearer authentication with the proxy + \\ Proxy-Authorization: Bearer + \\ + \\--proxy_basic_auth + \\ The user:password to send for basic authentication with the proxy + \\ Proxy-Authorization: Basic + \\ \\--log_level The log level: debug, info, warn, error or fatal. \\ Defaults to ++ (if (builtin.mode == .Debug) " info." else "warn.") ++ @@ -492,6 +509,31 @@ fn parseCommonArg( return true; } + if (std.mem.eql(u8, "--proxy_bearer_token", opt)) { + if (common.proxy_auth != null) { + log.fatal(.app, "proxy auth already set", .{ .arg = "--proxy_bearer_token" }); + return error.InvalidArgument; + } + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--proxy_bearer_token" }); + return error.InvalidArgument; + }; + common.proxy_auth = .{ .bearer = .{ .token = str } }; + return true; + } + if (std.mem.eql(u8, "--proxy_basic_auth", opt)) { + if (common.proxy_auth != null) { + log.fatal(.app, "proxy auth already set", .{ .arg = "--proxy_basic_auth" }); + return error.InvalidArgument; + } + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--proxy_basic_auth" }); + return error.InvalidArgument; + }; + common.proxy_auth = .{ .basic = .{ .user_pass = str } }; + return true; + } + if (std.mem.eql(u8, "--log_level", opt)) { const str = args.next() orelse { log.fatal(.app, "missing argument value", .{ .arg = "--log_level" }); From 2aa5eb85ada16e077d5d0c7a6852fbc7a91cf162 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 24 Jun 2025 20:08:14 +0800 Subject: [PATCH 3/7] Add element.dataset API Uses the State to store the dataset, but, on first load, loads the data attributes from the DOM. --- src/browser/State.zig | 2 + src/browser/html/DataSet.zig | 92 +++++++++++++++++++++++++++ src/browser/html/elements.zig | 38 +++++++++++ src/browser/html/html.zig | 1 + src/runtime/js.zig | 115 ++++++++++++++++++++++++---------- 5 files changed, 216 insertions(+), 32 deletions(-) create mode 100644 src/browser/html/DataSet.zig diff --git a/src/browser/State.zig b/src/browser/State.zig index d594d943..90e5494d 100644 --- a/src/browser/State.zig +++ b/src/browser/State.zig @@ -28,6 +28,7 @@ const Env = @import("env.zig").Env; const parser = @import("netsurf.zig"); +const DataSet = @import("html/DataSet.zig"); const CSSStyleDeclaration = @import("cssom/css_style_declaration.zig").CSSStyleDeclaration; // for HTMLScript (but probably needs to be added to more) @@ -36,6 +37,7 @@ onerror: ?Env.Function = null, // for HTMLElement style: CSSStyleDeclaration = .empty, +dataset: ?DataSet = null, // for html/document ready_state: ReadyState = .loading, diff --git a/src/browser/html/DataSet.zig b/src/browser/html/DataSet.zig new file mode 100644 index 00000000..40ba0417 --- /dev/null +++ b/src/browser/html/DataSet.zig @@ -0,0 +1,92 @@ +// Copyright (C) 2023-2024 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +const std = @import("std"); +const Page = @import("../page.zig").Page; +const Allocator = std.mem.Allocator; + +const DataSet = @This(); + +attributes: std.StringHashMapUnmanaged([]const u8), + +pub const empty: DataSet = .{ + .attributes = .empty, +}; + +const GetResult = union(enum) { + value: []const u8, + undefined: void, +}; +pub fn named_get(self: *const DataSet, name: []const u8, _: *bool) GetResult { + if (self.attributes.get(name)) |value| { + return .{ .value = value }; + } + return .{ .undefined = {} }; +} + +pub fn named_set(self: *DataSet, name: []const u8, value: []const u8, _: *bool, page: *Page) !void { + const arena = page.arena; + const gop = try self.attributes.getOrPut(arena, name); + errdefer _ = self.attributes.remove(name); + + if (!gop.found_existing) { + gop.key_ptr.* = try arena.dupe(u8, name); + } + gop.value_ptr.* = try arena.dupe(u8, value); +} + +pub fn named_delete(self: *DataSet, name: []const u8, _: *bool) void { + _ = self.attributes.remove(name); +} + +pub fn normalizeName(allocator: Allocator, name: []const u8) ![]const u8 { + std.debug.assert(std.mem.startsWith(u8, name, "data-")); + var owned = try allocator.alloc(u8, name.len - 5); + + var pos: usize = 0; + var capitalize = false; + for (name[5..]) |c| { + if (c == '-') { + capitalize = true; + continue; + } + + if (capitalize) { + capitalize = false; + owned[pos] = std.ascii.toUpper(c); + } else { + owned[pos] = c; + } + pos += 1; + } + return owned[0..pos]; +} + +const testing = @import("../../testing.zig"); +test "Browser.HTML.DataSet" { + var runner = try testing.jsRunner(testing.tracking_allocator, .{ .html = "" }); + defer runner.deinit(); + + try runner.testCases(&.{ + .{ "let el1 = document.createElement('div')", null }, + .{ "el1.dataset.x", "undefined" }, + .{ "el1.dataset.x = '123'", "123" }, + .{ "delete el1.dataset.x", "true" }, + .{ "el1.dataset.x", "undefined" }, + .{ "delete el1.dataset.other", "true" }, // yes, this is right + }, .{}); +} diff --git a/src/browser/html/elements.zig b/src/browser/html/elements.zig index 28179fee..a0ff8568 100644 --- a/src/browser/html/elements.zig +++ b/src/browser/html/elements.zig @@ -27,6 +27,7 @@ const urlStitch = @import("../../url.zig").URL.stitch; const URL = @import("../url/url.zig").URL; const Node = @import("../dom/node.zig").Node; const Element = @import("../dom/element.zig").Element; +const DataSet = @import("DataSet.zig"); const CSSStyleDeclaration = @import("../cssom/css_style_declaration.zig").CSSStyleDeclaration; @@ -122,6 +123,36 @@ pub const HTMLElement = struct { return &state.style; } + pub fn get_dataset(e: *parser.ElementHTML, page: *Page) !*DataSet { + const state = try page.getOrCreateNodeState(@ptrCast(e)); + if (state.dataset) |*ds| { + return ds; + } + + // The first time this is called, load the data attributes from the DOM + var ds: DataSet = .empty; + + if (try parser.nodeGetAttributes(@ptrCast(e))) |map| { + const arena = page.arena; + const count = try parser.namedNodeMapGetLength(map); + for (0..count) |i| { + const attr = try parser.namedNodeMapItem(map, @intCast(i)) orelse continue; + const name = try parser.attributeGetName(attr); + if (!std.mem.startsWith(u8, name, "data-")) { + continue; + } + const normalized_name = try DataSet.normalizeName(arena, name); + const value = try parser.attributeGetValue(attr) orelse ""; + // I don't think we need to dupe value, It'll live in libdom for + // as long as the page due to the fact that we're using an arena. + try ds.attributes.put(arena, normalized_name, value); + } + } + + state.dataset = ds; + return &state.dataset.?; + } + pub fn get_innerText(e: *parser.ElementHTML) ![]const u8 { const n = @as(*parser.Node, @ptrCast(e)); return try parser.nodeTextContent(n) orelse ""; @@ -1561,6 +1592,13 @@ test "Browser.HTML.Element" { }, .{}); } +test "Browser.HTML.Element.DataSet" { + var runner = try testing.jsRunner(testing.tracking_allocator, .{ .html = "
" }); + defer runner.deinit(); + + try runner.testCases(&.{ .{ "let div = document.getElementById('x')", null }, .{ "div.dataset.nope", "undefined" }, .{ "div.dataset.power", "over 9000" }, .{ "div.dataset.empty", "" }, .{ "div.dataset.someLongKey", "ok" }, .{ "delete div.dataset.power", "true" }, .{ "div.dataset.power", "undefined" } }, .{}); +} + test "Browser.HTML.HtmlInputElement.properties" { var runner = try testing.jsRunner(testing.tracking_allocator, .{ .url = "https://lightpanda.io/noslashattheend" }); defer runner.deinit(); diff --git a/src/browser/html/html.zig b/src/browser/html/html.zig index 1ba0a340..f6134bf8 100644 --- a/src/browser/html/html.zig +++ b/src/browser/html/html.zig @@ -36,6 +36,7 @@ pub const Interfaces = .{ History, Location, MediaQueryList, + @import("DataSet.zig"), @import("screen.zig").Interfaces, @import("error_event.zig").ErrorEvent, }; diff --git a/src/runtime/js.zig b/src/runtime/js.zig index cc48175b..150b97e1 100644 --- a/src/runtime/js.zig +++ b/src/runtime/js.zig @@ -1931,7 +1931,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type { } generateIndexer(Struct, template_proto); - generateNamedIndexer(Struct, template_proto); + generateNamedIndexer(Struct, template.getInstanceTemplate()); generateUndetectable(Struct, template.getInstanceTemplate()); } @@ -2116,7 +2116,8 @@ pub fn Env(comptime State: type, comptime WebApis: type) type { } return; } - const configuration = v8.NamedPropertyHandlerConfiguration{ + + var configuration = v8.NamedPropertyHandlerConfiguration{ .getter = struct { fn callback(c_name: ?*const v8.C_Name, raw_info: ?*const v8.C_PropertyCallbackInfo) callconv(.c) u8 { const info = v8.PropertyCallbackInfo.initFromV8(raw_info); @@ -2138,13 +2139,37 @@ pub fn Env(comptime State: type, comptime WebApis: type) type { .flags = v8.PropertyHandlerFlags.OnlyInterceptStrings | v8.PropertyHandlerFlags.NonMasking, }; - // If you're trying to implement setter, read: - // https://groups.google.com/g/v8-users/c/8tahYBsHpgY/m/IteS7Wn2AAAJ - // The issue I had was - // (a) where to attache it: does it go ont he instance_template - // instead of the prototype? - // (b) defining the getter or query to respond with the - // PropertyAttribute to indicate if the property can be set + if (@hasDecl(Struct, "named_set")) { + configuration.setter = struct { + fn callback(c_name: ?*const v8.C_Name, c_value: ?*const v8.C_Value, raw_info: ?*const v8.C_PropertyCallbackInfo) callconv(.c) u8 { + const info = v8.PropertyCallbackInfo.initFromV8(raw_info); + var caller = Caller(Self, State).init(info); + defer caller.deinit(); + + const named_function = comptime NamedFunction.init(Struct, "named_set"); + return caller.setNamedIndex(Struct, named_function, .{ .handle = c_name.? }, .{ .handle = c_value.? }, info) catch |err| blk: { + caller.handleError(Struct, named_function, err, info); + break :blk v8.Intercepted.No; + }; + } + }.callback; + } + + if (@hasDecl(Struct, "named_delete")) { + configuration.deleter = struct { + fn callback(c_name: ?*const v8.C_Name, raw_info: ?*const v8.C_PropertyCallbackInfo) callconv(.c) u8 { + const info = v8.PropertyCallbackInfo.initFromV8(raw_info); + var caller = Caller(Self, State).init(info); + defer caller.deinit(); + + const named_function = comptime NamedFunction.init(Struct, "named_delete"); + return caller.deleteNamedIndex(Struct, named_function, .{ .handle = c_name.? }, info) catch |err| blk: { + caller.handleError(Struct, named_function, err, info); + break :blk v8.Intercepted.No; + }; + } + }.callback; + } template_proto.setNamedProperty(configuration, null); } @@ -2646,37 +2671,63 @@ fn Caller(comptime E: type, comptime State: type) type { } fn getNamedIndex(self: *Self, comptime Struct: type, comptime named_function: NamedFunction, name: v8.Name, info: v8.PropertyCallbackInfo) !u8 { - const js_context = self.js_context; const func = @field(Struct, named_function.name); - const NamedGet = @TypeOf(func); - if (@typeInfo(NamedGet).@"fn".return_type == null) { - @compileError(named_function.full_name ++ " must have a return type"); - } + comptime assertSelfReceiver(Struct, named_function); var has_value = true; - var args: ParamterTypes(NamedGet) = undefined; - const arg_fields = @typeInfo(@TypeOf(args)).@"struct".fields; - switch (arg_fields.len) { - 0, 1, 2 => @compileError(named_function.full_name ++ " must take at least a u32 and *bool parameter"), - 3, 4 => { - const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis()); - comptime assertSelfReceiver(Struct, named_function); - @field(args, "0") = zig_instance; - @field(args, "1") = try self.nameToString(name); - @field(args, "2") = &has_value; - if (comptime arg_fields.len == 4) { - comptime assertIsStateArg(Struct, named_function, 3); - @field(args, "3") = js_context.state; - } - }, - else => @compileError(named_function.full_name ++ " has too many parmaters"), - } + var args = try self.getArgs(Struct, named_function, 3, info); + const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis()); + @field(args, "0") = zig_instance; + @field(args, "1") = try self.nameToString(name); + @field(args, "2") = &has_value; const res = @call(.auto, func, args); if (has_value == false) { return v8.Intercepted.No; } - info.getReturnValue().set(try js_context.zigValueToJs(res)); + info.getReturnValue().set(try self.js_context.zigValueToJs(res)); + return v8.Intercepted.Yes; + } + + fn setNamedIndex(self: *Self, comptime Struct: type, comptime named_function: NamedFunction, name: v8.Name, js_value: v8.Value, info: v8.PropertyCallbackInfo) !u8 { + const js_context = self.js_context; + const func = @field(Struct, named_function.name); + comptime assertSelfReceiver(Struct, named_function); + + var has_value = true; + var args = try self.getArgs(Struct, named_function, 4, info); + const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis()); + @field(args, "0") = zig_instance; + @field(args, "1") = try self.nameToString(name); + @field(args, "2") = try js_context.jsValueToZig(named_function, @TypeOf(@field(args, "2")), js_value); + @field(args, "3") = &has_value; + + const res = @call(.auto, func, args); + return namedSetOrDeleteCall(res, has_value); + } + + fn deleteNamedIndex(self: *Self, comptime Struct: type, comptime named_function: NamedFunction, name: v8.Name, info: v8.PropertyCallbackInfo) !u8 { + const func = @field(Struct, named_function.name); + comptime assertSelfReceiver(Struct, named_function); + + var has_value = true; + var args = try self.getArgs(Struct, named_function, 3, info); + const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis()); + @field(args, "0") = zig_instance; + @field(args, "1") = try self.nameToString(name); + @field(args, "2") = &has_value; + + const res = @call(.auto, func, args); + return namedSetOrDeleteCall(res, has_value); + } + + fn namedSetOrDeleteCall(res: anytype, has_value: bool) !u8 { + if (@typeInfo(@TypeOf(res)) == .error_union) { + _ = try res; + } + if (has_value == false) { + return v8.Intercepted.No; + } return v8.Intercepted.Yes; } From ec92f110b3cf06a0f8f1fdba658c9cd045df7bcf Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 25 Jun 2025 12:15:18 +0800 Subject: [PATCH 4/7] Change dataset to work directly off DOM element --- src/browser/html/DataSet.zig | 72 +++++++++++++++++++---------------- src/browser/html/elements.zig | 23 +---------- 2 files changed, 41 insertions(+), 54 deletions(-) diff --git a/src/browser/html/DataSet.zig b/src/browser/html/DataSet.zig index 40ba0417..df17c9dc 100644 --- a/src/browser/html/DataSet.zig +++ b/src/browser/html/DataSet.zig @@ -16,64 +16,66 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . const std = @import("std"); +const parser = @import("../netsurf.zig"); const Page = @import("../page.zig").Page; + const Allocator = std.mem.Allocator; const DataSet = @This(); -attributes: std.StringHashMapUnmanaged([]const u8), - -pub const empty: DataSet = .{ - .attributes = .empty, -}; +element: *parser.Element, const GetResult = union(enum) { value: []const u8, undefined: void, }; -pub fn named_get(self: *const DataSet, name: []const u8, _: *bool) GetResult { - if (self.attributes.get(name)) |value| { +pub fn named_get(self: *const DataSet, name: []const u8, _: *bool, page: *Page) !GetResult { + const normalized_name = try normalize(page.call_arena, name); + if (try parser.elementGetAttribute(self.element, normalized_name)) |value| { return .{ .value = value }; } return .{ .undefined = {} }; } pub fn named_set(self: *DataSet, name: []const u8, value: []const u8, _: *bool, page: *Page) !void { - const arena = page.arena; - const gop = try self.attributes.getOrPut(arena, name); - errdefer _ = self.attributes.remove(name); - - if (!gop.found_existing) { - gop.key_ptr.* = try arena.dupe(u8, name); - } - gop.value_ptr.* = try arena.dupe(u8, value); + const normalized_name = try normalize(page.call_arena, name); + try parser.elementSetAttribute(self.element, normalized_name, value); } -pub fn named_delete(self: *DataSet, name: []const u8, _: *bool) void { - _ = self.attributes.remove(name); +pub fn named_delete(self: *DataSet, name: []const u8, _: *bool, page: *Page) !void { + const normalized_name = try normalize(page.call_arena, name); + try parser.elementRemoveAttribute(self.element, normalized_name); } -pub fn normalizeName(allocator: Allocator, name: []const u8) ![]const u8 { - std.debug.assert(std.mem.startsWith(u8, name, "data-")); - var owned = try allocator.alloc(u8, name.len - 5); - - var pos: usize = 0; - var capitalize = false; - for (name[5..]) |c| { - if (c == '-') { - capitalize = true; - continue; +fn normalize(allocator: Allocator, name: []const u8) ![]const u8 { + var upper_count: usize = 0; + for (name) |c| { + if (std.ascii.isUpper(c)) { + upper_count += 1; } + } + // for every upper-case letter, we'll probably need a dash before it + // and we need the 'data-' prefix + var normalized = try allocator.alloc(u8, name.len + upper_count + 5); - if (capitalize) { - capitalize = false; - owned[pos] = std.ascii.toUpper(c); + @memcpy(normalized[0..5], "data-"); + if (upper_count == 0) { + @memcpy(normalized[5..], name); + return normalized; + } + + var pos: usize = 5; + for (name) |c| { + if (std.ascii.isUpper(c)) { + normalized[pos] = '-'; + pos += 1; + normalized[pos] = c + 32; } else { - owned[pos] = c; + normalized[pos] = c; } pos += 1; } - return owned[0..pos]; + return normalized; } const testing = @import("../../testing.zig"); @@ -88,5 +90,11 @@ test "Browser.HTML.DataSet" { .{ "delete el1.dataset.x", "true" }, .{ "el1.dataset.x", "undefined" }, .{ "delete el1.dataset.other", "true" }, // yes, this is right + + .{ "let ds1 = el1.dataset", null }, + .{ "ds1.helloWorld = 'yes'", null }, + .{ "el1.getAttribute('data-hello-world')", "yes" }, + .{ "el1.setAttribute('data-this-will-work', 'positive')", null }, + .{ "ds1.thisWillWork", "positive" }, }, .{}); } diff --git a/src/browser/html/elements.zig b/src/browser/html/elements.zig index a0ff8568..40a3b2b7 100644 --- a/src/browser/html/elements.zig +++ b/src/browser/html/elements.zig @@ -128,28 +128,7 @@ pub const HTMLElement = struct { if (state.dataset) |*ds| { return ds; } - - // The first time this is called, load the data attributes from the DOM - var ds: DataSet = .empty; - - if (try parser.nodeGetAttributes(@ptrCast(e))) |map| { - const arena = page.arena; - const count = try parser.namedNodeMapGetLength(map); - for (0..count) |i| { - const attr = try parser.namedNodeMapItem(map, @intCast(i)) orelse continue; - const name = try parser.attributeGetName(attr); - if (!std.mem.startsWith(u8, name, "data-")) { - continue; - } - const normalized_name = try DataSet.normalizeName(arena, name); - const value = try parser.attributeGetValue(attr) orelse ""; - // I don't think we need to dupe value, It'll live in libdom for - // as long as the page due to the fact that we're using an arena. - try ds.attributes.put(arena, normalized_name, value); - } - } - - state.dataset = ds; + state.dataset = DataSet{ .element = @ptrCast(e) }; return &state.dataset.?; } From 1e7ee4e0a1ea57918e7cef2149be0606c08af6bb Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 25 Jun 2025 12:21:44 +0800 Subject: [PATCH 5/7] proxy_type 'simple' renamed to 'forward' --- src/http/client.zig | 4 ++-- src/main.zig | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/http/client.zig b/src/http/client.zig index 859e2827..d1d4227e 100644 --- a/src/http/client.zig +++ b/src/http/client.zig @@ -42,7 +42,7 @@ const BUFFER_LEN = 32 * 1024; const MAX_HEADER_LINE_LEN = 4096; pub const ProxyType = enum { - simple, + forward, connect, }; @@ -202,7 +202,7 @@ pub const Client = struct { fn isSimpleProxy(self: *const Client) bool { const proxy_type = self.proxy_type orelse return false; - return proxy_type == .simple; + return proxy_type == .forward; } }; diff --git a/src/main.zig b/src/main.zig index 41a97373..c26d4082 100644 --- a/src/main.zig +++ b/src/main.zig @@ -226,10 +226,10 @@ const Command = struct { \\--http_proxy The HTTP proxy to use for all HTTP requests. \\ Defaults to none. \\ - \\--proxy_type The type of proxy: connect, simple. + \\--proxy_type The type of proxy: connect, forward. \\ 'connect' creates a tunnel through the proxy via \\ and initial CONNECT request. - \\ 'simple' sends the full URL in the request target + \\ 'forward' sends the full URL in the request target \\ and expects the proxy to MITM the request. \\ Defaults to connect when --http_proxy is set. \\ From 9c4088b24c240468e2ea353a367dc645470ccc98 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 25 Jun 2025 14:58:09 +0800 Subject: [PATCH 6/7] We cannot have empty Zig structs mapping to JS instances An empty struct will share the same address as its sibling (1) which will cause an collision in the identity map. (1) - This depends on Zig's non-guaranteed layout, so the collision might not be with its sibling, but rather some other [seemingly random] field. --- src/browser/crypto/crypto.zig | 2 ++ src/browser/css/css.zig | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/browser/crypto/crypto.zig b/src/browser/crypto/crypto.zig index bd9eac52..92d4df8c 100644 --- a/src/browser/crypto/crypto.zig +++ b/src/browser/crypto/crypto.zig @@ -21,6 +21,8 @@ const uuidv4 = @import("../../id.zig").uuidv4; // https://w3c.github.io/webcrypto/#crypto-interface pub const Crypto = struct { + _not_empty: bool = true, + pub fn _getRandomValues(_: *const Crypto, into: RandomValues) !RandomValues { const buf = into.asBuffer(); if (buf.len > 65_536) { diff --git a/src/browser/css/css.zig b/src/browser/css/css.zig index 6fda4782..e9d67735 100644 --- a/src/browser/css/css.zig +++ b/src/browser/css/css.zig @@ -29,6 +29,8 @@ pub const Interfaces = .{ // https://developer.mozilla.org/en-US/docs/Web/API/CSS pub const Css = struct { + _not_empty: bool = true, + pub fn _supports(_: *Css, _: []const u8, _: ?[]const u8) bool { // TODO: Actually respond with which CSS features we support. return true; From aea34264a99a58a3bfbb6ff352bb9d8f9ffa9e66 Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Wed, 25 Jun 2025 11:34:32 +0200 Subject: [PATCH 7/7] basic/bearer testing --- src/http/client.zig | 49 +++++++++++++++++++++++++++++++++++++++++---- src/main.zig | 16 +++++++++++++++ src/testing.zig | 5 ++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/http/client.zig b/src/http/client.zig index bc210553..b45f2b19 100644 --- a/src/http/client.zig +++ b/src/http/client.zig @@ -59,15 +59,15 @@ pub const ProxyAuth = union(enum) { var encoder = std.base64.standard.Encoder; const size = encoder.calcSize(auth.user_pass.len); var buffer = try allocator.alloc(u8, size + prefix.len); - std.mem.copyForwards(u8, buffer, prefix); + @memcpy(buffer[0..prefix.len], prefix); _ = std.base64.standard.Encoder.encode(buffer[prefix.len..], auth.user_pass); return buffer; }, .bearer => |*auth| { const prefix = "Bearer "; var buffer = try allocator.alloc(u8, auth.token.len + prefix.len); - std.mem.copyForwards(u8, buffer, prefix); - std.mem.copyForwards(u8, buffer[prefix.len..], auth.token); + @memcpy(buffer[0..prefix.len], prefix); + @memcpy(buffer[prefix.len..], auth.token); return buffer; }, } @@ -3078,15 +3078,56 @@ test "HttpClient: sync with body proxy CONNECT" { } try testing.expectEqual("over 9000!", try res.next()); try testing.expectEqual(201, res.header.status); - try testing.expectEqual(5, res.header.count()); + try testing.expectEqual(6, res.header.count()); try testing.expectEqual("Close", res.header.get("connection")); try testing.expectEqual("10", res.header.get("content-length")); try testing.expectEqual("127.0.0.1", res.header.get("_host")); try testing.expectEqual("Lightpanda/1.0", res.header.get("_user-agent")); try testing.expectEqual("*/*", res.header.get("_accept")); + // Proxy headers + try testing.expectEqual("127.0.0.1:9582", res.header.get("__host")); } } +test "HttpClient: basic authentication CONNECT" { + const proxy_uri = try Uri.parse("http://127.0.0.1:9582/"); + var client = try testClient(.{ .proxy_type = .connect, .http_proxy = proxy_uri, .proxy_auth = .{ .basic = .{ .user_pass = "user:pass" } } }); + defer client.deinit(); + + const uri = try Uri.parse("http://127.0.0.1:9582/http_client/echo"); + var req = try client.request(.GET, &uri); + defer req.deinit(); + + var res = try req.sendSync(.{}); + + try testing.expectEqual(201, res.header.status); + // Destination headers + try testing.expectEqual(null, res.header.get("_authorization")); + try testing.expectEqual(null, res.header.get("_proxy-authorization")); + // Proxy headers + try testing.expectEqual(null, res.header.get("__authorization")); + try testing.expectEqual("Basic dXNlcjpwYXNz", res.header.get("__proxy-authorization")); +} +test "HttpClient: bearer authentication CONNECT" { + const proxy_uri = try Uri.parse("http://127.0.0.1:9582/"); + var client = try testClient(.{ .proxy_type = .connect, .http_proxy = proxy_uri, .proxy_auth = .{ .bearer = .{ .token = "fruitsalad" } } }); + defer client.deinit(); + + const uri = try Uri.parse("http://127.0.0.1:9582/http_client/echo"); + var req = try client.request(.GET, &uri); + defer req.deinit(); + + var res = try req.sendSync(.{}); + + try testing.expectEqual(201, res.header.status); + // Destination headers + try testing.expectEqual(null, res.header.get("_authorization")); + try testing.expectEqual(null, res.header.get("_proxy-authorization")); + // Proxy headers + try testing.expectEqual(null, res.header.get("__authorization")); + try testing.expectEqual("Bearer fruitsalad", res.header.get("__proxy-authorization")); +} + test "HttpClient: sync with gzip body" { for (0..2) |i| { var client = try testClient(.{}); diff --git a/src/main.zig b/src/main.zig index 327be9ca..5e3a26de 100644 --- a/src/main.zig +++ b/src/main.zig @@ -648,6 +648,7 @@ fn serveHTTP(address: std.net.Address) !void { var conn = try listener.accept(); defer conn.stream.close(); var http_server = std.http.Server.init(conn, &read_buffer); + var connect_headers: std.ArrayListUnmanaged(std.http.Header) = .{}; REQUEST: while (true) { var request = http_server.receiveHead() catch |err| switch (err) { error.HttpConnectionClosing => continue :ACCEPT, @@ -659,6 +660,16 @@ fn serveHTTP(address: std.net.Address) !void { if (request.head.method == .CONNECT) { try request.respond("", .{ .status = .ok }); + + // Proxy headers and destination headers are separated in the case of a CONNECT proxy + // We store the CONNECT headers, then continue with the request for the destination + var it = request.iterateHeaders(); + while (it.next()) |hdr| { + try connect_headers.append(aa, .{ + .name = try std.fmt.allocPrint(aa, "__{s}", .{hdr.name}), + .value = try aa.dupe(u8, hdr.value), + }); + } continue :REQUEST; } @@ -699,6 +710,11 @@ fn serveHTTP(address: std.net.Address) !void { .value = hdr.value, }); } + + if (connect_headers.items.len > 0) { + try headers.appendSlice(aa, connect_headers.items); + connect_headers.clearRetainingCapacity(); + } try headers.append(aa, .{ .name = "Connection", .value = "Close" }); try request.respond("over 9000!", .{ diff --git a/src/testing.zig b/src/testing.zig index 843240bb..f5e9c20b 100644 --- a/src/testing.zig +++ b/src/testing.zig @@ -66,7 +66,10 @@ pub fn expectEqual(expected: anytype, actual: anytype) !void { if (@typeInfo(@TypeOf(expected)) == .null) { return std.testing.expectEqual(null, actual); } - return expectEqual(expected, actual.?); + if (actual) |_actual| { + return expectEqual(expected, _actual); + } + return std.testing.expectEqual(expected, null); }, .@"union" => |union_info| { if (union_info.tag_type == null) {