diff --git a/src/async/Client.zig b/src/async/Client.zig index 716905cf..0c3f0645 100644 --- a/src/async/Client.zig +++ b/src/async/Client.zig @@ -21,9 +21,6 @@ //! Connections are opened in a thread-safe manner, but individual Requests are not. //! //! TLS support may be disabled via `std.options.http_disable_tls`. -//! -//! This file is a copy of the original std.http.Client with little changes to -//! handle non-blocking I/O with the jsruntime.Loop. const std = @import("std"); const builtin = @import("builtin"); @@ -45,9 +42,7 @@ const tcp = @import("tcp.zig"); pub const disable_tls = std.options.http_disable_tls; -/// Allocator used for all allocations made by the client. -/// -/// This allocator must be thread-safe. +/// Used for all client allocations. Must be thread-safe. allocator: Allocator, // std.net.Stream implementation using jsruntime Loop @@ -63,14 +58,25 @@ next_https_rescan_certs: bool = true, /// The pool of connections that can be reused (and currently in use). connection_pool: ConnectionPool = .{}, -/// This is the proxy that will handle http:// connections. It *must not* be modified when the client has any active connections. -http_proxy: ?Proxy = null, - -/// This is the proxy that will handle https:// connections. It *must not* be modified when the client has any active connections. -https_proxy: ?Proxy = null, +/// If populated, all http traffic travels through this third party. +/// This field cannot be modified while the client has active connections. +/// Pointer to externally-owned memory. +http_proxy: ?*Proxy = null, +/// If populated, all https traffic travels through this third party. +/// This field cannot be modified while the client has active connections. +/// Pointer to externally-owned memory. +https_proxy: ?*Proxy = null, /// A set of linked lists of connections that can be reused. pub const ConnectionPool = struct { + mutex: std.Thread.Mutex = .{}, + /// Open connections that are currently in use. + used: Queue = .{}, + /// Open connections that are not currently in use. + free: Queue = .{}, + free_len: usize = 0, + free_size: usize = 32, + /// The criteria for a connection to be considered a match. pub const Criteria = struct { host: []const u8, @@ -81,14 +87,6 @@ pub const ConnectionPool = struct { const Queue = std.DoublyLinkedList(Connection); pub const Node = Queue.Node; - mutex: std.Thread.Mutex = .{}, - /// Open connections that are currently in use. - used: Queue = .{}, - /// Open connections that are not currently in use. - free: Queue = .{}, - free_len: usize = 0, - free_size: usize = 32, - /// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe. /// If no connection is found, null is returned. pub fn findConnection(pool: *ConnectionPool, criteria: Criteria) ?*Connection { @@ -217,11 +215,6 @@ pub const ConnectionPool = struct { /// An interface to either a plain or TLS connection. pub const Connection = struct { - pub const buffer_size = std.crypto.tls.max_ciphertext_record_len; - const BufferSize = std.math.IntFittingRange(0, buffer_size); - - pub const Protocol = enum { plain, tls }; - stream: Stream, /// undefined unless protocol is tls. tls_client: if (!disable_tls) *std.crypto.tls.Client else void, @@ -247,6 +240,11 @@ pub const Connection = struct { read_buf: [buffer_size]u8 = undefined, write_buf: [buffer_size]u8 = undefined, + pub const buffer_size = std.crypto.tls.max_ciphertext_record_len; + const BufferSize = std.math.IntFittingRange(0, buffer_size); + + pub const Protocol = enum { plain, tls }; + pub fn readvDirectTls(conn: *Connection, buffers: []std.posix.iovec) ReadError!usize { return conn.tls_client.readv(conn.stream, buffers) catch |err| { // https://github.com/ziglang/zig/issues/2473 @@ -367,7 +365,7 @@ pub const Connection = struct { /// Writes the given buffer to the connection. pub fn write(conn: *Connection, buffer: []const u8) WriteError!usize { - if (conn.write_end + buffer.len > conn.write_buf.len) { + if (conn.write_buf.len - conn.write_end < buffer.len) { try conn.flush(); if (buffer.len > conn.write_buf.len) { @@ -382,6 +380,13 @@ pub const Connection = struct { return buffer.len; } + /// Returns a buffer to be filled with exactly len bytes to write to the connection. + pub fn allocWriteBuffer(conn: *Connection, len: BufferSize) WriteError![]u8 { + if (conn.write_buf.len - conn.write_end < len) try conn.flush(); + defer conn.write_end += len; + return conn.write_buf[conn.write_end..][0..len]; + } + /// Flushes the write buffer to the connection. pub fn flush(conn: *Connection) WriteError!void { if (conn.write_end == 0) return; @@ -425,33 +430,65 @@ pub const RequestTransfer = union(enum) { /// The decompressor for response messages. pub const Compression = union(enum) { - pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader); - pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader); - pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{}); + pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader); + pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader); + // https://github.com/ziglang/zig/issues/18937 + //pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{}); deflate: DeflateDecompressor, gzip: GzipDecompressor, - zstd: ZstdDecompressor, + // https://github.com/ziglang/zig/issues/18937 + //zstd: ZstdDecompressor, none: void, }; /// A HTTP response originating from a server. pub const Response = struct { - pub const ParseError = Allocator.Error || error{ + version: http.Version, + status: http.Status, + reason: []const u8, + + /// Points into the user-provided `server_header_buffer`. + location: ?[]const u8 = null, + /// Points into the user-provided `server_header_buffer`. + content_type: ?[]const u8 = null, + /// Points into the user-provided `server_header_buffer`. + content_disposition: ?[]const u8 = null, + + keep_alive: bool, + + /// If present, the number of bytes in the response body. + content_length: ?u64 = null, + + /// If present, the transfer encoding of the response body, otherwise none. + transfer_encoding: http.TransferEncoding = .none, + + /// If present, the compression of the response body, otherwise identity (no compression). + transfer_compression: http.ContentEncoding = .identity, + + parser: proto.HeadersParser, + compression: Compression = .none, + + /// Whether the response body should be skipped. Any data read from the + /// response body will be discarded. + skip: bool = false, + + pub const ParseError = error{ HttpHeadersInvalid, HttpHeaderContinuationsUnsupported, HttpTransferEncodingUnsupported, HttpConnectionHeaderUnsupported, InvalidContentLength, - CompressionNotSupported, + CompressionUnsupported, }; - pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void { - var it = mem.tokenizeAny(u8, bytes, "\r\n"); + pub fn parse(res: *Response, bytes: []const u8) ParseError!void { + var it = mem.splitSequence(u8, bytes, "\r\n"); - const first_line = it.next() orelse return error.HttpHeadersInvalid; - if (first_line.len < 12) + const first_line = it.next().?; + if (first_line.len < 12) { return error.HttpHeadersInvalid; + } const version: http.Version = switch (int64(first_line[0..8])) { int64("HTTP/1.0") => .@"HTTP/1.0", @@ -465,25 +502,32 @@ pub const Response = struct { res.version = version; res.status = status; res.reason = reason; - - res.headers.clearRetainingCapacity(); + res.keep_alive = switch (version) { + .@"HTTP/1.0" => false, + .@"HTTP/1.1" => true, + }; while (it.next()) |line| { - if (line.len == 0) return error.HttpHeadersInvalid; + if (line.len == 0) return; switch (line[0]) { ' ', '\t' => return error.HttpHeaderContinuationsUnsupported, else => {}, } - var line_it = mem.tokenizeAny(u8, line, ": "); - const header_name = line_it.next() orelse return error.HttpHeadersInvalid; - const header_value = line_it.rest(); + var line_it = mem.splitScalar(u8, line, ':'); + const header_name = line_it.next().?; + const header_value = mem.trim(u8, line_it.rest(), " \t"); + if (header_name.len == 0) return error.HttpHeadersInvalid; - try res.headers.append(header_name, header_value); - - if (trailing) continue; - - if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) { + if (std.ascii.eqlIgnoreCase(header_name, "connection")) { + res.keep_alive = !std.ascii.eqlIgnoreCase(header_value, "close"); + } else if (std.ascii.eqlIgnoreCase(header_name, "content-type")) { + res.content_type = header_value; + } else if (std.ascii.eqlIgnoreCase(header_name, "location")) { + res.location = header_value; + } else if (std.ascii.eqlIgnoreCase(header_name, "content-disposition")) { + res.content_disposition = header_value; + } else if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) { // Transfer-Encoding: second, first // Transfer-Encoding: deflate, chunked var iter = mem.splitBackwardsScalar(u8, header_value, ','); @@ -529,6 +573,44 @@ pub const Response = struct { } } } + return error.HttpHeadersInvalid; // missing empty line + } + + test parse { + const response_bytes = "HTTP/1.1 200 OK\r\n" ++ + "LOcation:url\r\n" ++ + "content-tYpe: text/plain\r\n" ++ + "content-disposition:attachment; filename=example.txt \r\n" ++ + "content-Length:10\r\n" ++ + "TRansfer-encoding:\tdeflate, chunked \r\n" ++ + "connectioN:\t keep-alive \r\n\r\n"; + + var header_buffer: [1024]u8 = undefined; + var res = Response{ + .status = undefined, + .reason = undefined, + .version = undefined, + .keep_alive = false, + .parser = proto.HeadersParser.init(&header_buffer), + }; + + @memcpy(header_buffer[0..response_bytes.len], response_bytes); + res.parser.header_bytes_len = response_bytes.len; + + try res.parse(response_bytes); + + try testing.expectEqual(.@"HTTP/1.1", res.version); + try testing.expectEqualStrings("OK", res.reason); + try testing.expectEqual(.ok, res.status); + + try testing.expectEqualStrings("url", res.location.?); + try testing.expectEqualStrings("text/plain", res.content_type.?); + try testing.expectEqualStrings("attachment; filename=example.txt", res.content_disposition.?); + + try testing.expectEqual(true, res.keep_alive); + try testing.expectEqual(10, res.content_length.?); + try testing.expectEqual(.chunked, res.transfer_encoding); + try testing.expectEqual(.deflate, res.transfer_compression); } inline fn int64(array: *const [8]u8) u64 { @@ -552,60 +634,86 @@ pub const Response = struct { try expectEqual(@as(u10, 999), parseInt3("999")); } - /// The HTTP version this response is using. - version: http.Version, + pub fn iterateHeaders(r: Response) http.HeaderIterator { + return http.HeaderIterator.init(r.parser.get()); + } - /// The status code of the response. - status: http.Status, + test iterateHeaders { + const response_bytes = "HTTP/1.1 200 OK\r\n" ++ + "LOcation:url\r\n" ++ + "content-tYpe: text/plain\r\n" ++ + "content-disposition:attachment; filename=example.txt \r\n" ++ + "content-Length:10\r\n" ++ + "TRansfer-encoding:\tdeflate, chunked \r\n" ++ + "connectioN:\t keep-alive \r\n\r\n"; - /// The reason phrase of the response. - reason: []const u8, + var header_buffer: [1024]u8 = undefined; + var res = Response{ + .status = undefined, + .reason = undefined, + .version = undefined, + .keep_alive = false, + .parser = proto.HeadersParser.init(&header_buffer), + }; - /// If present, the number of bytes in the response body. - content_length: ?u64 = null, + @memcpy(header_buffer[0..response_bytes.len], response_bytes); + res.parser.header_bytes_len = response_bytes.len; - /// If present, the transfer encoding of the response body, otherwise none. - transfer_encoding: http.TransferEncoding = .none, - - /// If present, the compression of the response body, otherwise identity (no compression). - transfer_compression: http.ContentEncoding = .identity, - - /// The headers received from the server. - headers: http.Headers, - parser: proto.HeadersParser, - compression: Compression = .none, - - /// Whether the response body should be skipped. Any data read from the response body will be discarded. - skip: bool = false, + var it = res.iterateHeaders(); + { + const header = it.next().?; + try testing.expectEqualStrings("LOcation", header.name); + try testing.expectEqualStrings("url", header.value); + try testing.expect(!it.is_trailer); + } + { + const header = it.next().?; + try testing.expectEqualStrings("content-tYpe", header.name); + try testing.expectEqualStrings("text/plain", header.value); + try testing.expect(!it.is_trailer); + } + { + const header = it.next().?; + try testing.expectEqualStrings("content-disposition", header.name); + try testing.expectEqualStrings("attachment; filename=example.txt", header.value); + try testing.expect(!it.is_trailer); + } + { + const header = it.next().?; + try testing.expectEqualStrings("content-Length", header.name); + try testing.expectEqualStrings("10", header.value); + try testing.expect(!it.is_trailer); + } + { + const header = it.next().?; + try testing.expectEqualStrings("TRansfer-encoding", header.name); + try testing.expectEqualStrings("deflate, chunked", header.value); + try testing.expect(!it.is_trailer); + } + { + const header = it.next().?; + try testing.expectEqualStrings("connectioN", header.name); + try testing.expectEqualStrings("keep-alive", header.value); + try testing.expect(!it.is_trailer); + } + try testing.expectEqual(null, it.next()); + } }; /// A HTTP request that has been sent. /// /// Order of operations: open -> send[ -> write -> finish] -> wait -> read pub const Request = struct { - /// The uri that this request is being sent to. uri: Uri, - - /// The client that this request was created from. client: *Client, - - /// Underlying connection to the server. This is null when the connection is released. + /// This is null when the connection is released. connection: ?*Connection, + keep_alive: bool, method: http.Method, version: http.Version = .@"HTTP/1.1", - - /// The list of HTTP request headers. - headers: http.Headers, - - /// The transfer encoding of the request body. - transfer_encoding: RequestTransfer = .none, - - /// The redirect quota left for this request. - redirects_left: u32, - - /// Whether the request should follow redirects. - handle_redirects: bool, + transfer_encoding: RequestTransfer, + redirect_behavior: RedirectBehavior, /// Whether the request should handle a 100-continue response before sending the request body. handle_continue: bool, @@ -615,25 +723,60 @@ pub const Request = struct { /// This field is undefined until `wait` is called. response: Response, - /// Used as a allocator for resolving redirects locations. - arena: std.heap.ArenaAllocator, + /// Standard headers that have default, but overridable, behavior. + headers: Headers, + + /// These headers are kept including when following a redirect to a + /// different domain. + /// Externally-owned; must outlive the Request. + extra_headers: []const http.Header, + + /// These headers are stripped when following a redirect to a different + /// domain. + /// Externally-owned; must outlive the Request. + privileged_headers: []const http.Header, + + pub const Headers = struct { + host: Value = .default, + authorization: Value = .default, + user_agent: Value = .default, + connection: Value = .default, + accept_encoding: Value = .default, + content_type: Value = .default, + + pub const Value = union(enum) { + default, + omit, + override: []const u8, + }; + }; + + /// Any value other than `not_allowed` or `unhandled` means that integer represents + /// how many remaining redirects are allowed. + pub const RedirectBehavior = enum(u16) { + /// The next redirect will cause an error. + not_allowed = 0, + /// Redirects are passed to the client to analyze the redirect response + /// directly. + unhandled = std.math.maxInt(u16), + _, + + pub fn subtractOne(rb: *RedirectBehavior) void { + switch (rb.*) { + .not_allowed => unreachable, + .unhandled => unreachable, + _ => rb.* = @enumFromInt(@intFromEnum(rb.*) - 1), + } + } + + pub fn remaining(rb: RedirectBehavior) u16 { + assert(rb != .unhandled); + return @intFromEnum(rb); + } + }; /// Frees all resources associated with the request. pub fn deinit(req: *Request) void { - switch (req.response.compression) { - .none => {}, - .deflate => |*deflate| deflate.deinit(), - .gzip => |*gzip| gzip.deinit(), - .zstd => |*zstd| zstd.deinit(), - } - - req.headers.deinit(); - req.response.headers.deinit(); - - if (req.response.parser.header_bytes_owned) { - req.response.parser.header_bytes.deinit(req.client.allocator); - } - if (req.connection) |connection| { if (!req.response.parser.done) { // If the response wasn't fully read, then we need to close the connection. @@ -641,23 +784,15 @@ pub const Request = struct { } req.client.connection_pool.release(req.client.allocator, connection); } - - req.arena.deinit(); req.* = undefined; } - // This function must deallocate all resources associated with the request, or keep those which will be used - // This needs to be kept in sync with deinit and request + // This function must deallocate all resources associated with the request, + // or keep those which will be used. + // This needs to be kept in sync with deinit and request. fn redirect(req: *Request, uri: Uri) !void { assert(req.response.parser.done); - switch (req.response.compression) { - .none => {}, - .deflate => |*deflate| deflate.deinit(), - .gzip => |*gzip| gzip.deinit(), - .zstd => |*zstd| zstd.deinit(), - } - req.client.connection_pool.release(req.client.allocator, req.connection.?); req.connection = null; @@ -672,15 +807,14 @@ pub const Request = struct { req.uri = uri; req.connection = try req.client.connect(host, port, protocol); - req.redirects_left -= 1; - req.response.headers.clearRetainingCapacity(); + req.redirect_behavior.subtractOne(); req.response.parser.reset(); req.response = .{ + .version = undefined, .status = undefined, .reason = undefined, - .version = undefined, - .headers = req.response.headers, + .keep_alive = undefined, .parser = req.response.parser, }; } @@ -688,15 +822,17 @@ pub const Request = struct { pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding }; pub const SendOptions = struct { - /// Specifies that the uri should be used as is. You guarantee that the uri is already escaped. + /// Specifies that the uri is already escaped. raw_uri: bool = false, }; /// Send the HTTP request headers to the server. pub fn send(req: *Request, options: SendOptions) SendError!void { - if (!req.method.requestHasBody() and req.transfer_encoding != .none) return error.UnsupportedTransferEncoding; + if (!req.method.requestHasBody() and req.transfer_encoding != .none) + return error.UnsupportedTransferEncoding; - const w = req.connection.?.writer(); + const connection = req.connection.?; + const w = connection.writer(); try req.method.write(w); try w.writeByte(' '); @@ -705,9 +841,9 @@ pub const Request = struct { try req.uri.writeToStream(.{ .authority = true }, w); } else { try req.uri.writeToStream(.{ - .scheme = req.connection.?.proxied, - .authentication = req.connection.?.proxied, - .authority = req.connection.?.proxied, + .scheme = connection.proxied, + .authentication = connection.proxied, + .authority = connection.proxied, .path = true, .query = true, .raw = options.raw_uri, @@ -717,86 +853,93 @@ pub const Request = struct { try w.writeAll(@tagName(req.version)); try w.writeAll("\r\n"); - if (!req.headers.contains("host")) { - try w.writeAll("Host: "); + if (try emitOverridableHeader("host: ", req.headers.host, w)) { + try w.writeAll("host: "); try req.uri.writeToStream(.{ .authority = true }, w); try w.writeAll("\r\n"); } - if (!req.headers.contains("user-agent")) { - try w.writeAll("User-Agent: zig/"); + if (try emitOverridableHeader("authorization: ", req.headers.authorization, w)) { + if (req.uri.user != null or req.uri.password != null) { + try w.writeAll("authorization: "); + const authorization = try connection.allocWriteBuffer( + @intCast(basic_authorization.valueLengthFromUri(req.uri)), + ); + assert(basic_authorization.value(req.uri, authorization).len == authorization.len); + try w.writeAll("\r\n"); + } + } + + if (try emitOverridableHeader("user-agent: ", req.headers.user_agent, w)) { + try w.writeAll("user-agent: zig/"); try w.writeAll(builtin.zig_version_string); try w.writeAll(" (std.http)\r\n"); } - if (!req.headers.contains("connection")) { - try w.writeAll("Connection: keep-alive\r\n"); - } - - if (!req.headers.contains("accept-encoding")) { - try w.writeAll("Accept-Encoding: gzip, deflate, zstd\r\n"); - } - - if (!req.headers.contains("te")) { - try w.writeAll("TE: gzip, deflate, trailers\r\n"); - } - - const has_transfer_encoding = req.headers.contains("transfer-encoding"); - const has_content_length = req.headers.contains("content-length"); - - if (!has_transfer_encoding and !has_content_length) { - switch (req.transfer_encoding) { - .chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"), - .content_length => |content_length| try w.print("Content-Length: {d}\r\n", .{content_length}), - .none => {}, - } - } else { - if (has_transfer_encoding) { - const transfer_encoding = req.headers.getFirstValue("transfer-encoding").?; - if (std.mem.eql(u8, transfer_encoding, "chunked")) { - req.transfer_encoding = .chunked; - } else { - return error.UnsupportedTransferEncoding; - } - } else if (has_content_length) { - const content_length = std.fmt.parseInt(u64, req.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength; - - req.transfer_encoding = .{ .content_length = content_length }; + if (try emitOverridableHeader("connection: ", req.headers.connection, w)) { + if (req.keep_alive) { + try w.writeAll("connection: keep-alive\r\n"); } else { - req.transfer_encoding = .none; + try w.writeAll("connection: close\r\n"); } } - for (req.headers.list.items) |entry| { - if (entry.value.len == 0) continue; + if (try emitOverridableHeader("accept-encoding: ", req.headers.accept_encoding, w)) { + // https://github.com/ziglang/zig/issues/18937 + //try w.writeAll("accept-encoding: gzip, deflate, zstd\r\n"); + try w.writeAll("accept-encoding: gzip, deflate\r\n"); + } - try w.writeAll(entry.name); + switch (req.transfer_encoding) { + .chunked => try w.writeAll("transfer-encoding: chunked\r\n"), + .content_length => |len| try w.print("content-length: {d}\r\n", .{len}), + .none => {}, + } + + if (try emitOverridableHeader("content-type: ", req.headers.content_type, w)) { + // The default is to omit content-type if not provided because + // "application/octet-stream" is redundant. + } + + for (req.extra_headers) |header| { + assert(header.name.len != 0); + + try w.writeAll(header.name); try w.writeAll(": "); - try w.writeAll(entry.value); + try w.writeAll(header.value); try w.writeAll("\r\n"); } - if (req.connection.?.proxied) { - const proxy_headers: ?http.Headers = switch (req.connection.?.protocol) { - .plain => if (req.client.http_proxy) |proxy| proxy.headers else null, - .tls => if (req.client.https_proxy) |proxy| proxy.headers else null, - }; + if (connection.proxied) proxy: { + const proxy = switch (connection.protocol) { + .plain => req.client.http_proxy, + .tls => req.client.https_proxy, + } orelse break :proxy; - if (proxy_headers) |headers| { - for (headers.list.items) |entry| { - if (entry.value.len == 0) continue; - - try w.writeAll(entry.name); - try w.writeAll(": "); - try w.writeAll(entry.value); - try w.writeAll("\r\n"); - } - } + const authorization = proxy.authorization orelse break :proxy; + try w.writeAll("proxy-authorization: "); + try w.writeAll(authorization); + try w.writeAll("\r\n"); } try w.writeAll("\r\n"); - try req.connection.?.flush(); + try connection.flush(); + } + + /// Returns true if the default behavior is required, otherwise handles + /// writing (or not writing) the header. + fn emitOverridableHeader(prefix: []const u8, v: Headers.Value, w: anytype) !bool { + switch (v) { + .default => return true, + .omit => return false, + .override => |x| { + try w.writeAll(prefix); + try w.writeAll(x); + try w.writeAll("\r\n"); + return false; + }, + } } const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError; @@ -820,145 +963,172 @@ pub const Request = struct { return index; } - pub const WaitError = RequestError || SendError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, RedirectRequiresResend, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported }; + pub const WaitError = RequestError || SendError || TransferReadError || + proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || + error{ // TODO: file zig fmt issue for this bad indentation + TooManyHttpRedirects, + RedirectRequiresResend, + HttpRedirectLocationMissing, + HttpRedirectLocationInvalid, + CompressionInitializationFailed, + CompressionUnsupported, + }; /// Waits for a response from the server and parses any headers that are sent. /// This function will block until the final response is received. /// - /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow - /// redirects. If a request payload is present, then this function will error with error.RedirectRequiresResend. + /// If handling redirects and the request has no payload, then this + /// function will automatically follow redirects. If a request payload is + /// present, then this function will error with + /// error.RedirectRequiresResend. /// - /// Must be called after `send` and, if any data was written to the request body, then also after `finish`. + /// Must be called after `send` and, if any data was written to the request + /// body, then also after `finish`. pub fn wait(req: *Request) WaitError!void { - while (true) { // handle redirects - while (true) { // read headers - try req.connection.?.fill(); + while (true) { + // This while loop is for handling redirects, which means the request's + // connection may be different than the previous iteration. However, it + // is still guaranteed to be non-null with each iteration of this loop. + const connection = req.connection.?; - const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek()); - req.connection.?.drop(@intCast(nchecked)); + while (true) { // read headers + try connection.fill(); + + const nchecked = try req.response.parser.checkCompleteHead(connection.peek()); + connection.drop(@intCast(nchecked)); if (req.response.parser.state.isContent()) break; } - try req.response.parse(req.response.parser.header_bytes.items, false); + try req.response.parse(req.response.parser.get()); if (req.response.status == .@"continue") { - req.response.parser.done = true; // we're done parsing the continue response, reset to prepare for the real response + // We're done parsing the continue response; reset to prepare + // for the real response. + req.response.parser.done = true; req.response.parser.reset(); if (req.handle_continue) continue; - return; // we're not handling the 100-continue, return to the caller + return; // we're not handling the 100-continue } // we're switching protocols, so this connection is no longer doing http if (req.method == .CONNECT and req.response.status.class() == .success) { - req.connection.?.closing = false; + connection.closing = false; req.response.parser.done = true; - - return; // the connection is not HTTP past this point, return to the caller + return; // the connection is not HTTP past this point } - // we default to using keep-alive if not provided in the client if the server asks for it - const req_connection = req.headers.getFirstValue("connection"); - const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?); + connection.closing = !req.response.keep_alive or !req.keep_alive; - const res_connection = req.response.headers.getFirstValue("connection"); - const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?); - if (res_keepalive and (req_keepalive or req_connection == null)) { - req.connection.?.closing = false; - } else { - req.connection.?.closing = true; - } - - // Any response to a HEAD request and any response with a 1xx (Informational), 204 (No Content), or 304 (Not Modified) - // status code is always terminated by the first empty line after the header fields, regardless of the header fields - // present in the message - if (req.method == .HEAD or req.response.status.class() == .informational or req.response.status == .no_content or req.response.status == .not_modified) { + // Any response to a HEAD request and any response with a 1xx + // (Informational), 204 (No Content), or 304 (Not Modified) status + // code is always terminated by the first empty line after the + // header fields, regardless of the header fields present in the + // message. + if (req.method == .HEAD or req.response.status.class() == .informational or + req.response.status == .no_content or req.response.status == .not_modified) + { req.response.parser.done = true; - - return; // the response is empty, no further setup or redirection is necessary + return; // The response is empty; no further setup or redirection is necessary. } - if (req.response.transfer_encoding != .none) { - switch (req.response.transfer_encoding) { - .none => unreachable, - .chunked => { - req.response.parser.next_chunk_length = 0; - req.response.parser.state = .chunk_head_size; - }, - } - } else if (req.response.content_length) |cl| { - req.response.parser.next_chunk_length = cl; + switch (req.response.transfer_encoding) { + .none => { + if (req.response.content_length) |cl| { + req.response.parser.next_chunk_length = cl; - if (cl == 0) req.response.parser.done = true; - } else { - // read until the connection is closed - req.response.parser.next_chunk_length = std.math.maxInt(u64); + if (cl == 0) req.response.parser.done = true; + } else { + // read until the connection is closed + req.response.parser.next_chunk_length = std.math.maxInt(u64); + } + }, + .chunked => { + req.response.parser.next_chunk_length = 0; + req.response.parser.state = .chunk_head_size; + }, } - if (req.response.status.class() == .redirect and req.handle_redirects) { + if (req.response.status.class() == .redirect and req.redirect_behavior != .unhandled) { + // skip the body of the redirect response, this will at least + // leave the connection in a known good state. req.response.skip = true; + assert(try req.transferRead(&.{}) == 0); // we're skipping, no buffer is necessary - // skip the body of the redirect response, this will at least leave the connection in a known good state. - const empty = @as([*]u8, undefined)[0..0]; - assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary + if (req.redirect_behavior == .not_allowed) return error.TooManyHttpRedirects; - if (req.redirects_left == 0) return error.TooManyHttpRedirects; + const location = req.response.location orelse + return error.HttpRedirectLocationMissing; - const location = req.response.headers.getFirstValue("location") orelse - return error.HttpRedirectMissingLocation; + // This mutates the beginning of header_buffer and uses that + // for the backing memory of the returned new_uri. + const header_buffer = req.response.parser.header_bytes_buffer; + const new_uri = req.uri.resolve_inplace(location, header_buffer) catch + return error.HttpRedirectLocationInvalid; - const arena = req.arena.allocator(); + // The new URI references the beginning of header_bytes_buffer memory. + // That memory will be kept, but everything after it will be + // reused by the subsequent request. In other words, + // header_bytes_buffer must be large enough to store all + // redirect locations as well as the final request header. + const path_end = new_uri.path.ptr + new_uri.path.len; + // https://github.com/ziglang/zig/issues/1738 + const path_offset = @intFromPtr(path_end) - @intFromPtr(header_buffer.ptr); + const end_offset = @max(path_offset, location.len); + req.response.parser.header_bytes_buffer = header_buffer[end_offset..]; - const location_duped = try arena.dupe(u8, location); + const is_same_domain_or_subdomain = + std.ascii.endsWithIgnoreCase(new_uri.host.?, req.uri.host.?) and + (new_uri.host.?.len == req.uri.host.?.len or + new_uri.host.?[new_uri.host.?.len - req.uri.host.?.len - 1] == '.'); - const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped); - const resolved_url = try req.uri.resolve(new_url, false, arena); - - // is the redirect location on the same domain, or a subdomain of the original request? - const is_same_domain_or_subdomain = std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and (resolved_url.host.?.len == req.uri.host.?.len or resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.'); - - if (resolved_url.host == null or !is_same_domain_or_subdomain or !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme)) { - // we're redirecting to a different domain, strip privileged headers like cookies - _ = req.headers.delete("authorization"); - _ = req.headers.delete("www-authenticate"); - _ = req.headers.delete("cookie"); - _ = req.headers.delete("cookie2"); + if (new_uri.host == null or !is_same_domain_or_subdomain or + !std.ascii.eqlIgnoreCase(new_uri.scheme, req.uri.scheme)) + { + // When redirecting to a different domain, strip privileged headers. + req.privileged_headers = &.{}; } - if (req.response.status == .see_other or ((req.response.status == .moved_permanently or req.response.status == .found) and req.method == .POST)) { - // we're redirecting to a GET, so we need to change the method and remove the body + if (switch (req.response.status) { + .see_other => true, + .moved_permanently, .found => req.method == .POST, + else => false, + }) { + // A redirect to a GET must change the method and remove the body. req.method = .GET; req.transfer_encoding = .none; - _ = req.headers.delete("transfer-encoding"); - _ = req.headers.delete("content-length"); - _ = req.headers.delete("content-type"); + req.headers.content_type = .omit; } if (req.transfer_encoding != .none) { - return error.RedirectRequiresResend; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually. + // The request body has already been sent. The request is + // still in a valid state, but the redirect must be handled + // manually. + return error.RedirectRequiresResend; } - try req.redirect(resolved_url); - + try req.redirect(new_uri); try req.send(.{}); } else { req.response.skip = false; if (!req.response.parser.done) { switch (req.response.transfer_compression) { .identity => req.response.compression = .none, - .compress, .@"x-compress" => return error.CompressionNotSupported, + .compress, .@"x-compress" => return error.CompressionUnsupported, .deflate => req.response.compression = .{ - .deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed, + .deflate = std.compress.zlib.decompressor(req.transferReader()), }, .gzip, .@"x-gzip" => req.response.compression = .{ - .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed, - }, - .zstd => req.response.compression = .{ - .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()), + .gzip = std.compress.gzip.decompressor(req.transferReader()), }, + // https://github.com/ziglang/zig/issues/18937 + //.zstd => req.response.compression = .{ + // .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()), + //}, + .zstd => return error.CompressionUnsupported, } } @@ -967,7 +1137,8 @@ pub const Request = struct { } } - pub const ReadError = TransferReadError || proto.HeadersParser.CheckCompleteHeadError || error{ DecompressionFailure, InvalidTrailers }; + pub const ReadError = TransferReadError || proto.HeadersParser.CheckCompleteHeadError || + error{ DecompressionFailure, InvalidTrailers }; pub const Reader = std.io.Reader(*Request, ReadError, read); @@ -980,28 +1151,20 @@ pub const Request = struct { const out_index = switch (req.response.compression) { .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure, .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure, - .zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure, + // https://github.com/ziglang/zig/issues/18937 + //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure, else => try req.transferRead(buffer), }; + if (out_index > 0) return out_index; - if (out_index == 0) { - const has_trail = !req.response.parser.state.isContent(); + while (!req.response.parser.state.isContent()) { // read trailing headers + try req.connection.?.fill(); - while (!req.response.parser.state.isContent()) { // read trailing headers - try req.connection.?.fill(); - - const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek()); - req.connection.?.drop(@intCast(nchecked)); - } - - if (has_trail) { - // The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error. - // This will *only* fail for a malformed trailer. - req.response.parse(req.response.parser.header_bytes.items, true) catch return error.InvalidTrailers; - } + const nchecked = try req.response.parser.checkCompleteHead(req.connection.?.peek()); + req.connection.?.drop(@intCast(nchecked)); } - return out_index; + return 0; } /// Reads data from the response body. Must be called after `wait`. @@ -1028,9 +1191,11 @@ pub const Request = struct { pub fn write(req: *Request, bytes: []const u8) WriteError!usize { switch (req.transfer_encoding) { .chunked => { - try req.connection.?.writer().print("{x}\r\n", .{bytes.len}); - try req.connection.?.writer().writeAll(bytes); - try req.connection.?.writer().writeAll("\r\n"); + if (bytes.len > 0) { + try req.connection.?.writer().print("{x}\r\n", .{bytes.len}); + try req.connection.?.writer().writeAll(bytes); + try req.connection.?.writer().writeAll("\r\n"); + } return bytes.len; }, @@ -1069,16 +1234,12 @@ pub const Request = struct { } }; -/// A HTTP proxy server. pub const Proxy = struct { - allocator: Allocator, - headers: http.Headers, - protocol: Connection.Protocol, host: []const u8, + authorization: ?[]const u8, port: u16, - - supports_connect: bool = true, + supports_connect: bool, }; /// Release all associated resources with the client. @@ -1090,134 +1251,107 @@ pub fn deinit(client: *Client) void { client.connection_pool.deinit(client.allocator); - if (client.http_proxy) |*proxy| { - proxy.allocator.free(proxy.host); - proxy.headers.deinit(); - } - - if (client.https_proxy) |*proxy| { - proxy.allocator.free(proxy.host); - proxy.headers.deinit(); - } - if (!disable_tls) client.ca_bundle.deinit(client.allocator); client.* = undefined; } -/// Uses the *_proxy environment variable to set any unset proxies for the client. -/// This function *must not* be called when the client has any active connections. -pub fn loadDefaultProxies(client: *Client) !void { +/// Populates `http_proxy` and `https_proxy` via standard proxy environment variables. +/// Asserts the client has no active connections. +/// Uses `arena` for a few small allocations that must outlive the client, or +/// at least until those fields are set to different values. +pub fn initDefaultProxies(client: *Client, arena: Allocator) !void { // Prevent any new connections from being created. client.connection_pool.mutex.lock(); defer client.connection_pool.mutex.unlock(); - assert(client.connection_pool.used.first == null); // There are still active requests. + assert(client.connection_pool.used.first == null); // There are active requests. - if (client.http_proxy == null) http: { - const content: []const u8 = if (std.process.hasEnvVarConstant("http_proxy")) - try std.process.getEnvVarOwned(client.allocator, "http_proxy") - else if (std.process.hasEnvVarConstant("HTTP_PROXY")) - try std.process.getEnvVarOwned(client.allocator, "HTTP_PROXY") - else if (std.process.hasEnvVarConstant("all_proxy")) - try std.process.getEnvVarOwned(client.allocator, "all_proxy") - else if (std.process.hasEnvVarConstant("ALL_PROXY")) - try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY") - else - break :http; - defer client.allocator.free(content); - - const uri = Uri.parse(content) catch - Uri.parseWithoutScheme(content) catch - break :http; - - const protocol = if (uri.scheme.len == 0) - .plain // No scheme, assume http:// - else - protocol_map.get(uri.scheme) orelse break :http; // Unknown scheme, ignore - - const host = if (uri.host) |host| try client.allocator.dupe(u8, host) else break :http; // Missing host, ignore - client.http_proxy = .{ - .allocator = client.allocator, - .headers = .{ .allocator = client.allocator }, - - .protocol = protocol, - .host = host, - .port = uri.port orelse switch (protocol) { - .plain => 80, - .tls => 443, - }, - }; - - if (uri.user != null and uri.password != null) { - const prefix = "Basic "; - - const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? }); - defer client.allocator.free(unencoded); - - const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len) + prefix.len); - defer client.allocator.free(buffer); - - const result = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); - @memcpy(buffer[0..prefix.len], prefix); - - try client.http_proxy.?.headers.append("proxy-authorization", result); - } + if (client.http_proxy == null) { + client.http_proxy = try createProxyFromEnvVar(arena, &.{ + "http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY", + }); } - if (client.https_proxy == null) https: { - const content: []const u8 = if (std.process.hasEnvVarConstant("https_proxy")) - try std.process.getEnvVarOwned(client.allocator, "https_proxy") - else if (std.process.hasEnvVarConstant("HTTPS_PROXY")) - try std.process.getEnvVarOwned(client.allocator, "HTTPS_PROXY") - else if (std.process.hasEnvVarConstant("all_proxy")) - try std.process.getEnvVarOwned(client.allocator, "all_proxy") - else if (std.process.hasEnvVarConstant("ALL_PROXY")) - try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY") - else - break :https; - defer client.allocator.free(content); - - const uri = Uri.parse(content) catch - Uri.parseWithoutScheme(content) catch - break :https; - - const protocol = if (uri.scheme.len == 0) - .plain // No scheme, assume http:// - else - protocol_map.get(uri.scheme) orelse break :https; // Unknown scheme, ignore - - const host = if (uri.host) |host| try client.allocator.dupe(u8, host) else break :https; // Missing host, ignore - client.https_proxy = .{ - .allocator = client.allocator, - .headers = .{ .allocator = client.allocator }, - - .protocol = protocol, - .host = host, - .port = uri.port orelse switch (protocol) { - .plain => 80, - .tls => 443, - }, - }; - - if (uri.user != null and uri.password != null) { - const prefix = "Basic "; - - const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? }); - defer client.allocator.free(unencoded); - - const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len) + prefix.len); - defer client.allocator.free(buffer); - - const result = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); - @memcpy(buffer[0..prefix.len], prefix); - - try client.https_proxy.?.headers.append("proxy-authorization", result); - } + if (client.https_proxy == null) { + client.https_proxy = try createProxyFromEnvVar(arena, &.{ + "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY", + }); } } +fn createProxyFromEnvVar(arena: Allocator, env_var_names: []const []const u8) !?*Proxy { + const content = for (env_var_names) |name| { + break std.process.getEnvVarOwned(arena, name) catch |err| switch (err) { + error.EnvironmentVariableNotFound => continue, + else => |e| return e, + }; + } else return null; + + const uri = Uri.parse(content) catch try Uri.parseWithoutScheme(content); + + const protocol = if (uri.scheme.len == 0) + .plain // No scheme, assume http:// + else + protocol_map.get(uri.scheme) orelse return null; // Unknown scheme, ignore + + const host = uri.host orelse return error.HttpProxyMissingHost; + + const authorization: ?[]const u8 = if (uri.user != null or uri.password != null) a: { + const authorization = try arena.alloc(u8, basic_authorization.valueLengthFromUri(uri)); + assert(basic_authorization.value(uri, authorization).len == authorization.len); + break :a authorization; + } else null; + + const proxy = try arena.create(Proxy); + proxy.* = .{ + .protocol = protocol, + .host = host, + .authorization = authorization, + .port = uri.port orelse switch (protocol) { + .plain => 80, + .tls => 443, + }, + .supports_connect = true, + }; + return proxy; +} + +pub const basic_authorization = struct { + pub const max_user_len = 255; + pub const max_password_len = 255; + pub const max_value_len = valueLength(max_user_len, max_password_len); + + const prefix = "Basic "; + + pub fn valueLength(user_len: usize, password_len: usize) usize { + return prefix.len + std.base64.standard.Encoder.calcSize(user_len + 1 + password_len); + } + + pub fn valueLengthFromUri(uri: Uri) usize { + return valueLength( + if (uri.user) |user| user.len else 0, + if (uri.password) |password| password.len else 0, + ); + } + + pub fn value(uri: Uri, out: []u8) []u8 { + assert(uri.user == null or uri.user.?.len <= max_user_len); + assert(uri.password == null or uri.password.?.len <= max_password_len); + + @memcpy(out[0..prefix.len], prefix); + + var buf: [max_user_len + ":".len + max_password_len]u8 = undefined; + const unencoded = std.fmt.bufPrint(&buf, "{s}:{s}", .{ + uri.user orelse "", uri.password orelse "", + }) catch unreachable; + const base64 = std.base64.standard.Encoder.encode(out[prefix.len..], unencoded); + + return out[0 .. prefix.len + base64.len]; + } +}; + pub const ConnectTcpError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed }; /// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open. @@ -1278,7 +1412,8 @@ pub fn connectTcp(client: *Client, host: []const u8, port: u16, protocol: Connec return &conn.data; } -/// Connect to `tunnel_host:tunnel_port` using the specified proxy with HTTP CONNECT. This will reuse a connection if one is already open. +/// Connect to `tunnel_host:tunnel_port` using the specified proxy with HTTP +/// CONNECT. This will reuse a connection if one is already open. /// /// This function is threadsafe. pub fn connectTunnel( @@ -1304,7 +1439,7 @@ pub fn connectTunnel( client.connection_pool.release(client.allocator, conn); } - const uri = Uri{ + const uri: Uri = .{ .scheme = "http", .user = null, .password = null, @@ -1315,13 +1450,11 @@ pub fn connectTunnel( .fragment = null, }; - // we can use a small buffer here because a CONNECT response should be very small var buffer: [8096]u8 = undefined; - - var req = client.open(.CONNECT, uri, proxy.headers, .{ - .handle_redirects = false, + var req = client.open(.CONNECT, uri, .{ + .redirect_behavior = .unhandled, .connection = conn, - .header_strategy = .{ .static = &buffer }, + .server_header_buffer = &buffer, }) catch |err| { std.log.debug("err {}", .{err}); break :tunnel err; @@ -1360,45 +1493,51 @@ pub fn connectTunnel( const ConnectErrorPartial = ConnectTcpError || error{ UnsupportedUrlScheme, ConnectionRefused }; pub const ConnectError = ConnectErrorPartial || RequestError; -/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open. -/// If a proxy is configured for the client, then the proxy will be used to connect to the host. +/// Connect to `host:port` using the specified protocol. This will reuse a +/// connection if one is already open. +/// If a proxy is configured for the client, then the proxy will be used to +/// connect to the host. /// /// This function is threadsafe. -pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*Connection { - // pointer required so that `supports_connect` can be updated if a CONNECT fails - const potential_proxy: ?*Proxy = switch (protocol) { - .plain => if (client.http_proxy) |*proxy_info| proxy_info else null, - .tls => if (client.https_proxy) |*proxy_info| proxy_info else null, - }; +pub fn connect( + client: *Client, + host: []const u8, + port: u16, + protocol: Connection.Protocol, +) ConnectError!*Connection { + const proxy = switch (protocol) { + .plain => client.http_proxy, + .tls => client.https_proxy, + } orelse return client.connectTcp(host, port, protocol); - if (potential_proxy) |proxy| { - // don't attempt to proxy the proxy thru itself. - if (std.mem.eql(u8, proxy.host, host) and proxy.port == port and proxy.protocol == protocol) { - return client.connectTcp(host, port, protocol); - } - - if (proxy.supports_connect) tunnel: { - return connectTunnel(client, proxy, host, port) catch |err| switch (err) { - error.TunnelNotSupported => break :tunnel, - else => |e| return e, - }; - } - - // fall back to using the proxy as a normal http proxy - const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol); - errdefer { - conn.closing = true; - client.connection_pool.release(conn); - } - - conn.proxied = true; - return conn; + // Prevent proxying through itself. + if (std.ascii.eqlIgnoreCase(proxy.host, host) and + proxy.port == port and proxy.protocol == protocol) + { + return client.connectTcp(host, port, protocol); } - return client.connectTcp(host, port, protocol); + if (proxy.supports_connect) tunnel: { + return connectTunnel(client, proxy, host, port) catch |err| switch (err) { + error.TunnelNotSupported => break :tunnel, + else => |e| return e, + }; + } + + // fall back to using the proxy as a normal http proxy + const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol); + errdefer { + conn.closing = true; + client.connection_pool.release(conn); + } + + conn.proxied = true; + return conn; } -pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendError || std.fmt.ParseIntError || Connection.WriteError || error{ +pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendError || + std.fmt.ParseIntError || Connection.WriteError || + error{ // TODO: file a zig fmt issue for this bad indentation UnsupportedUrlScheme, UriMissingHost, @@ -1409,36 +1548,44 @@ pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendE pub const RequestOptions = struct { version: http.Version = .@"HTTP/1.1", - /// Automatically ignore 100 Continue responses. This assumes you don't care, and will have sent the body before you - /// wait for the response. + /// Automatically ignore 100 Continue responses. This assumes you don't + /// care, and will have sent the body before you wait for the response. /// - /// If this is not the case AND you know the server will send a 100 Continue, set this to false and wait for a - /// response before sending the body. If you wait AND the server does not send a 100 Continue before you finish the - /// request, then the request *will* deadlock. + /// If this is not the case AND you know the server will send a 100 + /// Continue, set this to false and wait for a response before sending the + /// body. If you wait AND the server does not send a 100 Continue before + /// you finish the request, then the request *will* deadlock. handle_continue: bool = true, - /// Automatically follow redirects. This will only follow redirects for repeatable requests (ie. with no payload or the server has acknowledged the payload) - handle_redirects: bool = true, + /// If false, close the connection after the one request. If true, + /// participate in the client connection pool. + keep_alive: bool = true, - /// How many redirects to follow before returning an error. - max_redirects: u32 = 3, - header_strategy: StorageStrategy = .{ .dynamic = 16 * 1024 }, + /// This field specifies whether to automatically follow redirects, and if + /// so, how many redirects to follow before returning an error. + /// + /// This will only follow redirects for repeatable requests (ie. with no + /// payload or the server has acknowledged the payload). + redirect_behavior: Request.RedirectBehavior = @enumFromInt(3), + + /// Externally-owned memory used to store the server's entire HTTP header. + /// `error.HttpHeadersOversize` is returned from read() when a + /// client sends too many bytes of HTTP headers. + server_header_buffer: []u8, /// Must be an already acquired connection. connection: ?*Connection = null, - pub const StorageStrategy = union(enum) { - /// In this case, the client's Allocator will be used to store the - /// entire HTTP header. This value is the maximum total size of - /// HTTP headers allowed, otherwise - /// error.HttpHeadersExceededSizeLimit is returned from read(). - dynamic: usize, - /// This is used to store the entire HTTP header. If the HTTP - /// header is too big to fit, `error.HttpHeadersExceededSizeLimit` - /// is returned from read(). When this is used, `error.OutOfMemory` - /// cannot be returned from `read()`. - static: []u8, - }; + /// Standard headers that have default, but overridable, behavior. + headers: Request.Headers = .{}, + /// These headers are kept including when following a redirect to a + /// different domain. + /// Externally-owned; must outlive the Request. + extra_headers: []const http.Header = &.{}, + /// These headers are stripped when following a redirect to a different + /// domain. + /// Externally-owned; must outlive the Request. + privileged_headers: []const http.Header = &.{}, }; pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{ @@ -1451,11 +1598,31 @@ pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{ /// Open a connection to the host specified by `uri` and prepare to send a HTTP request. /// /// `uri` must remain alive during the entire request. -/// `headers` is cloned and may be freed after this function returns. /// /// The caller is responsible for calling `deinit()` on the `Request`. /// This function is threadsafe. -pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: RequestOptions) RequestError!Request { +/// +/// Asserts that "\r\n" does not occur in any header name or value. +pub fn open( + client: *Client, + method: http.Method, + uri: Uri, + options: RequestOptions, +) RequestError!Request { + if (std.debug.runtime_safety) { + for (options.extra_headers) |header| { + assert(header.name.len != 0); + assert(std.mem.indexOfScalar(u8, header.name, ':') == null); + assert(std.mem.indexOfPosLinear(u8, header.name, 0, "\r\n") == null); + assert(std.mem.indexOfPosLinear(u8, header.value, 0, "\r\n") == null); + } + for (options.privileged_headers) |header| { + assert(header.name.len != 0); + assert(std.mem.indexOfPosLinear(u8, header.name, 0, "\r\n") == null); + assert(std.mem.indexOfPosLinear(u8, header.value, 0, "\r\n") == null); + } + } + const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme; const port: u16 = uri.port orelse switch (protocol) { @@ -1465,7 +1632,7 @@ pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Header const host = uri.host orelse return error.UriMissingHost; - if (protocol == .tls and @atomicLoad(bool, &client.next_https_rescan_certs, .Acquire)) { + if (protocol == .tls and @atomicLoad(bool, &client.next_https_rescan_certs, .acquire)) { if (disable_tls) unreachable; client.ca_bundle_mutex.lock(); @@ -1473,7 +1640,7 @@ pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Header if (client.next_https_rescan_certs) { client.ca_bundle.rescan(client.allocator) catch return error.CertificateBundleLoadFailure; - @atomicStore(bool, &client.next_https_rescan_certs, false, .Release); + @atomicStore(bool, &client.next_https_rescan_certs, false, .release); } } @@ -1483,163 +1650,132 @@ pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Header .uri = uri, .client = client, .connection = conn, - .headers = try headers.clone(client.allocator), // Headers must be cloned to properly handle header transformations in redirects. + .keep_alive = options.keep_alive, .method = method, .version = options.version, - .redirects_left = options.max_redirects, - .handle_redirects = options.handle_redirects, + .transfer_encoding = .none, + .redirect_behavior = options.redirect_behavior, .handle_continue = options.handle_continue, .response = .{ + .version = undefined, .status = undefined, .reason = undefined, - .version = undefined, - .headers = http.Headers{ .allocator = client.allocator, .owned = false }, - .parser = switch (options.header_strategy) { - .dynamic => |max| proto.HeadersParser.initDynamic(max), - .static => |buf| proto.HeadersParser.initStatic(buf), - }, + .keep_alive = undefined, + .parser = proto.HeadersParser.init(options.server_header_buffer), }, - .arena = undefined, + .headers = options.headers, + .extra_headers = options.extra_headers, + .privileged_headers = options.privileged_headers, }; errdefer req.deinit(); - req.arena = std.heap.ArenaAllocator.init(client.allocator); - return req; } pub const FetchOptions = struct { + server_header_buffer: ?[]u8 = null, + redirect_behavior: ?Request.RedirectBehavior = null, + + /// If the server sends a body, it will be appended to this ArrayList. + /// `max_append_size` provides an upper limit for how much they can grow. + response_storage: ResponseStorage = .ignore, + max_append_size: ?usize = null, + + location: Location, + method: ?http.Method = null, + payload: ?[]const u8 = null, + raw_uri: bool = false, + keep_alive: bool = true, + + /// Standard headers that have default, but overridable, behavior. + headers: Request.Headers = .{}, + /// These headers are kept including when following a redirect to a + /// different domain. + /// Externally-owned; must outlive the Request. + extra_headers: []const http.Header = &.{}, + /// These headers are stripped when following a redirect to a different + /// domain. + /// Externally-owned; must outlive the Request. + privileged_headers: []const http.Header = &.{}, + pub const Location = union(enum) { url: []const u8, uri: Uri, }; - pub const Payload = union(enum) { - string: []const u8, - file: std.fs.File, - none, + pub const ResponseStorage = union(enum) { + ignore, + /// Only the existing capacity will be used. + static: *std.ArrayListUnmanaged(u8), + dynamic: *std.ArrayList(u8), }; - - pub const ResponseStrategy = union(enum) { - storage: RequestOptions.StorageStrategy, - file: std.fs.File, - none, - }; - - header_strategy: RequestOptions.StorageStrategy = .{ .dynamic = 16 * 1024 }, - response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } }, - - location: Location, - method: http.Method = .GET, - headers: http.Headers = http.Headers{ .allocator = std.heap.page_allocator, .owned = false }, - payload: Payload = .none, - raw_uri: bool = false, }; pub const FetchResult = struct { status: http.Status, - body: ?[]const u8 = null, - headers: http.Headers, - - allocator: Allocator, - options: FetchOptions, - - pub fn deinit(res: *FetchResult) void { - if (res.options.response_strategy == .storage and res.options.response_strategy.storage == .dynamic) { - if (res.body) |body| res.allocator.free(body); - } - - res.headers.deinit(); - } }; /// Perform a one-shot HTTP request with the provided options. /// /// This function is threadsafe. -pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult { - const has_transfer_encoding = options.headers.contains("transfer-encoding"); - const has_content_length = options.headers.contains("content-length"); - - if (has_content_length or has_transfer_encoding) return error.UnsupportedHeader; - +pub fn fetch(client: *Client, options: FetchOptions) !FetchResult { const uri = switch (options.location) { .url => |u| try Uri.parse(u), .uri => |u| u, }; + var server_header_buffer: [16 * 1024]u8 = undefined; - var req = try open(client, options.method, uri, options.headers, .{ - .header_strategy = options.header_strategy, - .handle_redirects = options.payload == .none, + const method: http.Method = options.method orelse + if (options.payload != null) .POST else .GET; + + var req = try open(client, method, uri, .{ + .server_header_buffer = options.server_header_buffer orelse &server_header_buffer, + .redirect_behavior = options.redirect_behavior orelse + if (options.payload == null) @enumFromInt(3) else .unhandled, + .headers = options.headers, + .extra_headers = options.extra_headers, + .privileged_headers = options.privileged_headers, + .keep_alive = options.keep_alive, }); defer req.deinit(); - { // Block to maintain lock of file to attempt to prevent a race condition where another process modifies the file while we are reading it. - // This relies on other processes actually obeying the advisory lock, which is not guaranteed. - if (options.payload == .file) try options.payload.file.lock(.shared); - defer if (options.payload == .file) options.payload.file.unlock(); + if (options.payload) |payload| req.transfer_encoding = .{ .content_length = payload.len }; - switch (options.payload) { - .string => |str| req.transfer_encoding = .{ .content_length = str.len }, - .file => |file| req.transfer_encoding = .{ .content_length = (try file.stat()).size }, - .none => {}, - } + try req.send(.{ .raw_uri = options.raw_uri }); - try req.send(.{ .raw_uri = options.raw_uri }); - - switch (options.payload) { - .string => |str| try req.writeAll(str), - .file => |file| { - try file.seekTo(0); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init(); - try fifo.pump(file.reader(), req.writer()); - }, - .none => {}, - } - - try req.finish(); - } + if (options.payload) |payload| try req.writeAll(payload); + try req.finish(); try req.wait(); - var res = FetchResult{ - .status = req.response.status, - .headers = try req.response.headers.clone(allocator), - - .allocator = allocator, - .options = options, - }; - - switch (options.response_strategy) { - .storage => |storage| switch (storage) { - .dynamic => |max| res.body = try req.reader().readAllAlloc(allocator, max), - .static => |buf| res.body = buf[0..try req.reader().readAll(buf)], - }, - .file => |file| { - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init(); - try fifo.pump(req.reader(), file.writer()); - }, - .none => { // Take advantage of request internals to discard the response body and make the connection available for another request. + switch (options.response_storage) { + .ignore => { + // Take advantage of request internals to discard the response body + // and make the connection available for another request. req.response.skip = true; - - const empty = @as([*]u8, undefined)[0..0]; - assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary + assert(try req.transferRead(&.{}) == 0); // No buffer is necessary when skipping. + }, + .dynamic => |list| { + const max_append_size = options.max_append_size orelse 2 * 1024 * 1024; + try req.reader().readAllArrayList(list, max_append_size); + }, + .static => |list| { + const buf = b: { + const buf = list.unusedCapacitySlice(); + if (options.max_append_size) |len| { + if (len < buf.len) break :b buf[0..len]; + } + break :b buf; + }; + list.items.len += try req.reader().readAll(buf); }, } - return res; + return .{ + .status = req.response.status, + }; } test { - const native_endian = comptime builtin.cpu.arch.endian(); - if (builtin.zig_backend == .stage2_llvm and native_endian == .big) { - // https://github.com/ziglang/zig/issues/13782 - return error.SkipZigTest; - } - - if (builtin.os.tag == .wasi) return error.SkipZigTest; - - if (builtin.zig_backend == .stage2_x86_64 and - !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest; - - std.testing.refAllDecls(@This()); + _ = &initDefaultProxies; } diff --git a/src/async/stream.zig b/src/async/stream.zig index c6e4d35d..9ef73d01 100644 --- a/src/async/stream.zig +++ b/src/async/stream.zig @@ -31,7 +31,7 @@ pub const Stream = struct { handle: posix.socket_t, pub fn close(self: Stream) void { - posix.closeSocket(self.handle); + posix.close(self.handle); self.alloc.destroy(self.conn); } diff --git a/src/async/tcp.zig b/src/async/tcp.zig index a8ef8c34..61a49548 100644 --- a/src/async/tcp.zig +++ b/src/async/tcp.zig @@ -98,7 +98,7 @@ pub fn tcpConnectToHost(alloc: std.mem.Allocator, loop: *Loop, name: []const u8, pub fn tcpConnectToAddress(alloc: std.mem.Allocator, loop: *Loop, addr: net.Address) !Stream { const sockfd = try std.posix.socket(addr.any.family, std.posix.SOCK.STREAM, std.posix.IPPROTO.TCP); - errdefer std.posix.closeSocket(sockfd); + errdefer std.posix.close(sockfd); var conn = try alloc.create(Conn); conn.* = Conn{ .loop = loop }; diff --git a/src/async/test.zig b/src/async/test.zig index 6ec1cef4..0a103525 100644 --- a/src/async/test.zig +++ b/src/async/test.zig @@ -40,11 +40,9 @@ test "blocking mode fetch API" { // force client's CA cert scan from system. try client.ca_bundle.rescan(client.allocator); - var res = try client.fetch(alloc, .{ + const res = try client.fetch(.{ .location = .{ .uri = try std.Uri.parse(url) }, - .payload = .none, }); - defer res.deinit(); try std.testing.expect(res.status == .ok); } @@ -64,10 +62,10 @@ test "blocking mode open/send/wait API" { // force client's CA cert scan from system. try client.ca_bundle.rescan(client.allocator); - var headers = try std.http.Headers.initList(alloc, &[_]std.http.Field{}); - defer headers.deinit(); - - var req = try client.open(.GET, try std.Uri.parse(url), headers, .{}); + var buf: [2014]u8 = undefined; + var req = try client.open(.GET, try std.Uri.parse(url), .{ + .server_header_buffer = &buf, + }); defer req.deinit(); try req.send(.{}); @@ -87,7 +85,6 @@ const AsyncClient = struct { cli: *Client, uri: std.Uri, - headers: std.http.Headers, req: ?Request = undefined, state: State = .new, @@ -95,9 +92,10 @@ const AsyncClient = struct { impl: YieldImpl, err: ?anyerror = null, + buf: [2014]u8 = undefined, + pub fn deinit(self: *AsyncRequest) void { if (self.req) |*r| r.deinit(); - self.headers.deinit(); } pub fn fetch(self: *AsyncRequest) void { @@ -116,7 +114,9 @@ const AsyncClient = struct { switch (self.state) { .new => { self.state = .open; - self.req = self.cli.open(.GET, self.uri, self.headers, .{}) catch |e| return self.onerr(e); + self.req = self.cli.open(.GET, self.uri, .{ + .server_header_buffer = &self.buf, + }) catch |e| return self.onerr(e); }, .open => { self.state = .send; @@ -164,7 +164,6 @@ const AsyncClient = struct { .impl = YieldImpl.init(self.cli.loop), .cli = &self.cli, .uri = uri, - .headers = .{ .allocator = self.cli.allocator, .owned = false }, }; } }; diff --git a/src/browser/browser.zig b/src/browser/browser.zig index 0157e352..675680a9 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -247,29 +247,39 @@ pub const Page = struct { // TODO handle redirection if (req.response.status != .ok) { - log.debug("{?} {d} {s}\n{any}", .{ + log.debug("{?} {d} {s}", .{ req.response.version, req.response.status, req.response.reason, - req.response.headers, + // TODO log headers }); return error.BadStatusCode; } // TODO handle charset // https://html.spec.whatwg.org/#content-type - const ct = req.response.headers.getFirstValue("Content-Type") orelse { + var it = req.response.iterateHeaders(); + var ct: ?[]const u8 = null; + while (true) { + const h = it.next() orelse break; + if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) { + ct = try alloc.dupe(u8, h.value); + } + } + if (ct == null) { // no content type in HTTP headers. // TODO try to sniff mime type from the body. log.info("no content-type HTTP header", .{}); return; - }; - log.debug("header content-type: {s}", .{ct}); - const mime = try Mime.parse(ct); + } + defer alloc.free(ct.?); + + log.debug("header content-type: {s}", .{ct.?}); + const mime = try Mime.parse(ct.?); if (mime.eql(Mime.HTML)) { try self.loadHTMLDoc(req.reader(), mime.charset orelse "utf-8"); } else { - log.info("non-HTML document: {s}", .{ct}); + log.info("non-HTML document: {s}", .{ct.?}); // save the body into the page. self.raw_data = try req.reader().readAllAlloc(alloc, 16 * 1024 * 1024); @@ -500,20 +510,24 @@ pub const Page = struct { log.debug("starting fetch script {s}", .{src}); - const u = std.Uri.parse(src) catch try std.Uri.parseWithoutScheme(src); - const ru = try std.Uri.resolve(self.uri, u, false, alloc); + var buffer: [1024]u8 = undefined; + const u = try std.Uri.resolve_inplace(self.uri, src, &buffer); - var fetchres = try self.session.loader.fetch(alloc, ru); + var fetchres = try self.session.loader.get(alloc, u); defer fetchres.deinit(); - log.info("fech script {any}: {d}", .{ ru, fetchres.status }); + const resp = fetchres.req.response; - if (fetchres.status != .ok) return FetchError.BadStatusCode; + log.info("fech script {any}: {d}", .{ u, resp.status }); + + if (resp.status != .ok) return FetchError.BadStatusCode; // TODO check content-type + const body = try fetchres.req.reader().readAllAlloc(alloc, 16 * 1024 * 1024); + defer alloc.free(body); // check no body - if (fetchres.body == null) return FetchError.NoBody; + if (body.len == 0) return FetchError.NoBody; var res = try self.session.env.execTryCatch(alloc, fetchres.body.?, src); defer res.deinit(alloc); diff --git a/src/browser/loader.zig b/src/browser/loader.zig index 40007258..3fbebacf 100644 --- a/src/browser/loader.zig +++ b/src/browser/loader.zig @@ -22,6 +22,7 @@ const user_agent = "Lightpanda.io/1.0"; pub const Loader = struct { client: std.http.Client, + server_header_buffer: [1024]u8 = undefined, pub const Response = struct { alloc: std.mem.Allocator, @@ -45,42 +46,26 @@ pub const Loader = struct { self.client.deinit(); } - // the caller must deinit the FetchResult. - pub fn fetch(self: *Loader, alloc: std.mem.Allocator, uri: std.Uri) !std.http.Client.FetchResult { - var headers = try std.http.Headers.initList(alloc, &[_]std.http.Field{ - .{ .name = "User-Agent", .value = user_agent }, - .{ .name = "Accept", .value = "*/*" }, - .{ .name = "Accept-Language", .value = "en-US,en;q=0.5" }, - }); - defer headers.deinit(); - - return try self.client.fetch(alloc, .{ - .location = .{ .uri = uri }, - .headers = headers, - .payload = .none, - }); - } - // see // https://ziglang.org/documentation/master/std/#A;std:http.Client.fetch // for reference. // The caller is responsible for calling `deinit()` on the `Response`. pub fn get(self: *Loader, alloc: std.mem.Allocator, uri: std.Uri) !Response { - var headers = try std.http.Headers.initList(alloc, &[_]std.http.Field{ - .{ .name = "User-Agent", .value = user_agent }, - .{ .name = "Accept", .value = "*/*" }, - .{ .name = "Accept-Language", .value = "en-US,en;q=0.5" }, - }); - defer headers.deinit(); - var resp = Response{ .alloc = alloc, .req = try alloc.create(std.http.Client.Request), }; errdefer alloc.destroy(resp.req); - resp.req.* = try self.client.open(.GET, uri, headers, .{ - .handle_redirects = true, // TODO handle redirects manually + resp.req.* = try self.client.open(.GET, uri, .{ + .headers = .{ + .user_agent = .{ .override = user_agent }, + }, + .extra_headers = &.{ + .{ .name = "Accept", .value = "*/*" }, + .{ .name = "Accept-Language", .value = "en-US,en;q=0.5" }, + }, + .server_header_buffer = &self.server_header_buffer, }); errdefer resp.req.deinit(); @@ -92,13 +77,13 @@ pub const Loader = struct { } }; -test "basic url fetch" { +test "basic url get" { const alloc = std.testing.allocator; var loader = Loader.init(alloc); defer loader.deinit(); - var result = try loader.fetch(alloc, "https://en.wikipedia.org/wiki/Main_Page"); + var result = try loader.get(alloc, "https://en.wikipedia.org/wiki/Main_Page"); defer result.deinit(); - try std.testing.expect(result.status == std.http.Status.ok); + try std.testing.expect(result.req.response.status == std.http.Status.ok); } diff --git a/src/main_get.zig b/src/main_get.zig index cc14aeac..6f455553 100644 --- a/src/main_get.zig +++ b/src/main_get.zig @@ -25,8 +25,8 @@ const apiweb = @import("apiweb.zig"); pub const Types = jsruntime.reflect(apiweb.Interfaces); pub const UserContext = apiweb.UserContext; -pub const std_options = struct { - pub const log_level = .debug; +pub const std_options = std.Options{ + .log_level = .debug, }; const usage = diff --git a/src/xhr/xhr.zig b/src/xhr/xhr.zig index e395144a..5513ade8 100644 --- a/src/xhr/xhr.zig +++ b/src/xhr/xhr.zig @@ -95,6 +95,45 @@ pub const XMLHttpRequestBodyInit = union(XMLHttpRequestBodyInitTag) { }; pub const XMLHttpRequest = struct { + proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{}, + alloc: std.mem.Allocator, + cli: Client, + impl: YieldImpl, + + priv_state: PrivState = .new, + req: ?Client.Request = null, + + method: std.http.Method, + state: u16, + url: ?[]const u8, + uri: std.Uri, + // request headers + headers: Headers, + sync: bool = true, + err: ?anyerror = null, + + // TODO uncomment this field causes casting issue with + // XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but + // not sure. see + // https://lightpanda.slack.com/archives/C05TRU6RBM1/p1707819010681019 + // upload: ?XMLHttpRequestUpload = null, + + timeout: u32 = 0, + withCredentials: bool = false, + // TODO: response readonly attribute any response; + response_bytes: ?[]const u8 = null, + response_type: ResponseType = .Empty, + response_headers: Headers, + // used by zig client to parse reponse headers. + response_header_buffer: [1024]u8 = undefined, + response_status: u10 = 0, + response_override_mime_type: ?[]const u8 = null, + response_mime: Mime = undefined, + response_obj: ?ResponseObj = null, + send_flag: bool = false, + + payload: ?[]const u8 = null, + pub const prototype = *XMLHttpRequestEventTarget; pub const mem_guarantied = true; @@ -116,6 +155,91 @@ pub const XMLHttpRequest = struct { const JSONValue = std.json.Value; + const Headers = struct { + alloc: std.mem.Allocator, + list: List, + + const List = std.ArrayListUnmanaged(std.http.Header); + + fn init(alloc: std.mem.Allocator) Headers { + return .{ + .alloc = alloc, + .list = List{}, + }; + } + + fn deinit(self: *Headers) void { + self.free(); + self.list.deinit(self.alloc); + } + + fn append(self: *Headers, k: []const u8, v: []const u8) !void { + // duplicate strings + const kk = try self.alloc.dupe(u8, k); + const vv = try self.alloc.dupe(u8, v); + try self.list.append(self.alloc, .{ .name = kk, .value = vv }); + } + + // free all strings allocated. + fn free(self: *Headers) void { + for (self.list.items) |h| { + self.alloc.free(h.name); + self.alloc.free(h.value); + } + } + + fn clearAndFree(self: *Headers) void { + self.free(); + self.list.clearAndFree(self.alloc); + } + + fn has(self: Headers, k: []const u8) bool { + for (self.list.items) |h| { + if (std.ascii.eqlIgnoreCase(k, h.value)) { + return true; + } + } + + return false; + } + + fn getFirstValue(self: Headers, k: []const u8) ?[]const u8 { + for (self.list.items) |h| { + if (std.ascii.eqlIgnoreCase(k, h.value)) { + return h.value; + } + } + + return null; + } + + // replace any existing header with the same key + fn set(self: *Headers, k: []const u8, v: []const u8) !void { + for (self.list.items, 0..) |h, i| { + if (std.ascii.eqlIgnoreCase(k, h.value)) { + const hh = self.list.swapRemove(i); + self.alloc.free(hh.name); + self.alloc.free(hh.value); + } + } + self.append(k, v); + } + + // TODO + fn sort(_: *Headers) void {} + + fn all(self: Headers) []std.http.Header { + return self.list.items; + } + + fn load(self: *Headers, it: *std.http.HeaderIterator) !void { + while (true) { + const h = it.next() orelse break; + _ = try self.append(h.name, h.value); + } + } + }; + const Response = union(ResponseType) { Empty: void, Text: []const u8, @@ -149,49 +273,13 @@ pub const XMLHttpRequest = struct { const PrivState = enum { new, open, send, write, finish, wait, done }; - proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{}, - alloc: std.mem.Allocator, - cli: *Client, - impl: YieldImpl, - - priv_state: PrivState = .new, - req: ?Client.Request = null, - - method: std.http.Method, - state: u16, - url: ?[]const u8, - uri: std.Uri, - headers: std.http.Headers, - sync: bool = true, - err: ?anyerror = null, - - // TODO uncomment this field causes casting issue with - // XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but - // not sure. see - // https://lightpanda.slack.com/archives/C05TRU6RBM1/p1707819010681019 - // upload: ?XMLHttpRequestUpload = null, - - timeout: u32 = 0, - withCredentials: bool = false, - // TODO: response readonly attribute any response; - response_bytes: ?[]const u8 = null, - response_type: ResponseType = .Empty, - response_headers: std.http.Headers, - response_status: u10 = 0, - response_override_mime_type: ?[]const u8 = null, - response_mime: Mime = undefined, - response_obj: ?ResponseObj = null, - send_flag: bool = false, - - payload: ?[]const u8 = null, - const min_delay: u64 = 50000000; // 50ms pub fn constructor(alloc: std.mem.Allocator, loop: *Loop, userctx: UserContext) !XMLHttpRequest { return .{ .alloc = alloc, - .headers = .{ .allocator = alloc, .owned = true }, - .response_headers = .{ .allocator = alloc, .owned = true }, + .headers = Headers.init(alloc), + .response_headers = Headers.init(alloc), .impl = YieldImpl.init(loop), .method = undefined, .url = null, @@ -385,7 +473,7 @@ pub const XMLHttpRequest = struct { const body_init = XMLHttpRequestBodyInit{ .String = body.? }; // keep the user content type from request headers. - if (self.headers.getFirstEntry("Content-Type") == null) { + if (self.headers.has("Content-Type")) { // https://fetch.spec.whatwg.org/#bodyinit-safely-extract try self.headers.append("Content-Type", try body_init.contentType()); } @@ -411,7 +499,10 @@ pub const XMLHttpRequest = struct { switch (self.priv_state) { .new => { self.priv_state = .open; - self.req = self.cli.open(self.method, self.uri, self.headers, .{}) catch |e| return self.onErr(e); + self.req = self.cli.open(self.method, self.uri, .{ + .server_header_buffer = &self.response_header_buffer, + .extra_headers = self.headers.all(), + }) catch |e| return self.onErr(e); }, .open => { // prepare payload transfert. @@ -441,7 +532,8 @@ pub const XMLHttpRequest = struct { log.info("{any} {any} {d}", .{ self.method, self.uri, self.req.?.response.status }); self.priv_state = .done; - self.response_headers = self.req.?.response.headers.clone(self.response_headers.allocator) catch |e| return self.onErr(e); + var it = self.req.?.response.iterateHeaders(); + self.response_headers.load(&it) catch |e| return self.onErr(e); // extract a mime type from headers. const ct = self.response_headers.getFirstValue("Content-Type") orelse "text/xml";