diff --git a/src/apiweb.zig b/src/apiweb.zig index 40cc8779..a08f9f41 100644 --- a/src/apiweb.zig +++ b/src/apiweb.zig @@ -5,6 +5,7 @@ const Console = @import("jsruntime").Console; const DOM = @import("dom/dom.zig"); const HTML = @import("html/html.zig"); const Events = @import("events/event.zig"); +const XHR = @import("xhr/xhr.zig"); pub const HTMLDocument = @import("html/document.zig").HTMLDocument; @@ -14,4 +15,5 @@ pub const Interfaces = generate.Tuple(.{ DOM.Interfaces, Events.Interfaces, HTML.Interfaces, + XHR.Interfaces, }); diff --git a/src/async/Client.zig b/src/async/Client.zig new file mode 100644 index 00000000..3af59134 --- /dev/null +++ b/src/async/Client.zig @@ -0,0 +1,1627 @@ +//! HTTP(S) Client implementation. +//! +//! Connections are opened in a thread-safe manner, but individual Requests are not. +//! +//! TLS support may be disabled via `std.options.http_disable_tls`. +//! +//! This file is a copy of the original std.http.Client with little changes to +//! handle non-blocking I/O with the jsruntime.Loop. + +const std = @import("std"); +const builtin = @import("builtin"); +const Stream = @import("stream.zig").Stream; +const testing = std.testing; +const http = std.http; +const mem = std.mem; +const net = std.net; +const Uri = std.Uri; +const Allocator = mem.Allocator; +const assert = std.debug.assert; +const use_vectors = builtin.zig_backend != .stage2_x86_64; + +const Client = @This(); +const proto = http.protocol; + +const Loop = @import("jsruntime").Loop; +const tcp = @import("tcp.zig"); + +pub const disable_tls = std.options.http_disable_tls; + +/// Allocator used for all allocations made by the client. +/// +/// This allocator must be thread-safe. +allocator: Allocator, + +// std.net.Stream implementation using jsruntime Loop +loop: *Loop, + +ca_bundle: if (disable_tls) void else std.crypto.Certificate.Bundle = if (disable_tls) {} else .{}, +ca_bundle_mutex: std.Thread.Mutex = .{}, + +/// When this is `true`, the next time this client performs an HTTPS request, +/// it will first rescan the system for root certificates. +next_https_rescan_certs: bool = true, + +/// The pool of connections that can be reused (and currently in use). +connection_pool: ConnectionPool = .{}, + +/// This is the proxy that will handle http:// connections. It *must not* be modified when the client has any active connections. +http_proxy: ?Proxy = null, + +/// This is the proxy that will handle https:// connections. It *must not* be modified when the client has any active connections. +https_proxy: ?Proxy = null, + +/// A set of linked lists of connections that can be reused. +pub const ConnectionPool = struct { + /// The criteria for a connection to be considered a match. + pub const Criteria = struct { + host: []const u8, + port: u16, + protocol: Connection.Protocol, + }; + + const Queue = std.DoublyLinkedList(Connection); + pub const Node = Queue.Node; + + mutex: std.Thread.Mutex = .{}, + /// Open connections that are currently in use. + used: Queue = .{}, + /// Open connections that are not currently in use. + free: Queue = .{}, + free_len: usize = 0, + free_size: usize = 32, + + /// Finds and acquires a connection from the connection pool matching the criteria. This function is threadsafe. + /// If no connection is found, null is returned. + pub fn findConnection(pool: *ConnectionPool, criteria: Criteria) ?*Connection { + pool.mutex.lock(); + defer pool.mutex.unlock(); + + var next = pool.free.last; + while (next) |node| : (next = node.prev) { + if (node.data.protocol != criteria.protocol) continue; + if (node.data.port != criteria.port) continue; + + // Domain names are case-insensitive (RFC 5890, Section 2.3.2.4) + if (!std.ascii.eqlIgnoreCase(node.data.host, criteria.host)) continue; + + pool.acquireUnsafe(node); + return &node.data; + } + + return null; + } + + /// Acquires an existing connection from the connection pool. This function is not threadsafe. + pub fn acquireUnsafe(pool: *ConnectionPool, node: *Node) void { + pool.free.remove(node); + pool.free_len -= 1; + + pool.used.append(node); + } + + /// Acquires an existing connection from the connection pool. This function is threadsafe. + pub fn acquire(pool: *ConnectionPool, node: *Node) void { + pool.mutex.lock(); + defer pool.mutex.unlock(); + + return pool.acquireUnsafe(node); + } + + /// Tries to release a connection back to the connection pool. This function is threadsafe. + /// If the connection is marked as closing, it will be closed instead. + /// + /// The allocator must be the owner of all nodes in this pool. + /// The allocator must be the owner of all resources associated with the connection. + pub fn release(pool: *ConnectionPool, allocator: Allocator, connection: *Connection) void { + pool.mutex.lock(); + defer pool.mutex.unlock(); + + const node = @fieldParentPtr(Node, "data", connection); + + pool.used.remove(node); + + if (node.data.closing or pool.free_size == 0) { + node.data.close(allocator); + return allocator.destroy(node); + } + + if (pool.free_len >= pool.free_size) { + const popped = pool.free.popFirst() orelse unreachable; + pool.free_len -= 1; + + popped.data.close(allocator); + allocator.destroy(popped); + } + + if (node.data.proxied) { + pool.free.prepend(node); // proxied connections go to the end of the queue, always try direct connections first + } else { + pool.free.append(node); + } + + pool.free_len += 1; + } + + /// Adds a newly created node to the pool of used connections. This function is threadsafe. + pub fn addUsed(pool: *ConnectionPool, node: *Node) void { + pool.mutex.lock(); + defer pool.mutex.unlock(); + + pool.used.append(node); + } + + /// Resizes the connection pool. This function is threadsafe. + /// + /// If the new size is smaller than the current size, then idle connections will be closed until the pool is the new size. + pub fn resize(pool: *ConnectionPool, allocator: Allocator, new_size: usize) void { + pool.mutex.lock(); + defer pool.mutex.unlock(); + + const next = pool.free.first; + _ = next; + while (pool.free_len > new_size) { + const popped = pool.free.popFirst() orelse unreachable; + pool.free_len -= 1; + + popped.data.close(allocator); + allocator.destroy(popped); + } + + pool.free_size = new_size; + } + + /// Frees the connection pool and closes all connections within. This function is threadsafe. + /// + /// All future operations on the connection pool will deadlock. + pub fn deinit(pool: *ConnectionPool, allocator: Allocator) void { + pool.mutex.lock(); + + var next = pool.free.first; + while (next) |node| { + defer allocator.destroy(node); + next = node.next; + + node.data.close(allocator); + } + + next = pool.used.first; + while (next) |node| { + defer allocator.destroy(node); + next = node.next; + + node.data.close(allocator); + } + + pool.* = undefined; + } +}; + +/// An interface to either a plain or TLS connection. +pub const Connection = struct { + pub const buffer_size = std.crypto.tls.max_ciphertext_record_len; + const BufferSize = std.math.IntFittingRange(0, buffer_size); + + pub const Protocol = enum { plain, tls }; + + stream: Stream, + /// undefined unless protocol is tls. + tls_client: if (!disable_tls) *std.crypto.tls.Client else void, + + /// The protocol that this connection is using. + protocol: Protocol, + + /// The host that this connection is connected to. + host: []u8, + + /// The port that this connection is connected to. + port: u16, + + /// Whether this connection is proxied and is not directly connected. + proxied: bool = false, + + /// Whether this connection is closing when we're done with it. + closing: bool = false, + + read_start: BufferSize = 0, + read_end: BufferSize = 0, + write_end: BufferSize = 0, + read_buf: [buffer_size]u8 = undefined, + write_buf: [buffer_size]u8 = undefined, + + pub fn readvDirectTls(conn: *Connection, buffers: []std.os.iovec) ReadError!usize { + return conn.tls_client.readv(conn.stream, buffers) catch |err| { + // https://github.com/ziglang/zig/issues/2473 + if (mem.startsWith(u8, @errorName(err), "TlsAlert")) return error.TlsAlert; + + switch (err) { + error.TlsConnectionTruncated, error.TlsRecordOverflow, error.TlsDecodeError, error.TlsBadRecordMac, error.TlsBadLength, error.TlsIllegalParameter, error.TlsUnexpectedMessage => return error.TlsFailure, + error.ConnectionTimedOut => return error.ConnectionTimedOut, + error.ConnectionResetByPeer, error.BrokenPipe => return error.ConnectionResetByPeer, + else => return error.UnexpectedReadFailure, + } + }; + } + + pub fn readvDirect(conn: *Connection, buffers: []std.os.iovec) ReadError!usize { + if (conn.protocol == .tls) { + if (disable_tls) unreachable; + + return conn.readvDirectTls(buffers); + } + + return conn.stream.readv(buffers) catch |err| switch (err) { + error.ConnectionTimedOut => return error.ConnectionTimedOut, + error.ConnectionResetByPeer, error.BrokenPipe => return error.ConnectionResetByPeer, + else => return error.UnexpectedReadFailure, + }; + } + + /// Refills the read buffer with data from the connection. + pub fn fill(conn: *Connection) ReadError!void { + if (conn.read_end != conn.read_start) return; + + var iovecs = [1]std.os.iovec{ + .{ .iov_base = &conn.read_buf, .iov_len = conn.read_buf.len }, + }; + const nread = try conn.readvDirect(&iovecs); + if (nread == 0) return error.EndOfStream; + conn.read_start = 0; + conn.read_end = @intCast(nread); + } + + /// Returns the current slice of buffered data. + pub fn peek(conn: *Connection) []const u8 { + return conn.read_buf[conn.read_start..conn.read_end]; + } + + /// Discards the given number of bytes from the read buffer. + pub fn drop(conn: *Connection, num: BufferSize) void { + conn.read_start += num; + } + + /// Reads data from the connection into the given buffer. + pub fn read(conn: *Connection, buffer: []u8) ReadError!usize { + const available_read = conn.read_end - conn.read_start; + const available_buffer = buffer.len; + + if (available_read > available_buffer) { // partially read buffered data + @memcpy(buffer[0..available_buffer], conn.read_buf[conn.read_start..conn.read_end][0..available_buffer]); + conn.read_start += @intCast(available_buffer); + + return available_buffer; + } else if (available_read > 0) { // fully read buffered data + @memcpy(buffer[0..available_read], conn.read_buf[conn.read_start..conn.read_end]); + conn.read_start += available_read; + + return available_read; + } + + var iovecs = [2]std.os.iovec{ + .{ .iov_base = buffer.ptr, .iov_len = buffer.len }, + .{ .iov_base = &conn.read_buf, .iov_len = conn.read_buf.len }, + }; + const nread = try conn.readvDirect(&iovecs); + + if (nread > buffer.len) { + conn.read_start = 0; + conn.read_end = @intCast(nread - buffer.len); + return buffer.len; + } + + return nread; + } + + pub const ReadError = error{ + TlsFailure, + TlsAlert, + ConnectionTimedOut, + ConnectionResetByPeer, + UnexpectedReadFailure, + EndOfStream, + }; + + pub const Reader = std.io.Reader(*Connection, ReadError, read); + + pub fn reader(conn: *Connection) Reader { + return Reader{ .context = conn }; + } + + pub fn writeAllDirectTls(conn: *Connection, buffer: []const u8) WriteError!void { + return conn.tls_client.writeAll(conn.stream, buffer) catch |err| switch (err) { + error.BrokenPipe, error.ConnectionResetByPeer => return error.ConnectionResetByPeer, + else => return error.UnexpectedWriteFailure, + }; + } + + pub fn writeAllDirect(conn: *Connection, buffer: []const u8) WriteError!void { + if (conn.protocol == .tls) { + if (disable_tls) unreachable; + + return conn.writeAllDirectTls(buffer); + } + + return conn.stream.writeAll(buffer) catch |err| switch (err) { + error.BrokenPipe, error.ConnectionResetByPeer => return error.ConnectionResetByPeer, + else => return error.UnexpectedWriteFailure, + }; + } + + /// Writes the given buffer to the connection. + pub fn write(conn: *Connection, buffer: []const u8) WriteError!usize { + if (conn.write_end + buffer.len > conn.write_buf.len) { + try conn.flush(); + + if (buffer.len > conn.write_buf.len) { + try conn.writeAllDirect(buffer); + return buffer.len; + } + } + + @memcpy(conn.write_buf[conn.write_end..][0..buffer.len], buffer); + conn.write_end += @intCast(buffer.len); + + return buffer.len; + } + + /// Flushes the write buffer to the connection. + pub fn flush(conn: *Connection) WriteError!void { + if (conn.write_end == 0) return; + + try conn.writeAllDirect(conn.write_buf[0..conn.write_end]); + conn.write_end = 0; + } + + pub const WriteError = error{ + ConnectionResetByPeer, + UnexpectedWriteFailure, + }; + + pub const Writer = std.io.Writer(*Connection, WriteError, write); + + pub fn writer(conn: *Connection) Writer { + return Writer{ .context = conn }; + } + + /// Closes the connection. + pub fn close(conn: *Connection, allocator: Allocator) void { + if (conn.protocol == .tls) { + if (disable_tls) unreachable; + + // try to cleanly close the TLS connection, for any server that cares. + _ = conn.tls_client.writeEnd(conn.stream, "", true) catch {}; + allocator.destroy(conn.tls_client); + } + + conn.stream.close(); + allocator.free(conn.host); + } +}; + +/// The mode of transport for requests. +pub const RequestTransfer = union(enum) { + content_length: u64, + chunked: void, + none: void, +}; + +/// The decompressor for response messages. +pub const Compression = union(enum) { + pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader); + pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader); + pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{}); + + deflate: DeflateDecompressor, + gzip: GzipDecompressor, + zstd: ZstdDecompressor, + none: void, +}; + +/// A HTTP response originating from a server. +pub const Response = struct { + pub const ParseError = Allocator.Error || error{ + HttpHeadersInvalid, + HttpHeaderContinuationsUnsupported, + HttpTransferEncodingUnsupported, + HttpConnectionHeaderUnsupported, + InvalidContentLength, + CompressionNotSupported, + }; + + pub fn parse(res: *Response, bytes: []const u8, trailing: bool) ParseError!void { + var it = mem.tokenizeAny(u8, bytes, "\r\n"); + + const first_line = it.next() orelse return error.HttpHeadersInvalid; + if (first_line.len < 12) + return error.HttpHeadersInvalid; + + const version: http.Version = switch (int64(first_line[0..8])) { + int64("HTTP/1.0") => .@"HTTP/1.0", + int64("HTTP/1.1") => .@"HTTP/1.1", + else => return error.HttpHeadersInvalid, + }; + if (first_line[8] != ' ') return error.HttpHeadersInvalid; + const status: http.Status = @enumFromInt(parseInt3(first_line[9..12])); + const reason = mem.trimLeft(u8, first_line[12..], " "); + + res.version = version; + res.status = status; + res.reason = reason; + + res.headers.clearRetainingCapacity(); + + while (it.next()) |line| { + if (line.len == 0) return error.HttpHeadersInvalid; + switch (line[0]) { + ' ', '\t' => return error.HttpHeaderContinuationsUnsupported, + else => {}, + } + + var line_it = mem.tokenizeAny(u8, line, ": "); + const header_name = line_it.next() orelse return error.HttpHeadersInvalid; + const header_value = line_it.rest(); + + try res.headers.append(header_name, header_value); + + if (trailing) continue; + + if (std.ascii.eqlIgnoreCase(header_name, "transfer-encoding")) { + // Transfer-Encoding: second, first + // Transfer-Encoding: deflate, chunked + var iter = mem.splitBackwardsScalar(u8, header_value, ','); + + const first = iter.first(); + const trimmed_first = mem.trim(u8, first, " "); + + var next: ?[]const u8 = first; + if (std.meta.stringToEnum(http.TransferEncoding, trimmed_first)) |transfer| { + if (res.transfer_encoding != .none) return error.HttpHeadersInvalid; // we already have a transfer encoding + res.transfer_encoding = transfer; + + next = iter.next(); + } + + if (next) |second| { + const trimmed_second = mem.trim(u8, second, " "); + + if (std.meta.stringToEnum(http.ContentEncoding, trimmed_second)) |transfer| { + if (res.transfer_compression != .identity) return error.HttpHeadersInvalid; // double compression is not supported + res.transfer_compression = transfer; + } else { + return error.HttpTransferEncodingUnsupported; + } + } + + if (iter.next()) |_| return error.HttpTransferEncodingUnsupported; + } else if (std.ascii.eqlIgnoreCase(header_name, "content-length")) { + const content_length = std.fmt.parseInt(u64, header_value, 10) catch return error.InvalidContentLength; + + if (res.content_length != null and res.content_length != content_length) return error.HttpHeadersInvalid; + + res.content_length = content_length; + } else if (std.ascii.eqlIgnoreCase(header_name, "content-encoding")) { + if (res.transfer_compression != .identity) return error.HttpHeadersInvalid; + + const trimmed = mem.trim(u8, header_value, " "); + + if (std.meta.stringToEnum(http.ContentEncoding, trimmed)) |ce| { + res.transfer_compression = ce; + } else { + return error.HttpTransferEncodingUnsupported; + } + } + } + } + + inline fn int64(array: *const [8]u8) u64 { + return @bitCast(array.*); + } + + fn parseInt3(text: *const [3]u8) u10 { + if (use_vectors) { + const nnn: @Vector(3, u8) = text.*; + const zero: @Vector(3, u8) = .{ '0', '0', '0' }; + const mmm: @Vector(3, u10) = .{ 100, 10, 1 }; + return @reduce(.Add, @as(@Vector(3, u10), nnn -% zero) *% mmm); + } + return std.fmt.parseInt(u10, text, 10) catch unreachable; + } + + test parseInt3 { + const expectEqual = testing.expectEqual; + try expectEqual(@as(u10, 0), parseInt3("000")); + try expectEqual(@as(u10, 418), parseInt3("418")); + try expectEqual(@as(u10, 999), parseInt3("999")); + } + + /// The HTTP version this response is using. + version: http.Version, + + /// The status code of the response. + status: http.Status, + + /// The reason phrase of the response. + reason: []const u8, + + /// If present, the number of bytes in the response body. + content_length: ?u64 = null, + + /// If present, the transfer encoding of the response body, otherwise none. + transfer_encoding: http.TransferEncoding = .none, + + /// If present, the compression of the response body, otherwise identity (no compression). + transfer_compression: http.ContentEncoding = .identity, + + /// The headers received from the server. + headers: http.Headers, + parser: proto.HeadersParser, + compression: Compression = .none, + + /// Whether the response body should be skipped. Any data read from the response body will be discarded. + skip: bool = false, +}; + +/// A HTTP request that has been sent. +/// +/// Order of operations: open -> send[ -> write -> finish] -> wait -> read +pub const Request = struct { + /// The uri that this request is being sent to. + uri: Uri, + + /// The client that this request was created from. + client: *Client, + + /// Underlying connection to the server. This is null when the connection is released. + connection: ?*Connection, + + method: http.Method, + version: http.Version = .@"HTTP/1.1", + + /// The list of HTTP request headers. + headers: http.Headers, + + /// The transfer encoding of the request body. + transfer_encoding: RequestTransfer = .none, + + /// The redirect quota left for this request. + redirects_left: u32, + + /// Whether the request should follow redirects. + handle_redirects: bool, + + /// Whether the request should handle a 100-continue response before sending the request body. + handle_continue: bool, + + /// The response associated with this request. + /// + /// This field is undefined until `wait` is called. + response: Response, + + /// Used as a allocator for resolving redirects locations. + arena: std.heap.ArenaAllocator, + + /// Frees all resources associated with the request. + pub fn deinit(req: *Request) void { + switch (req.response.compression) { + .none => {}, + .deflate => |*deflate| deflate.deinit(), + .gzip => |*gzip| gzip.deinit(), + .zstd => |*zstd| zstd.deinit(), + } + + req.headers.deinit(); + req.response.headers.deinit(); + + if (req.response.parser.header_bytes_owned) { + req.response.parser.header_bytes.deinit(req.client.allocator); + } + + if (req.connection) |connection| { + if (!req.response.parser.done) { + // If the response wasn't fully read, then we need to close the connection. + connection.closing = true; + } + req.client.connection_pool.release(req.client.allocator, connection); + } + + req.arena.deinit(); + req.* = undefined; + } + + // This function must deallocate all resources associated with the request, or keep those which will be used + // This needs to be kept in sync with deinit and request + fn redirect(req: *Request, uri: Uri) !void { + assert(req.response.parser.done); + + switch (req.response.compression) { + .none => {}, + .deflate => |*deflate| deflate.deinit(), + .gzip => |*gzip| gzip.deinit(), + .zstd => |*zstd| zstd.deinit(), + } + + req.client.connection_pool.release(req.client.allocator, req.connection.?); + req.connection = null; + + const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme; + + const port: u16 = uri.port orelse switch (protocol) { + .plain => 80, + .tls => 443, + }; + + const host = uri.host orelse return error.UriMissingHost; + + req.uri = uri; + req.connection = try req.client.connect(host, port, protocol); + req.redirects_left -= 1; + req.response.headers.clearRetainingCapacity(); + req.response.parser.reset(); + + req.response = .{ + .status = undefined, + .reason = undefined, + .version = undefined, + .headers = req.response.headers, + .parser = req.response.parser, + }; + } + + pub const SendError = Connection.WriteError || error{ InvalidContentLength, UnsupportedTransferEncoding }; + + pub const SendOptions = struct { + /// Specifies that the uri should be used as is. You guarantee that the uri is already escaped. + raw_uri: bool = false, + }; + + /// Send the HTTP request headers to the server. + pub fn send(req: *Request, options: SendOptions) SendError!void { + if (!req.method.requestHasBody() and req.transfer_encoding != .none) return error.UnsupportedTransferEncoding; + + const w = req.connection.?.writer(); + + try req.method.write(w); + try w.writeByte(' '); + + if (req.method == .CONNECT) { + try req.uri.writeToStream(.{ .authority = true }, w); + } else { + try req.uri.writeToStream(.{ + .scheme = req.connection.?.proxied, + .authentication = req.connection.?.proxied, + .authority = req.connection.?.proxied, + .path = true, + .query = true, + .raw = options.raw_uri, + }, w); + } + try w.writeByte(' '); + try w.writeAll(@tagName(req.version)); + try w.writeAll("\r\n"); + + if (!req.headers.contains("host")) { + try w.writeAll("Host: "); + try req.uri.writeToStream(.{ .authority = true }, w); + try w.writeAll("\r\n"); + } + + if (!req.headers.contains("user-agent")) { + try w.writeAll("User-Agent: zig/"); + try w.writeAll(builtin.zig_version_string); + try w.writeAll(" (std.http)\r\n"); + } + + if (!req.headers.contains("connection")) { + try w.writeAll("Connection: keep-alive\r\n"); + } + + if (!req.headers.contains("accept-encoding")) { + try w.writeAll("Accept-Encoding: gzip, deflate, zstd\r\n"); + } + + if (!req.headers.contains("te")) { + try w.writeAll("TE: gzip, deflate, trailers\r\n"); + } + + const has_transfer_encoding = req.headers.contains("transfer-encoding"); + const has_content_length = req.headers.contains("content-length"); + + if (!has_transfer_encoding and !has_content_length) { + switch (req.transfer_encoding) { + .chunked => try w.writeAll("Transfer-Encoding: chunked\r\n"), + .content_length => |content_length| try w.print("Content-Length: {d}\r\n", .{content_length}), + .none => {}, + } + } else { + if (has_transfer_encoding) { + const transfer_encoding = req.headers.getFirstValue("transfer-encoding").?; + if (std.mem.eql(u8, transfer_encoding, "chunked")) { + req.transfer_encoding = .chunked; + } else { + return error.UnsupportedTransferEncoding; + } + } else if (has_content_length) { + const content_length = std.fmt.parseInt(u64, req.headers.getFirstValue("content-length").?, 10) catch return error.InvalidContentLength; + + req.transfer_encoding = .{ .content_length = content_length }; + } else { + req.transfer_encoding = .none; + } + } + + for (req.headers.list.items) |entry| { + if (entry.value.len == 0) continue; + + try w.writeAll(entry.name); + try w.writeAll(": "); + try w.writeAll(entry.value); + try w.writeAll("\r\n"); + } + + if (req.connection.?.proxied) { + const proxy_headers: ?http.Headers = switch (req.connection.?.protocol) { + .plain => if (req.client.http_proxy) |proxy| proxy.headers else null, + .tls => if (req.client.https_proxy) |proxy| proxy.headers else null, + }; + + if (proxy_headers) |headers| { + for (headers.list.items) |entry| { + if (entry.value.len == 0) continue; + + try w.writeAll(entry.name); + try w.writeAll(": "); + try w.writeAll(entry.value); + try w.writeAll("\r\n"); + } + } + } + + try w.writeAll("\r\n"); + + try req.connection.?.flush(); + } + + const TransferReadError = Connection.ReadError || proto.HeadersParser.ReadError; + + const TransferReader = std.io.Reader(*Request, TransferReadError, transferRead); + + fn transferReader(req: *Request) TransferReader { + return .{ .context = req }; + } + + fn transferRead(req: *Request, buf: []u8) TransferReadError!usize { + if (req.response.parser.done) return 0; + + var index: usize = 0; + while (index == 0) { + const amt = try req.response.parser.read(req.connection.?, buf[index..], req.response.skip); + if (amt == 0 and req.response.parser.done) break; + index += amt; + } + + return index; + } + + pub const WaitError = RequestError || SendError || TransferReadError || proto.HeadersParser.CheckCompleteHeadError || Response.ParseError || Uri.ParseError || error{ TooManyHttpRedirects, RedirectRequiresResend, HttpRedirectMissingLocation, CompressionInitializationFailed, CompressionNotSupported }; + + /// Waits for a response from the server and parses any headers that are sent. + /// This function will block until the final response is received. + /// + /// If `handle_redirects` is true and the request has no payload, then this function will automatically follow + /// redirects. If a request payload is present, then this function will error with error.RedirectRequiresResend. + /// + /// Must be called after `send` and, if any data was written to the request body, then also after `finish`. + pub fn wait(req: *Request) WaitError!void { + while (true) { // handle redirects + while (true) { // read headers + try req.connection.?.fill(); + + const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek()); + req.connection.?.drop(@intCast(nchecked)); + + if (req.response.parser.state.isContent()) break; + } + + try req.response.parse(req.response.parser.header_bytes.items, false); + + if (req.response.status == .@"continue") { + req.response.parser.done = true; // we're done parsing the continue response, reset to prepare for the real response + req.response.parser.reset(); + + if (req.handle_continue) + continue; + + return; // we're not handling the 100-continue, return to the caller + } + + // we're switching protocols, so this connection is no longer doing http + if (req.method == .CONNECT and req.response.status.class() == .success) { + req.connection.?.closing = false; + req.response.parser.done = true; + + return; // the connection is not HTTP past this point, return to the caller + } + + // we default to using keep-alive if not provided in the client if the server asks for it + const req_connection = req.headers.getFirstValue("connection"); + const req_keepalive = req_connection != null and !std.ascii.eqlIgnoreCase("close", req_connection.?); + + const res_connection = req.response.headers.getFirstValue("connection"); + const res_keepalive = res_connection != null and !std.ascii.eqlIgnoreCase("close", res_connection.?); + if (res_keepalive and (req_keepalive or req_connection == null)) { + req.connection.?.closing = false; + } else { + req.connection.?.closing = true; + } + + // Any response to a HEAD request and any response with a 1xx (Informational), 204 (No Content), or 304 (Not Modified) + // status code is always terminated by the first empty line after the header fields, regardless of the header fields + // present in the message + if (req.method == .HEAD or req.response.status.class() == .informational or req.response.status == .no_content or req.response.status == .not_modified) { + req.response.parser.done = true; + + return; // the response is empty, no further setup or redirection is necessary + } + + if (req.response.transfer_encoding != .none) { + switch (req.response.transfer_encoding) { + .none => unreachable, + .chunked => { + req.response.parser.next_chunk_length = 0; + req.response.parser.state = .chunk_head_size; + }, + } + } else if (req.response.content_length) |cl| { + req.response.parser.next_chunk_length = cl; + + if (cl == 0) req.response.parser.done = true; + } else { + // read until the connection is closed + req.response.parser.next_chunk_length = std.math.maxInt(u64); + } + + if (req.response.status.class() == .redirect and req.handle_redirects) { + req.response.skip = true; + + // skip the body of the redirect response, this will at least leave the connection in a known good state. + const empty = @as([*]u8, undefined)[0..0]; + assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary + + if (req.redirects_left == 0) return error.TooManyHttpRedirects; + + const location = req.response.headers.getFirstValue("location") orelse + return error.HttpRedirectMissingLocation; + + const arena = req.arena.allocator(); + + const location_duped = try arena.dupe(u8, location); + + const new_url = Uri.parse(location_duped) catch try Uri.parseWithoutScheme(location_duped); + const resolved_url = try req.uri.resolve(new_url, false, arena); + + // is the redirect location on the same domain, or a subdomain of the original request? + const is_same_domain_or_subdomain = std.ascii.endsWithIgnoreCase(resolved_url.host.?, req.uri.host.?) and (resolved_url.host.?.len == req.uri.host.?.len or resolved_url.host.?[resolved_url.host.?.len - req.uri.host.?.len - 1] == '.'); + + if (resolved_url.host == null or !is_same_domain_or_subdomain or !std.ascii.eqlIgnoreCase(resolved_url.scheme, req.uri.scheme)) { + // we're redirecting to a different domain, strip privileged headers like cookies + _ = req.headers.delete("authorization"); + _ = req.headers.delete("www-authenticate"); + _ = req.headers.delete("cookie"); + _ = req.headers.delete("cookie2"); + } + + if (req.response.status == .see_other or ((req.response.status == .moved_permanently or req.response.status == .found) and req.method == .POST)) { + // we're redirecting to a GET, so we need to change the method and remove the body + req.method = .GET; + req.transfer_encoding = .none; + _ = req.headers.delete("transfer-encoding"); + _ = req.headers.delete("content-length"); + _ = req.headers.delete("content-type"); + } + + if (req.transfer_encoding != .none) { + return error.RedirectRequiresResend; // The request body has already been sent. The request is still in a valid state, but the redirect must be handled manually. + } + + try req.redirect(resolved_url); + + try req.send(.{}); + } else { + req.response.skip = false; + if (!req.response.parser.done) { + switch (req.response.transfer_compression) { + .identity => req.response.compression = .none, + .compress, .@"x-compress" => return error.CompressionNotSupported, + .deflate => req.response.compression = .{ + .deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed, + }, + .gzip, .@"x-gzip" => req.response.compression = .{ + .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed, + }, + .zstd => req.response.compression = .{ + .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()), + }, + } + } + + break; + } + } + } + + pub const ReadError = TransferReadError || proto.HeadersParser.CheckCompleteHeadError || error{ DecompressionFailure, InvalidTrailers }; + + pub const Reader = std.io.Reader(*Request, ReadError, read); + + pub fn reader(req: *Request) Reader { + return .{ .context = req }; + } + + /// Reads data from the response body. Must be called after `wait`. + pub fn read(req: *Request, buffer: []u8) ReadError!usize { + const out_index = switch (req.response.compression) { + .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure, + .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure, + .zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure, + else => try req.transferRead(buffer), + }; + + if (out_index == 0) { + const has_trail = !req.response.parser.state.isContent(); + + while (!req.response.parser.state.isContent()) { // read trailing headers + try req.connection.?.fill(); + + const nchecked = try req.response.parser.checkCompleteHead(req.client.allocator, req.connection.?.peek()); + req.connection.?.drop(@intCast(nchecked)); + } + + if (has_trail) { + // The response headers before the trailers are already guaranteed to be valid, so they will always be parsed again and cannot return an error. + // This will *only* fail for a malformed trailer. + req.response.parse(req.response.parser.header_bytes.items, true) catch return error.InvalidTrailers; + } + } + + return out_index; + } + + /// Reads data from the response body. Must be called after `wait`. + pub fn readAll(req: *Request, buffer: []u8) !usize { + var index: usize = 0; + while (index < buffer.len) { + const amt = try read(req, buffer[index..]); + if (amt == 0) break; + index += amt; + } + return index; + } + + pub const WriteError = Connection.WriteError || error{ NotWriteable, MessageTooLong }; + + pub const Writer = std.io.Writer(*Request, WriteError, write); + + pub fn writer(req: *Request) Writer { + return .{ .context = req }; + } + + /// Write `bytes` to the server. The `transfer_encoding` field determines how data will be sent. + /// Must be called after `send` and before `finish`. + pub fn write(req: *Request, bytes: []const u8) WriteError!usize { + switch (req.transfer_encoding) { + .chunked => { + try req.connection.?.writer().print("{x}\r\n", .{bytes.len}); + try req.connection.?.writer().writeAll(bytes); + try req.connection.?.writer().writeAll("\r\n"); + + return bytes.len; + }, + .content_length => |*len| { + if (len.* < bytes.len) return error.MessageTooLong; + + const amt = try req.connection.?.write(bytes); + len.* -= amt; + return amt; + }, + .none => return error.NotWriteable, + } + } + + /// Write `bytes` to the server. The `transfer_encoding` field determines how data will be sent. + /// Must be called after `send` and before `finish`. + pub fn writeAll(req: *Request, bytes: []const u8) WriteError!void { + var index: usize = 0; + while (index < bytes.len) { + index += try write(req, bytes[index..]); + } + } + + pub const FinishError = WriteError || error{MessageNotCompleted}; + + /// Finish the body of a request. This notifies the server that you have no more data to send. + /// Must be called after `send`. + pub fn finish(req: *Request) FinishError!void { + switch (req.transfer_encoding) { + .chunked => try req.connection.?.writer().writeAll("0\r\n\r\n"), + .content_length => |len| if (len != 0) return error.MessageNotCompleted, + .none => {}, + } + + try req.connection.?.flush(); + } +}; + +/// A HTTP proxy server. +pub const Proxy = struct { + allocator: Allocator, + headers: http.Headers, + + protocol: Connection.Protocol, + host: []const u8, + port: u16, + + supports_connect: bool = true, +}; + +/// Release all associated resources with the client. +/// +/// All pending requests must be de-initialized and all active connections released +/// before calling this function. +pub fn deinit(client: *Client) void { + assert(client.connection_pool.used.first == null); // There are still active requests. + + client.connection_pool.deinit(client.allocator); + + if (client.http_proxy) |*proxy| { + proxy.allocator.free(proxy.host); + proxy.headers.deinit(); + } + + if (client.https_proxy) |*proxy| { + proxy.allocator.free(proxy.host); + proxy.headers.deinit(); + } + + if (!disable_tls) + client.ca_bundle.deinit(client.allocator); + + client.* = undefined; +} + +/// Uses the *_proxy environment variable to set any unset proxies for the client. +/// This function *must not* be called when the client has any active connections. +pub fn loadDefaultProxies(client: *Client) !void { + // Prevent any new connections from being created. + client.connection_pool.mutex.lock(); + defer client.connection_pool.mutex.unlock(); + + assert(client.connection_pool.used.first == null); // There are still active requests. + + if (client.http_proxy == null) http: { + const content: []const u8 = if (std.process.hasEnvVarConstant("http_proxy")) + try std.process.getEnvVarOwned(client.allocator, "http_proxy") + else if (std.process.hasEnvVarConstant("HTTP_PROXY")) + try std.process.getEnvVarOwned(client.allocator, "HTTP_PROXY") + else if (std.process.hasEnvVarConstant("all_proxy")) + try std.process.getEnvVarOwned(client.allocator, "all_proxy") + else if (std.process.hasEnvVarConstant("ALL_PROXY")) + try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY") + else + break :http; + defer client.allocator.free(content); + + const uri = Uri.parse(content) catch + Uri.parseWithoutScheme(content) catch + break :http; + + const protocol = if (uri.scheme.len == 0) + .plain // No scheme, assume http:// + else + protocol_map.get(uri.scheme) orelse break :http; // Unknown scheme, ignore + + const host = if (uri.host) |host| try client.allocator.dupe(u8, host) else break :http; // Missing host, ignore + client.http_proxy = .{ + .allocator = client.allocator, + .headers = .{ .allocator = client.allocator }, + + .protocol = protocol, + .host = host, + .port = uri.port orelse switch (protocol) { + .plain => 80, + .tls => 443, + }, + }; + + if (uri.user != null and uri.password != null) { + const prefix = "Basic "; + + const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? }); + defer client.allocator.free(unencoded); + + const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len) + prefix.len); + defer client.allocator.free(buffer); + + const result = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); + @memcpy(buffer[0..prefix.len], prefix); + + try client.http_proxy.?.headers.append("proxy-authorization", result); + } + } + + if (client.https_proxy == null) https: { + const content: []const u8 = if (std.process.hasEnvVarConstant("https_proxy")) + try std.process.getEnvVarOwned(client.allocator, "https_proxy") + else if (std.process.hasEnvVarConstant("HTTPS_PROXY")) + try std.process.getEnvVarOwned(client.allocator, "HTTPS_PROXY") + else if (std.process.hasEnvVarConstant("all_proxy")) + try std.process.getEnvVarOwned(client.allocator, "all_proxy") + else if (std.process.hasEnvVarConstant("ALL_PROXY")) + try std.process.getEnvVarOwned(client.allocator, "ALL_PROXY") + else + break :https; + defer client.allocator.free(content); + + const uri = Uri.parse(content) catch + Uri.parseWithoutScheme(content) catch + break :https; + + const protocol = if (uri.scheme.len == 0) + .plain // No scheme, assume http:// + else + protocol_map.get(uri.scheme) orelse break :https; // Unknown scheme, ignore + + const host = if (uri.host) |host| try client.allocator.dupe(u8, host) else break :https; // Missing host, ignore + client.https_proxy = .{ + .allocator = client.allocator, + .headers = .{ .allocator = client.allocator }, + + .protocol = protocol, + .host = host, + .port = uri.port orelse switch (protocol) { + .plain => 80, + .tls => 443, + }, + }; + + if (uri.user != null and uri.password != null) { + const prefix = "Basic "; + + const unencoded = try std.fmt.allocPrint(client.allocator, "{s}:{s}", .{ uri.user.?, uri.password.? }); + defer client.allocator.free(unencoded); + + const buffer = try client.allocator.alloc(u8, std.base64.standard.Encoder.calcSize(unencoded.len) + prefix.len); + defer client.allocator.free(buffer); + + const result = std.base64.standard.Encoder.encode(buffer[prefix.len..], unencoded); + @memcpy(buffer[0..prefix.len], prefix); + + try client.https_proxy.?.headers.append("proxy-authorization", result); + } + } +} + +pub const ConnectTcpError = Allocator.Error || error{ ConnectionRefused, NetworkUnreachable, ConnectionTimedOut, ConnectionResetByPeer, TemporaryNameServerFailure, NameServerFailure, UnknownHostName, HostLacksNetworkAddresses, UnexpectedConnectFailure, TlsInitializationFailed }; + +/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open. +/// +/// This function is threadsafe. +pub fn connectTcp(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectTcpError!*Connection { + if (client.connection_pool.findConnection(.{ + .host = host, + .port = port, + .protocol = protocol, + })) |node| + return node; + + if (disable_tls and protocol == .tls) + return error.TlsInitializationFailed; + + const conn = try client.allocator.create(ConnectionPool.Node); + errdefer client.allocator.destroy(conn); + conn.* = .{ .data = undefined }; + + const stream = tcp.tcpConnectToHost(client.allocator, client.loop, host, port) catch |err| switch (err) { + error.ConnectionRefused => return error.ConnectionRefused, + error.NetworkUnreachable => return error.NetworkUnreachable, + error.ConnectionTimedOut => return error.ConnectionTimedOut, + error.ConnectionResetByPeer => return error.ConnectionResetByPeer, + error.TemporaryNameServerFailure => return error.TemporaryNameServerFailure, + error.NameServerFailure => return error.NameServerFailure, + error.UnknownHostName => return error.UnknownHostName, + error.HostLacksNetworkAddresses => return error.HostLacksNetworkAddresses, + else => return error.UnexpectedConnectFailure, + }; + errdefer stream.close(); + + conn.data = .{ + .stream = stream, + .tls_client = undefined, + + .protocol = protocol, + .host = try client.allocator.dupe(u8, host), + .port = port, + }; + errdefer client.allocator.free(conn.data.host); + + if (protocol == .tls) { + if (disable_tls) unreachable; + + conn.data.tls_client = try client.allocator.create(std.crypto.tls.Client); + errdefer client.allocator.destroy(conn.data.tls_client); + + conn.data.tls_client.* = std.crypto.tls.Client.init(stream, client.ca_bundle, host) catch return error.TlsInitializationFailed; + // This is appropriate for HTTPS because the HTTP headers contain + // the content length which is used to detect truncation attacks. + conn.data.tls_client.allow_truncation_attacks = true; + } + + client.connection_pool.addUsed(conn); + + return &conn.data; +} + +/// Connect to `tunnel_host:tunnel_port` using the specified proxy with HTTP CONNECT. This will reuse a connection if one is already open. +/// +/// This function is threadsafe. +pub fn connectTunnel( + client: *Client, + proxy: *Proxy, + tunnel_host: []const u8, + tunnel_port: u16, +) !*Connection { + if (!proxy.supports_connect) return error.TunnelNotSupported; + + if (client.connection_pool.findConnection(.{ + .host = tunnel_host, + .port = tunnel_port, + .protocol = proxy.protocol, + })) |node| + return node; + + var maybe_valid = false; + (tunnel: { + const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol); + errdefer { + conn.closing = true; + client.connection_pool.release(client.allocator, conn); + } + + const uri = Uri{ + .scheme = "http", + .user = null, + .password = null, + .host = tunnel_host, + .port = tunnel_port, + .path = "", + .query = null, + .fragment = null, + }; + + // we can use a small buffer here because a CONNECT response should be very small + var buffer: [8096]u8 = undefined; + + var req = client.open(.CONNECT, uri, proxy.headers, .{ + .handle_redirects = false, + .connection = conn, + .header_strategy = .{ .static = &buffer }, + }) catch |err| { + std.log.debug("err {}", .{err}); + break :tunnel err; + }; + defer req.deinit(); + + req.send(.{ .raw_uri = true }) catch |err| break :tunnel err; + req.wait() catch |err| break :tunnel err; + + if (req.response.status.class() == .server_error) { + maybe_valid = true; + break :tunnel error.ServerError; + } + + if (req.response.status != .ok) break :tunnel error.ConnectionRefused; + + // this connection is now a tunnel, so we can't use it for anything else, it will only be released when the client is de-initialized. + req.connection = null; + + client.allocator.free(conn.host); + conn.host = try client.allocator.dupe(u8, tunnel_host); + errdefer client.allocator.free(conn.host); + + conn.port = tunnel_port; + conn.closing = false; + + return conn; + }) catch { + // something went wrong with the tunnel + proxy.supports_connect = maybe_valid; + return error.TunnelNotSupported; + }; +} + +// Prevents a dependency loop in open() +const ConnectErrorPartial = ConnectTcpError || error{ UnsupportedUrlScheme, ConnectionRefused }; +pub const ConnectError = ConnectErrorPartial || RequestError; + +/// Connect to `host:port` using the specified protocol. This will reuse a connection if one is already open. +/// If a proxy is configured for the client, then the proxy will be used to connect to the host. +/// +/// This function is threadsafe. +pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) ConnectError!*Connection { + // pointer required so that `supports_connect` can be updated if a CONNECT fails + const potential_proxy: ?*Proxy = switch (protocol) { + .plain => if (client.http_proxy) |*proxy_info| proxy_info else null, + .tls => if (client.https_proxy) |*proxy_info| proxy_info else null, + }; + + if (potential_proxy) |proxy| { + // don't attempt to proxy the proxy thru itself. + if (std.mem.eql(u8, proxy.host, host) and proxy.port == port and proxy.protocol == protocol) { + return client.connectTcp(host, port, protocol); + } + + if (proxy.supports_connect) tunnel: { + return connectTunnel(client, proxy, host, port) catch |err| switch (err) { + error.TunnelNotSupported => break :tunnel, + else => |e| return e, + }; + } + + // fall back to using the proxy as a normal http proxy + const conn = try client.connectTcp(proxy.host, proxy.port, proxy.protocol); + errdefer { + conn.closing = true; + client.connection_pool.release(conn); + } + + conn.proxied = true; + return conn; + } + + return client.connectTcp(host, port, protocol); +} + +pub const RequestError = ConnectTcpError || ConnectErrorPartial || Request.SendError || std.fmt.ParseIntError || Connection.WriteError || error{ + UnsupportedUrlScheme, + UriMissingHost, + + CertificateBundleLoadFailure, + UnsupportedTransferEncoding, +}; + +pub const RequestOptions = struct { + version: http.Version = .@"HTTP/1.1", + + /// Automatically ignore 100 Continue responses. This assumes you don't care, and will have sent the body before you + /// wait for the response. + /// + /// If this is not the case AND you know the server will send a 100 Continue, set this to false and wait for a + /// response before sending the body. If you wait AND the server does not send a 100 Continue before you finish the + /// request, then the request *will* deadlock. + handle_continue: bool = true, + + /// Automatically follow redirects. This will only follow redirects for repeatable requests (ie. with no payload or the server has acknowledged the payload) + handle_redirects: bool = true, + + /// How many redirects to follow before returning an error. + max_redirects: u32 = 3, + header_strategy: StorageStrategy = .{ .dynamic = 16 * 1024 }, + + /// Must be an already acquired connection. + connection: ?*Connection = null, + + pub const StorageStrategy = union(enum) { + /// In this case, the client's Allocator will be used to store the + /// entire HTTP header. This value is the maximum total size of + /// HTTP headers allowed, otherwise + /// error.HttpHeadersExceededSizeLimit is returned from read(). + dynamic: usize, + /// This is used to store the entire HTTP header. If the HTTP + /// header is too big to fit, `error.HttpHeadersExceededSizeLimit` + /// is returned from read(). When this is used, `error.OutOfMemory` + /// cannot be returned from `read()`. + static: []u8, + }; +}; + +pub const protocol_map = std.ComptimeStringMap(Connection.Protocol, .{ + .{ "http", .plain }, + .{ "ws", .plain }, + .{ "https", .tls }, + .{ "wss", .tls }, +}); + +/// Open a connection to the host specified by `uri` and prepare to send a HTTP request. +/// +/// `uri` must remain alive during the entire request. +/// `headers` is cloned and may be freed after this function returns. +/// +/// The caller is responsible for calling `deinit()` on the `Request`. +/// This function is threadsafe. +pub fn open(client: *Client, method: http.Method, uri: Uri, headers: http.Headers, options: RequestOptions) RequestError!Request { + const protocol = protocol_map.get(uri.scheme) orelse return error.UnsupportedUrlScheme; + + const port: u16 = uri.port orelse switch (protocol) { + .plain => 80, + .tls => 443, + }; + + const host = uri.host orelse return error.UriMissingHost; + + if (protocol == .tls and @atomicLoad(bool, &client.next_https_rescan_certs, .Acquire)) { + if (disable_tls) unreachable; + + client.ca_bundle_mutex.lock(); + defer client.ca_bundle_mutex.unlock(); + + if (client.next_https_rescan_certs) { + client.ca_bundle.rescan(client.allocator) catch return error.CertificateBundleLoadFailure; + @atomicStore(bool, &client.next_https_rescan_certs, false, .Release); + } + } + + const conn = options.connection orelse try client.connect(host, port, protocol); + + var req: Request = .{ + .uri = uri, + .client = client, + .connection = conn, + .headers = try headers.clone(client.allocator), // Headers must be cloned to properly handle header transformations in redirects. + .method = method, + .version = options.version, + .redirects_left = options.max_redirects, + .handle_redirects = options.handle_redirects, + .handle_continue = options.handle_continue, + .response = .{ + .status = undefined, + .reason = undefined, + .version = undefined, + .headers = http.Headers{ .allocator = client.allocator, .owned = false }, + .parser = switch (options.header_strategy) { + .dynamic => |max| proto.HeadersParser.initDynamic(max), + .static => |buf| proto.HeadersParser.initStatic(buf), + }, + }, + .arena = undefined, + }; + errdefer req.deinit(); + + req.arena = std.heap.ArenaAllocator.init(client.allocator); + + return req; +} + +pub const FetchOptions = struct { + pub const Location = union(enum) { + url: []const u8, + uri: Uri, + }; + + pub const Payload = union(enum) { + string: []const u8, + file: std.fs.File, + none, + }; + + pub const ResponseStrategy = union(enum) { + storage: RequestOptions.StorageStrategy, + file: std.fs.File, + none, + }; + + header_strategy: RequestOptions.StorageStrategy = .{ .dynamic = 16 * 1024 }, + response_strategy: ResponseStrategy = .{ .storage = .{ .dynamic = 16 * 1024 * 1024 } }, + + location: Location, + method: http.Method = .GET, + headers: http.Headers = http.Headers{ .allocator = std.heap.page_allocator, .owned = false }, + payload: Payload = .none, + raw_uri: bool = false, +}; + +pub const FetchResult = struct { + status: http.Status, + body: ?[]const u8 = null, + headers: http.Headers, + + allocator: Allocator, + options: FetchOptions, + + pub fn deinit(res: *FetchResult) void { + if (res.options.response_strategy == .storage and res.options.response_strategy.storage == .dynamic) { + if (res.body) |body| res.allocator.free(body); + } + + res.headers.deinit(); + } +}; + +/// Perform a one-shot HTTP request with the provided options. +/// +/// This function is threadsafe. +pub fn fetch(client: *Client, allocator: Allocator, options: FetchOptions) !FetchResult { + const has_transfer_encoding = options.headers.contains("transfer-encoding"); + const has_content_length = options.headers.contains("content-length"); + + if (has_content_length or has_transfer_encoding) return error.UnsupportedHeader; + + const uri = switch (options.location) { + .url => |u| try Uri.parse(u), + .uri => |u| u, + }; + + var req = try open(client, options.method, uri, options.headers, .{ + .header_strategy = options.header_strategy, + .handle_redirects = options.payload == .none, + }); + defer req.deinit(); + + { // Block to maintain lock of file to attempt to prevent a race condition where another process modifies the file while we are reading it. + // This relies on other processes actually obeying the advisory lock, which is not guaranteed. + if (options.payload == .file) try options.payload.file.lock(.shared); + defer if (options.payload == .file) options.payload.file.unlock(); + + switch (options.payload) { + .string => |str| req.transfer_encoding = .{ .content_length = str.len }, + .file => |file| req.transfer_encoding = .{ .content_length = (try file.stat()).size }, + .none => {}, + } + + try req.send(.{ .raw_uri = options.raw_uri }); + + switch (options.payload) { + .string => |str| try req.writeAll(str), + .file => |file| { + try file.seekTo(0); + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init(); + try fifo.pump(file.reader(), req.writer()); + }, + .none => {}, + } + + try req.finish(); + } + + try req.wait(); + + var res = FetchResult{ + .status = req.response.status, + .headers = try req.response.headers.clone(allocator), + + .allocator = allocator, + .options = options, + }; + + switch (options.response_strategy) { + .storage => |storage| switch (storage) { + .dynamic => |max| res.body = try req.reader().readAllAlloc(allocator, max), + .static => |buf| res.body = buf[0..try req.reader().readAll(buf)], + }, + .file => |file| { + var fifo = std.fifo.LinearFifo(u8, .{ .Static = 8192 }).init(); + try fifo.pump(req.reader(), file.writer()); + }, + .none => { // Take advantage of request internals to discard the response body and make the connection available for another request. + req.response.skip = true; + + const empty = @as([*]u8, undefined)[0..0]; + assert(try req.transferRead(empty) == 0); // we're skipping, no buffer is necessary + }, + } + + return res; +} + +test { + const native_endian = comptime builtin.cpu.arch.endian(); + if (builtin.zig_backend == .stage2_llvm and native_endian == .big) { + // https://github.com/ziglang/zig/issues/13782 + return error.SkipZigTest; + } + + if (builtin.os.tag == .wasi) return error.SkipZigTest; + + if (builtin.zig_backend == .stage2_x86_64 and + !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .avx)) return error.SkipZigTest; + + std.testing.refAllDecls(@This()); +} diff --git a/src/async/stream.zig b/src/async/stream.zig new file mode 100644 index 00000000..4aa5c04f --- /dev/null +++ b/src/async/stream.zig @@ -0,0 +1,115 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const os = std.os; +const io = std.io; +const assert = std.debug.assert; + +const tcp = @import("tcp.zig"); + +pub const Stream = struct { + alloc: std.mem.Allocator, + conn: *tcp.Conn, + + handle: std.os.socket_t, + + pub fn close(self: Stream) void { + os.closeSocket(self.handle); + self.alloc.destroy(self.conn); + } + + pub const ReadError = os.ReadError; + pub const WriteError = os.WriteError; + + pub const Reader = io.Reader(Stream, ReadError, read); + pub const Writer = io.Writer(Stream, WriteError, write); + + pub fn reader(self: Stream) Reader { + return .{ .context = self }; + } + + pub fn writer(self: Stream) Writer { + return .{ .context = self }; + } + + pub fn read(self: Stream, buffer: []u8) ReadError!usize { + return self.conn.receive(self.handle, buffer) catch |err| switch (err) { + else => return error.Unexpected, + }; + } + + pub fn readv(s: Stream, iovecs: []const os.iovec) ReadError!usize { + return os.readv(s.handle, iovecs); + } + + /// Returns the number of bytes read. If the number read is smaller than + /// `buffer.len`, it means the stream reached the end. Reaching the end of + /// a stream is not an error condition. + pub fn readAll(s: Stream, buffer: []u8) ReadError!usize { + return readAtLeast(s, buffer, buffer.len); + } + + /// Returns the number of bytes read, calling the underlying read function + /// the minimal number of times until the buffer has at least `len` bytes + /// filled. If the number read is less than `len` it means the stream + /// reached the end. Reaching the end of the stream is not an error + /// condition. + pub fn readAtLeast(s: Stream, buffer: []u8, len: usize) ReadError!usize { + assert(len <= buffer.len); + var index: usize = 0; + while (index < len) { + const amt = try s.read(buffer[index..]); + if (amt == 0) break; + index += amt; + } + return index; + } + + /// TODO in evented I/O mode, this implementation incorrectly uses the event loop's + /// file system thread instead of non-blocking. It needs to be reworked to properly + /// use non-blocking I/O. + pub fn write(self: Stream, buffer: []const u8) WriteError!usize { + return self.conn.send(self.handle, buffer) catch |err| switch (err) { + error.AccessDenied => error.AccessDenied, + error.WouldBlock => error.WouldBlock, + error.ConnectionResetByPeer => error.ConnectionResetByPeer, + error.MessageTooBig => error.FileTooBig, + error.BrokenPipe => error.BrokenPipe, + else => return error.Unexpected, + }; + } + + pub fn writeAll(self: Stream, bytes: []const u8) WriteError!void { + var index: usize = 0; + while (index < bytes.len) { + index += try self.write(bytes[index..]); + } + } + + /// See https://github.com/ziglang/zig/issues/7699 + /// See equivalent function: `std.fs.File.writev`. + pub fn writev(self: Stream, iovecs: []const os.iovec_const) WriteError!usize { + if (iovecs.len == 0) return 0; + const first_buffer = iovecs[0].iov_base[0..iovecs[0].iov_len]; + return try self.write(first_buffer); + } + + /// The `iovecs` parameter is mutable because this function needs to mutate the fields in + /// order to handle partial writes from the underlying OS layer. + /// See https://github.com/ziglang/zig/issues/7699 + /// See equivalent function: `std.fs.File.writevAll`. + pub fn writevAll(self: Stream, iovecs: []os.iovec_const) WriteError!void { + if (iovecs.len == 0) return; + + var i: usize = 0; + while (true) { + var amt = try self.writev(iovecs[i..]); + while (amt >= iovecs[i].iov_len) { + amt -= iovecs[i].iov_len; + i += 1; + if (i >= iovecs.len) return; + } + iovecs[i].iov_base += amt; + iovecs[i].iov_len -= amt; + } + } +}; diff --git a/src/async/tcp.zig b/src/async/tcp.zig new file mode 100644 index 00000000..c4284bfd --- /dev/null +++ b/src/async/tcp.zig @@ -0,0 +1,94 @@ +const std = @import("std"); +const net = std.net; +const Stream = @import("stream.zig").Stream; +const Loop = @import("jsruntime").Loop; +const NetworkImpl = Loop.Network(Conn.Command); + +// Conn is a TCP connection using jsruntime Loop async I/O. +// connect, send and receive are blocking, but use async I/O in the background. +// Client doesn't own the socket used for the connection, the caller is +// responsible for closing it. +pub const Conn = struct { + const Command = struct { + impl: NetworkImpl, + + done: bool = false, + err: ?anyerror = null, + ln: usize = 0, + + fn ok(self: *Command, err: ?anyerror, ln: usize) void { + self.err = err; + self.ln = ln; + self.done = true; + } + + fn wait(self: *Command) !usize { + while (!self.done) try self.impl.tick(); + + if (self.err) |err| return err; + return self.ln; + } + pub fn onConnect(self: *Command, err: ?anyerror) void { + self.ok(err, 0); + } + pub fn onSend(self: *Command, ln: usize, err: ?anyerror) void { + self.ok(err, ln); + } + pub fn onReceive(self: *Command, ln: usize, err: ?anyerror) void { + self.ok(err, ln); + } + }; + + loop: *Loop, + + pub fn connect(self: *Conn, socket: std.os.socket_t, address: std.net.Address) !void { + var cmd = Command{ .impl = NetworkImpl.init(self.loop) }; + cmd.impl.connect(&cmd, socket, address); + _ = try cmd.wait(); + } + + pub fn send(self: *Conn, socket: std.os.socket_t, buffer: []const u8) !usize { + var cmd = Command{ .impl = NetworkImpl.init(self.loop) }; + cmd.impl.send(&cmd, socket, buffer); + return try cmd.wait(); + } + + pub fn receive(self: *Conn, socket: std.os.socket_t, buffer: []u8) !usize { + var cmd = Command{ .impl = NetworkImpl.init(self.loop) }; + cmd.impl.receive(&cmd, socket, buffer); + return try cmd.wait(); + } +}; + +pub fn tcpConnectToHost(alloc: std.mem.Allocator, loop: *Loop, name: []const u8, port: u16) !Stream { + // TODO async resolve + const list = try net.getAddressList(alloc, name, port); + defer list.deinit(); + + if (list.addrs.len == 0) return error.UnknownHostName; + + for (list.addrs) |addr| { + return tcpConnectToAddress(alloc, loop, addr) catch |err| switch (err) { + error.ConnectionRefused => { + continue; + }, + else => return err, + }; + } + return std.os.ConnectError.ConnectionRefused; +} + +pub fn tcpConnectToAddress(alloc: std.mem.Allocator, loop: *Loop, addr: net.Address) !Stream { + const sockfd = try std.os.socket(addr.any.family, std.os.SOCK.STREAM, std.os.IPPROTO.TCP); + errdefer std.os.closeSocket(sockfd); + + var conn = try alloc.create(Conn); + conn.* = Conn{ .loop = loop }; + try conn.connect(sockfd, addr); + + return Stream{ + .alloc = alloc, + .conn = conn, + .handle = sockfd, + }; +} diff --git a/src/async/test.zig b/src/async/test.zig new file mode 100644 index 00000000..56bfc09c --- /dev/null +++ b/src/async/test.zig @@ -0,0 +1,172 @@ +const std = @import("std"); +const http = std.http; +const Client = @import("Client.zig"); +const Request = @import("Client.zig").Request; + +pub const Loop = @import("jsruntime").Loop; + +const url = "https://w3.org"; + +test "blocking mode fetch API" { + const alloc = std.testing.allocator; + + var loop = try Loop.init(alloc); + defer loop.deinit(); + + var client: Client = .{ + .allocator = alloc, + .loop = &loop, + }; + defer client.deinit(); + + // force client's CA cert scan from system. + try client.ca_bundle.rescan(client.allocator); + + var res = try client.fetch(alloc, .{ + .location = .{ .uri = try std.Uri.parse(url) }, + .payload = .none, + }); + defer res.deinit(); + + try std.testing.expect(res.status == .ok); +} + +test "blocking mode open/send/wait API" { + const alloc = std.testing.allocator; + + var loop = try Loop.init(alloc); + defer loop.deinit(); + + var client: Client = .{ + .allocator = alloc, + .loop = &loop, + }; + defer client.deinit(); + + // force client's CA cert scan from system. + try client.ca_bundle.rescan(client.allocator); + + var headers = try std.http.Headers.initList(alloc, &[_]std.http.Field{}); + defer headers.deinit(); + + var req = try client.open(.GET, try std.Uri.parse(url), headers, .{}); + defer req.deinit(); + + try req.send(.{}); + try req.finish(); + try req.wait(); + + try std.testing.expect(req.response.status == .ok); +} + +// Example how to write an async http client using the modified standard client. +const AsyncClient = struct { + cli: Client, + + const YieldImpl = Loop.Yield(AsyncRequest); + const AsyncRequest = struct { + const State = enum { new, open, send, finish, wait, done }; + + cli: *Client, + uri: std.Uri, + headers: std.http.Headers, + + req: ?Request = undefined, + state: State = .new, + + impl: YieldImpl, + err: ?anyerror = null, + + pub fn deinit(self: *AsyncRequest) void { + if (self.req) |*r| r.deinit(); + self.headers.deinit(); + } + + pub fn fetch(self: *AsyncRequest) void { + self.state = .new; + return self.impl.yield(self); + } + + fn onerr(self: *AsyncRequest, err: anyerror) void { + self.state = .done; + self.err = err; + } + + pub fn onYield(self: *AsyncRequest, err: ?anyerror) void { + if (err) |e| return self.onerr(e); + + switch (self.state) { + .new => { + self.state = .open; + self.req = self.cli.open(.GET, self.uri, self.headers, .{}) catch |e| return self.onerr(e); + }, + .open => { + self.state = .send; + self.req.?.send(.{}) catch |e| return self.onerr(e); + }, + .send => { + self.state = .finish; + self.req.?.finish() catch |e| return self.onerr(e); + }, + .finish => { + self.state = .wait; + self.req.?.wait() catch |e| return self.onerr(e); + }, + .wait => { + self.state = .done; + return; + }, + .done => return, + } + + return self.impl.yield(self); + } + + pub fn wait(self: *AsyncRequest) !void { + while (self.state != .done) try self.impl.tick(); + if (self.err) |err| return err; + } + }; + + pub fn init(alloc: std.mem.Allocator, loop: *Loop) AsyncClient { + return .{ + .cli = .{ + .allocator = alloc, + .loop = loop, + }, + }; + } + + pub fn deinit(self: *AsyncClient) void { + self.cli.deinit(); + } + + pub fn createRequest(self: *AsyncClient, uri: std.Uri) !AsyncRequest { + return .{ + .impl = YieldImpl.init(self.cli.loop), + .cli = &self.cli, + .uri = uri, + .headers = .{ .allocator = self.cli.allocator, .owned = false }, + }; + } +}; + +test "non blocking client" { + const alloc = std.testing.allocator; + + var loop = try Loop.init(alloc); + defer loop.deinit(); + + var client = AsyncClient.init(alloc, &loop); + defer client.deinit(); + + var reqs: [3]AsyncClient.AsyncRequest = undefined; + for (0..reqs.len) |i| { + reqs[i] = try client.createRequest(try std.Uri.parse(url)); + reqs[i].fetch(); + } + for (0..reqs.len) |i| { + try reqs[i].wait(); + reqs[i].deinit(); + } +} diff --git a/src/browser/browser.zig b/src/browser/browser.zig index 00dbc9c2..8de182b8 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -176,7 +176,15 @@ pub const Page = struct { log.info("GET {any} {d}", .{ self.uri, req.response.status }); // TODO handle redirection - if (req.response.status != .ok) return error.BadStatusCode; + if (req.response.status != .ok) { + log.debug("{?} {d} {s}\n{any}", .{ + req.response.version, + req.response.status, + req.response.reason, + req.response.headers, + }); + return error.BadStatusCode; + } // TODO handle charset // https://html.spec.whatwg.org/#content-type diff --git a/src/browser/mime.zig b/src/browser/mime.zig index a880daee..f7ec4763 100644 --- a/src/browser/mime.zig +++ b/src/browser/mime.zig @@ -17,6 +17,7 @@ params: []const u8 = "", charset: ?[]const u8 = null, boundary: ?[]const u8 = null, +pub const Empty = Self{ .mtype = "", .msubtype = "" }; pub const HTML = Self{ .mtype = "text", .msubtype = "html" }; pub const Javascript = Self{ .mtype = "application", .msubtype = "javascript" }; diff --git a/src/events/event.zig b/src/events/event.zig index fe939621..83f12824 100644 --- a/src/events/event.zig +++ b/src/events/event.zig @@ -13,6 +13,16 @@ const DOMException = @import("../dom/exceptions.zig").DOMException; const EventTarget = @import("../dom/event_target.zig").EventTarget; const EventTargetUnion = @import("../dom/event_target.zig").Union; +const ProgressEvent = @import("../xhr/progress_event.zig").ProgressEvent; + +// Event interfaces +pub const Interfaces = generate.Tuple(.{ + Event, + ProgressEvent, +}); +const Generated = generate.Union.compile(Interfaces); +pub const Union = Generated._union; + // https://dom.spec.whatwg.org/#event pub const Event = struct { pub const Self = parser.Event; @@ -28,6 +38,13 @@ pub const Event = struct { pub const _AT_TARGET = 2; pub const _BUBBLING_PHASE = 3; + pub fn toInterface(evt: *parser.Event) !Union { + return switch (try parser.eventGetInternalType(evt)) { + .event => .{ .Event = evt }, + .progress_event => .{ .ProgressEvent = @as(*ProgressEvent, @ptrCast(evt)).* }, + }; + } + pub fn constructor(eventType: []const u8, opts: ?EventInit) !*parser.Event { const event = try parser.eventCreate(); try parser.eventInit(event, eventType, opts orelse EventInit{}); @@ -104,13 +121,6 @@ pub const Event = struct { } }; -// Event interfaces -pub const Interfaces = generate.Tuple(.{ - Event, -}); -const Generated = generate.Union.compile(Interfaces); -pub const Union = Generated._union; - pub fn testExecFn( _: std.mem.Allocator, js_env: *jsruntime.Env, @@ -198,4 +208,13 @@ pub fn testExecFn( .{ .src = "nb", .ex = "1" }, }; try checkCases(js_env, &legacy); + + var remove = [_]Case{ + .{ .src = "var nb = 0; var evt = null; function cbk(event) { nb ++; evt=event; }", .ex = "undefined" }, + .{ .src = "document.addEventListener('count', cbk)", .ex = "undefined" }, + .{ .src = "document.removeEventListener('count', cbk)", .ex = "undefined" }, + .{ .src = "document.dispatchEvent(new Event('count'))", .ex = "true" }, + .{ .src = "nb", .ex = "0" }, + }; + try checkCases(js_env, &remove); } diff --git a/src/netsurf.zig b/src/netsurf.zig index b0d7c387..442a0fb7 100644 --- a/src/netsurf.zig +++ b/src/netsurf.zig @@ -4,9 +4,11 @@ const c = @cImport({ @cInclude("dom/dom.h"); @cInclude("dom/bindings/hubbub/parser.h"); @cInclude("events/event_target.h"); + @cInclude("events/event.h"); }); const Callback = @import("jsruntime").Callback; +const EventToInterface = @import("events/event.zig").Event.toInterface; // Vtable // ------ @@ -360,6 +362,10 @@ pub const EventInit = struct { composed: bool = false, }; +pub fn eventDestroy(evt: *Event) void { + c._dom_event_destroy(evt); +} + pub fn eventInit(evt: *Event, typ: []const u8, opts: EventInit) !void { const s = try strFromData(typ); const err = c._dom_event_init(evt, s, opts.bubbles, opts.cancelable); @@ -444,6 +450,23 @@ pub fn eventPreventDefault(evt: *Event) !void { try DOMErr(err); } +pub fn eventGetInternalType(evt: *Event) !EventType { + var res: u32 = undefined; + const err = c._dom_event_get_internal_type(evt, &res); + try DOMErr(err); + return @enumFromInt(res); +} + +pub fn eventSetInternalType(evt: *Event, internal_type: EventType) !void { + const err = c._dom_event_set_internal_type(evt, @intFromEnum(internal_type)); + try DOMErr(err); +} + +pub const EventType = enum(u8) { + event = 0, + progress_event = 1, +}; + // EventHandler fn event_handler_cbk(data: *anyopaque) *Callback { const ptr: *align(@alignOf(*Callback)) anyopaque = @alignCast(data); @@ -454,7 +477,14 @@ const event_handler = struct { fn handle(event: ?*Event, data: ?*anyopaque) callconv(.C) void { if (data) |d| { const func = event_handler_cbk(d); - func.call(.{event}) catch unreachable; + + if (event) |evt| { + func.call(.{ + EventToInterface(evt) catch unreachable, + }) catch unreachable; + } else { + func.call(.{event}) catch unreachable; + } // NOTE: we can not call func.deinit here // b/c the handler can be called several times // either on this dispatch event or in anoter one @@ -671,7 +701,12 @@ pub const EventTargetTBase = extern struct { pub fn dispatch_event(et: [*c]c.dom_event_target, evt: ?*c.struct_dom_event, res: [*c]bool) callconv(.C) c.dom_exception { const self = @as(*Self, @ptrCast(et)); - return c._dom_event_target_dispatch(et, &self.eti, evt, c.DOM_BUBBLING_PHASE, res); + // Set the event target to the target dispatched. + const e = c._dom_event_set_target(evt, et); + if (e != c.DOM_NO_ERR) { + return e; + } + return c._dom_event_target_dispatch(et, &self.eti, evt, c.DOM_AT_TARGET, res); } pub fn remove_event_listener(et: [*c]c.dom_event_target, t: [*c]c.dom_string, l: ?*c.struct_dom_event_listener, capture: bool) callconv(.C) c.dom_exception { diff --git a/src/run_tests.zig b/src/run_tests.zig index aac7fff7..e69ace1e 100644 --- a/src/run_tests.zig +++ b/src/run_tests.zig @@ -7,6 +7,7 @@ const generate = @import("generate.zig"); const parser = @import("netsurf.zig"); const apiweb = @import("apiweb.zig"); const Window = @import("html/window.zig").Window; +const xhr = @import("xhr/xhr.zig"); const documentTestExecFn = @import("dom/document.zig").testExecFn; const HTMLDocumentTestExecFn = @import("html/document.zig").testExecFn; @@ -23,6 +24,8 @@ const NodeListTestExecFn = @import("dom/nodelist.zig").testExecFn; const AttrTestExecFn = @import("dom/attribute.zig").testExecFn; const EventTargetTestExecFn = @import("dom/event_target.zig").testExecFn; const EventTestExecFn = @import("events/event.zig").testExecFn; +const XHRTestExecFn = xhr.testExecFn; +const ProgressEventTestExecFn = @import("xhr/progress_event.zig").testExecFn; pub const Types = jsruntime.reflect(apiweb.Interfaces); @@ -78,6 +81,8 @@ fn testsAllExecFn( AttrTestExecFn, EventTargetTestExecFn, EventTestExecFn, + XHRTestExecFn, + ProgressEventTestExecFn, }; inline for (testFns) |testFn| { @@ -93,6 +98,11 @@ pub fn main() !void { } } +test { + const TestAsync = @import("async/test.zig"); + std.testing.refAllDecls(TestAsync); +} + test "jsruntime" { // generate tests try generate.tests(); @@ -157,3 +167,20 @@ test "DocumentHTML is a libdom event target" { const et = parser.toEventTarget(parser.DocumentHTML, doc); _ = try parser.eventTargetDispatchEvent(et, event); } + +test "XMLHttpRequest.validMethod" { + // valid methods + for ([_][]const u8{ "get", "GET", "head", "HEAD" }) |tc| { + _ = try xhr.XMLHttpRequest.validMethod(tc); + } + + // forbidden + for ([_][]const u8{ "connect", "CONNECT" }) |tc| { + try std.testing.expectError(parser.DOMError.Security, xhr.XMLHttpRequest.validMethod(tc)); + } + + // syntax + for ([_][]const u8{ "foo", "BAR" }) |tc| { + try std.testing.expectError(parser.DOMError.Syntax, xhr.XMLHttpRequest.validMethod(tc)); + } +} diff --git a/src/xhr/event_target.zig b/src/xhr/event_target.zig new file mode 100644 index 00000000..5830f57a --- /dev/null +++ b/src/xhr/event_target.zig @@ -0,0 +1,96 @@ +const std = @import("std"); + +const jsruntime = @import("jsruntime"); +const Callback = jsruntime.Callback; + +const EventTarget = @import("../dom/event_target.zig").EventTarget; + +const parser = @import("../netsurf.zig"); + +const log = std.log.scoped(.xhr); + +pub const XMLHttpRequestEventTarget = struct { + pub const prototype = *EventTarget; + pub const mem_guarantied = true; + + // Extend libdom event target for pure zig struct. + base: parser.EventTargetTBase = parser.EventTargetTBase{}, + + onloadstart_cbk: ?Callback = null, + onprogress_cbk: ?Callback = null, + onabort_cbk: ?Callback = null, + onload_cbk: ?Callback = null, + ontimeout_cbk: ?Callback = null, + onloadend_cbk: ?Callback = null, + + fn register(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, typ: []const u8, cbk: Callback) !void { + try parser.eventTargetAddEventListener(@as(*parser.EventTarget, @ptrCast(self)), alloc, typ, cbk, false); + } + fn unregister(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, typ: []const u8, cbk: Callback) !void { + const et = @as(*parser.EventTarget, @ptrCast(self)); + // check if event target has already this listener + const lst = try parser.eventTargetHasListener(et, typ, false, cbk.id()); + if (lst == null) { + return; + } + + // remove listener + try parser.eventTargetRemoveEventListener(et, alloc, typ, lst.?, false); + } + + pub fn get_onloadstart(self: *XMLHttpRequestEventTarget) ?Callback { + return self.onloadstart_cbk; + } + pub fn get_onprogress(self: *XMLHttpRequestEventTarget) ?Callback { + return self.onprogress_cbk; + } + pub fn get_onabort(self: *XMLHttpRequestEventTarget) ?Callback { + return self.onabort_cbk; + } + pub fn get_onload(self: *XMLHttpRequestEventTarget) ?Callback { + return self.onload_cbk; + } + pub fn get_ontimeout(self: *XMLHttpRequestEventTarget) ?Callback { + return self.ontimeout_cbk; + } + pub fn get_onloadend(self: *XMLHttpRequestEventTarget) ?Callback { + return self.onloadend_cbk; + } + + pub fn set_onloadstart(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, handler: Callback) !void { + if (self.onloadstart_cbk) |cbk| try self.unregister(alloc, "loadstart", cbk); + try self.register(alloc, "loadstart", handler); + self.onloadstart_cbk = handler; + } + pub fn set_onprogress(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, handler: Callback) !void { + if (self.onprogress_cbk) |cbk| try self.unregister(alloc, "progress", cbk); + try self.register(alloc, "progress", handler); + self.onprogress_cbk = handler; + } + pub fn set_onabort(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, handler: Callback) !void { + if (self.onabort_cbk) |cbk| try self.unregister(alloc, "abort", cbk); + try self.register(alloc, "abort", handler); + self.onabort_cbk = handler; + } + pub fn set_onload(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, handler: Callback) !void { + if (self.onload_cbk) |cbk| try self.unregister(alloc, "load", cbk); + try self.register(alloc, "load", handler); + self.onload_cbk = handler; + } + pub fn set_ontimeout(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, handler: Callback) !void { + if (self.ontimeout_cbk) |cbk| try self.unregister(alloc, "timeout", cbk); + try self.register(alloc, "timeout", handler); + self.ontimeout_cbk = handler; + } + pub fn set_onloadend(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator, handler: Callback) !void { + if (self.onloadend_cbk) |cbk| try self.unregister(alloc, "loadend", cbk); + try self.register(alloc, "loadend", handler); + self.onloadend_cbk = handler; + } + + pub fn deinit(self: *XMLHttpRequestEventTarget, alloc: std.mem.Allocator) void { + parser.eventTargetRemoveAllEventListeners(@as(*parser.EventTarget, @ptrCast(self)), alloc) catch |e| { + log.err("remove all listeners: {any}", .{e}); + }; + } +}; diff --git a/src/xhr/progress_event.zig b/src/xhr/progress_event.zig new file mode 100644 index 00000000..afdf36a9 --- /dev/null +++ b/src/xhr/progress_event.zig @@ -0,0 +1,72 @@ +const std = @import("std"); + +const jsruntime = @import("jsruntime"); +const Case = jsruntime.test_utils.Case; +const checkCases = jsruntime.test_utils.checkCases; + +const parser = @import("../netsurf.zig"); +const Event = @import("../events/event.zig").Event; + +const DOMException = @import("../dom/exceptions.zig").DOMException; + +pub const ProgressEvent = struct { + pub const prototype = *Event; + pub const Exception = DOMException; + pub const mem_guarantied = true; + + pub const EventInit = struct { + lengthComputable: bool = false, + loaded: u64 = 0, + total: u64 = 0, + }; + + proto: parser.Event, + lengthComputable: bool, + loaded: u64 = 0, + total: u64 = 0, + + pub fn constructor(eventType: []const u8, opts: ?EventInit) !ProgressEvent { + const event = try parser.eventCreate(); + defer parser.eventDestroy(event); + try parser.eventInit(event, eventType, .{}); + try parser.eventSetInternalType(event, .progress_event); + + const o = opts orelse EventInit{}; + + return .{ + .proto = event.*, + .lengthComputable = o.lengthComputable, + .loaded = o.loaded, + .total = o.total, + }; + } + + pub fn get_lengthComputable(self: ProgressEvent) bool { + return self.lengthComputable; + } + + pub fn get_loaded(self: ProgressEvent) u64 { + return self.loaded; + } + + pub fn get_total(self: ProgressEvent) u64 { + return self.total; + } +}; + +pub fn testExecFn( + _: std.mem.Allocator, + js_env: *jsruntime.Env, +) anyerror!void { + var progress_event = [_]Case{ + .{ .src = "let pevt = new ProgressEvent('foo');", .ex = "undefined" }, + .{ .src = "pevt.loaded", .ex = "0" }, + .{ .src = "pevt instanceof ProgressEvent", .ex = "true" }, + .{ .src = "var nnb = 0; var eevt = null; function ccbk(event) { nnb ++; eevt = event; }", .ex = "undefined" }, + .{ .src = "document.addEventListener('foo', ccbk)", .ex = "undefined" }, + .{ .src = "document.dispatchEvent(pevt)", .ex = "true" }, + .{ .src = "eevt.type", .ex = "foo" }, + .{ .src = "eevt instanceof ProgressEvent", .ex = "true" }, + }; + try checkCases(js_env, &progress_event); +} diff --git a/src/xhr/xhr.zig b/src/xhr/xhr.zig new file mode 100644 index 00000000..f3840d66 --- /dev/null +++ b/src/xhr/xhr.zig @@ -0,0 +1,833 @@ +const std = @import("std"); + +const jsruntime = @import("jsruntime"); +const Case = jsruntime.test_utils.Case; +const checkCases = jsruntime.test_utils.checkCases; +const generate = @import("../generate.zig"); + +const DOMError = @import("../netsurf.zig").DOMError; +const DOMException = @import("../dom/exceptions.zig").DOMException; + +const ProgressEvent = @import("progress_event.zig").ProgressEvent; +const XMLHttpRequestEventTarget = @import("event_target.zig").XMLHttpRequestEventTarget; + +const Mime = @import("../browser/mime.zig"); + +const Loop = jsruntime.Loop; +const YieldImpl = Loop.Yield(XMLHttpRequest); +const Client = @import("../async/Client.zig"); + +const parser = @import("../netsurf.zig"); + +const log = std.log.scoped(.xhr); + +// XHR interfaces +// https://xhr.spec.whatwg.org/#interface-xmlhttprequest +pub const Interfaces = generate.Tuple(.{ + XMLHttpRequestEventTarget, + XMLHttpRequestUpload, + XMLHttpRequest, +}); + +pub const XMLHttpRequestUpload = struct { + pub const prototype = *XMLHttpRequestEventTarget; + pub const mem_guarantied = true; + + proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{}, +}; + +pub const XMLHttpRequestBodyInitTag = enum { + Blob, + BufferSource, + FormData, + URLSearchParams, + String, +}; + +pub const XMLHttpRequestBodyInit = union(XMLHttpRequestBodyInitTag) { + Blob: []const u8, + BufferSource: []const u8, + FormData: []const u8, + URLSearchParams: []const u8, + String: []const u8, + + fn contentType(self: XMLHttpRequestBodyInit) ![]const u8 { + return switch (self) { + .Blob => error.NotImplemented, + .BufferSource => error.NotImplemented, + .FormData => "multipart/form-data; boundary=TODO", + .URLSearchParams => "application/x-www-form-urlencoded; charset=UTF-8", + .String => "text/plain; charset=UTF-8", + }; + } + + // Duplicate the body content. + // The caller owns the allocated string. + fn dupe(self: XMLHttpRequestBodyInit, alloc: std.mem.Allocator) ![]const u8 { + return switch (self) { + .Blob => error.NotImplemented, + .BufferSource => error.NotImplemented, + .FormData => error.NotImplemented, + .URLSearchParams => error.NotImplemented, + .String => |v| try alloc.dupe(u8, v), + }; + } +}; + +pub const XMLHttpRequest = struct { + pub const prototype = *XMLHttpRequestEventTarget; + pub const mem_guarantied = true; + + pub const UNSENT: u16 = 0; + pub const OPENED: u16 = 1; + pub const HEADERS_RECEIVED: u16 = 2; + pub const LOADING: u16 = 3; + pub const DONE: u16 = 4; + + // https://xhr.spec.whatwg.org/#response-type + const ResponseType = enum { + Empty, + Text, + ArrayBuffer, + Blob, + Document, + JSON, + }; + + // TODO use std.json.Value instead, but it causes comptime error. + // blocked by https://github.com/lightpanda-io/jsruntime-lib/issues/204 + // const JSONValue = std.json.Value; + const JSONValue = u8; + + const Response = union(ResponseType) { + Empty: void, + Text: []const u8, + ArrayBuffer: void, + Blob: void, + Document: *parser.Document, + JSON: JSONValue, + }; + + const ResponseObjTag = enum { + Document, + Failure, + JSON, + }; + const ResponseObj = union(ResponseObjTag) { + Document: *parser.Document, + Failure: bool, + JSON: std.json.Parsed(JSONValue), + + fn deinit(self: ResponseObj) void { + return switch (self) { + .Document => |d| { + const doc = @as(*parser.DocumentHTML, @ptrCast(d)); + parser.documentHTMLClose(doc) catch {}; + }, + .JSON => |p| p.deinit(), + .Failure => {}, + }; + } + }; + + const PrivState = enum { new, open, send, write, finish, wait, done }; + + proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{}, + alloc: std.mem.Allocator, + cli: Client, + impl: YieldImpl, + + priv_state: PrivState = .new, + req: ?Client.Request = null, + + method: std.http.Method, + state: u16, + url: ?[]const u8, + uri: std.Uri, + headers: std.http.Headers, + sync: bool = true, + err: ?anyerror = null, + + // TODO uncomment this field causes casting issue with + // XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but + // not sure. see + // https://lightpanda.slack.com/archives/C05TRU6RBM1/p1707819010681019 + // upload: ?XMLHttpRequestUpload = null, + + timeout: u32 = 0, + withCredentials: bool = false, + // TODO: response readonly attribute any response; + response_bytes: ?[]const u8 = null, + response_type: ResponseType = .Empty, + response_headers: std.http.Headers, + response_status: u10 = 0, + response_override_mime_type: ?[]const u8 = null, + response_mime: Mime = undefined, + response_obj: ?ResponseObj = null, + send_flag: bool = false, + + payload: ?[]const u8 = null, + + pub fn constructor(alloc: std.mem.Allocator, loop: *Loop) !XMLHttpRequest { + return .{ + .alloc = alloc, + .headers = .{ .allocator = alloc, .owned = true }, + .response_headers = .{ .allocator = alloc, .owned = true }, + .impl = YieldImpl.init(loop), + .method = undefined, + .url = null, + .uri = undefined, + .state = UNSENT, + // TODO retrieve the HTTP client globally to reuse existing connections. + .cli = .{ .allocator = alloc, .loop = loop }, + }; + } + + pub fn reset(self: *XMLHttpRequest, alloc: std.mem.Allocator) void { + if (self.url) |v| alloc.free(v); + self.url = null; + + if (self.payload) |v| alloc.free(v); + self.payload = null; + + if (self.response_bytes) |v| alloc.free(v); + if (self.response_obj) |v| v.deinit(); + + self.response_obj = null; + self.response_mime = Mime.Empty; + self.response_type = .Empty; + + // TODO should we clearRetainingCapacity instead? + self.headers.clearAndFree(); + self.response_headers.clearAndFree(); + self.response_status = 0; + + self.send_flag = false; + + self.priv_state = .new; + + if (self.req) |*r| { + r.deinit(); + self.req = null; + } + } + + pub fn deinit(self: *XMLHttpRequest, alloc: std.mem.Allocator) void { + self.reset(); + self.headers.deinit(); + self.response_headers.deinit(); + + self.proto.deinit(alloc); + + // TODO the client must be shared between requests. + self.cli.deinit(); + } + + pub fn get_readyState(self: *XMLHttpRequest) u16 { + return self.state; + } + + pub fn get_timeout(self: *XMLHttpRequest) u32 { + return self.timeout; + } + + pub fn set_timeout(self: *XMLHttpRequest, timeout: u32) !void { + // TODO If the current global object is a Window object and this’s + // synchronous flag is set, then throw an "InvalidAccessError" + // DOMException. + // https://xhr.spec.whatwg.org/#dom-xmlhttprequest-timeout + self.timeout = timeout; + } + + pub fn get_withCredentials(self: *XMLHttpRequest) bool { + return self.withCredentials; + } + + pub fn set_withCredentials(self: *XMLHttpRequest, withCredentials: bool) !void { + if (self.state != OPENED and self.state != UNSENT) return DOMError.InvalidState; + if (self.send_flag) return DOMError.InvalidState; + + self.withCredentials = withCredentials; + } + + pub fn _open( + self: *XMLHttpRequest, + alloc: std.mem.Allocator, + method: []const u8, + url: []const u8, + asyn: ?bool, + username: ?[]const u8, + password: ?[]const u8, + ) !void { + _ = username; + _ = password; + + // TODO If this’s relevant global object is a Window object and its + // associated Document is not fully active, then throw an + // "InvalidStateError" DOMException. + + self.method = try validMethod(method); + + self.reset(alloc); + + self.url = try alloc.dupe(u8, url); + self.uri = std.Uri.parse(self.url.?) catch return DOMError.Syntax; + self.sync = if (asyn) |b| !b else false; + + self.state = OPENED; + self.dispatchEvt("readystatechange"); + } + + // dispatch request event. + // errors are logged only. + fn dispatchEvt(self: *XMLHttpRequest, typ: []const u8) void { + const evt = parser.eventCreate() catch |e| { + return log.err("dispatch event create: {any}", .{e}); + }; + + // We can we defer event destroy once the event is dispatched. + defer parser.eventDestroy(evt); + + parser.eventInit(evt, typ, .{ .bubbles = true, .cancelable = true }) catch |e| { + return log.err("dispatch event init: {any}", .{e}); + }; + _ = parser.eventTargetDispatchEvent(@as(*parser.EventTarget, @ptrCast(self)), evt) catch |e| { + return log.err("dispatch event: {any}", .{e}); + }; + } + + fn dispatchProgressEvent( + self: *XMLHttpRequest, + typ: []const u8, + opts: ProgressEvent.EventInit, + ) void { + var evt = ProgressEvent.constructor(typ, .{ + // https://xhr.spec.whatwg.org/#firing-events-using-the-progressevent-interface + .lengthComputable = opts.total > 0, + .total = opts.total, + .loaded = opts.loaded, + }) catch |e| { + return log.err("construct progress event: {any}", .{e}); + }; + + _ = parser.eventTargetDispatchEvent( + @as(*parser.EventTarget, @ptrCast(self)), + @as(*parser.Event, @ptrCast(&evt)), + ) catch |e| { + return log.err("dispatch progress event: {any}", .{e}); + }; + } + + const methods = [_]struct { + tag: std.http.Method, + name: []const u8, + }{ + .{ .tag = .DELETE, .name = "DELETE" }, + .{ .tag = .GET, .name = "GET" }, + .{ .tag = .HEAD, .name = "HEAD" }, + .{ .tag = .OPTIONS, .name = "OPTIONS" }, + .{ .tag = .POST, .name = "POST" }, + .{ .tag = .PUT, .name = "PUT" }, + }; + const methods_forbidden = [_][]const u8{ "CONNECT", "TRACE", "TRACK" }; + + pub fn validMethod(m: []const u8) DOMError!std.http.Method { + for (methods) |method| { + if (std.ascii.eqlIgnoreCase(method.name, m)) { + return method.tag; + } + } + // If method is a forbidden method, then throw a "SecurityError" DOMException. + for (methods_forbidden) |method| { + if (std.ascii.eqlIgnoreCase(method, m)) { + return DOMError.Security; + } + } + + // If method is not a method, then throw a "SyntaxError" DOMException. + return DOMError.Syntax; + } + + pub fn _setRequestHeader(self: *XMLHttpRequest, name: []const u8, value: []const u8) !void { + if (self.state != OPENED) return DOMError.InvalidState; + if (self.send_flag) return DOMError.InvalidState; + return try self.headers.append(name, value); + } + + // TODO body can be either a XMLHttpRequestBodyInit or a document + pub fn _send(self: *XMLHttpRequest, alloc: std.mem.Allocator, body: ?[]const u8) !void { + if (self.state != OPENED) return DOMError.InvalidState; + if (self.send_flag) return DOMError.InvalidState; + + // The body argument provides the request body, if any, and is ignored + // if the request method is GET or HEAD. + // https://xhr.spec.whatwg.org/#the-send()-method + // var used_body: ?XMLHttpRequestBodyInit = null; + if (body != null and self.method != .GET and self.method != .HEAD) { + // TODO If body is a Document, then set this’s request body to body, serialized, converted, and UTF-8 encoded. + + const body_init = XMLHttpRequestBodyInit{ .String = body.? }; + + // keep the user content type from request headers. + if (self.headers.getFirstEntry("Content-Type") == null) { + // https://fetch.spec.whatwg.org/#bodyinit-safely-extract + try self.headers.append("Content-Type", try body_init.contentType()); + } + + // copy the payload + if (self.payload) |v| alloc.free(v); + self.payload = try body_init.dupe(alloc); + } + + log.debug("{any} {any}", .{ self.method, self.uri }); + + self.send_flag = true; + self.impl.yield(self); + } + + // onYield is a callback called between each request's steps. + // Between each step, the code is blocking. + // Yielding allows pseudo-async and gives a chance to other async process + // to be called. + pub fn onYield(self: *XMLHttpRequest, err: ?anyerror) void { + if (err) |e| return self.onErr(e); + + switch (self.priv_state) { + .new => { + self.priv_state = .open; + self.req = self.cli.open(self.method, self.uri, self.headers, .{}) catch |e| return self.onErr(e); + }, + .open => { + // prepare payload transfert. + if (self.payload) |v| self.req.?.transfer_encoding = .{ .content_length = v.len }; + + self.priv_state = .send; + self.req.?.send(.{}) catch |e| return self.onErr(e); + }, + .send => { + if (self.payload) |payload| { + self.priv_state = .write; + self.req.?.writeAll(payload) catch |e| return self.onErr(e); + } else { + self.priv_state = .finish; + self.req.?.finish() catch |e| return self.onErr(e); + } + }, + .write => { + self.priv_state = .finish; + self.req.?.finish() catch |e| return self.onErr(e); + }, + .finish => { + self.priv_state = .wait; + self.req.?.wait() catch |e| return self.onErr(e); + }, + .wait => { + log.info("{any} {any} {d}", .{ self.method, self.uri, self.req.?.response.status }); + + self.priv_state = .done; + self.response_headers = self.req.?.response.headers.clone(self.response_headers.allocator) catch |e| return self.onErr(e); + + // extract a mime type from headers. + const ct = self.response_headers.getFirstValue("Content-Type") orelse "text/xml"; + self.response_mime = Mime.parse(ct) catch |e| return self.onErr(e); + + // TODO handle override mime type + + self.state = HEADERS_RECEIVED; + self.dispatchEvt("readystatechange"); + + self.response_status = @intFromEnum(self.req.?.response.status); + + var buf: std.ArrayListUnmanaged(u8) = .{}; + + // TODO set correct length + const total = 0; + var loaded: u64 = 0; + + // dispatch a progress event loadstart. + self.dispatchProgressEvent("loadstart", .{ .loaded = loaded, .total = total }); + + const reader = self.req.?.reader(); + var buffer: [1024]u8 = undefined; + var ln = buffer.len; + while (ln > 0) { + ln = reader.read(&buffer) catch |e| { + buf.deinit(self.alloc); + return self.onErr(e); + }; + buf.appendSlice(self.alloc, buffer[0..ln]) catch |e| { + buf.deinit(self.alloc); + return self.onErr(e); + }; + loaded = loaded + ln; + + // TODO dispatch only if 50ms have passed. + + self.state = LOADING; + self.dispatchEvt("readystatechange"); + + // dispatch a progress event progress. + self.dispatchProgressEvent("progress", .{ + .loaded = loaded, + .total = total, + }); + } + self.response_bytes = buf.items; + self.send_flag = false; + + self.state = DONE; + self.dispatchEvt("readystatechange"); + + // dispatch a progress event load. + self.dispatchProgressEvent("load", .{ .loaded = loaded, .total = total }); + // dispatch a progress event loadend. + self.dispatchProgressEvent("loadend", .{ .loaded = loaded, .total = total }); + }, + .done => { + if (self.req) |*r| { + r.deinit(); + self.req = null; + } + + // finalize fetch process. + return; + }, + } + + self.impl.yield(self); + } + + fn onErr(self: *XMLHttpRequest, err: anyerror) void { + self.priv_state = .done; + if (self.req) |*r| { + r.deinit(); + self.req = null; + } + + self.err = err; + self.state = DONE; + self.send_flag = false; + self.dispatchEvt("readystatechange"); + self.dispatchProgressEvent("error", .{}); + self.dispatchProgressEvent("loadend", .{}); + + log.debug("{any} {any} {any}", .{ self.method, self.uri, self.err }); + } + + pub fn _abort(self: *XMLHttpRequest) void { + self.onErr(DOMError.Abort); + } + + pub fn get_responseType(self: *XMLHttpRequest) []const u8 { + return switch (self.response_type) { + .Empty => "", + .ArrayBuffer => "arraybuffer", + .Blob => "blob", + .Document => "document", + .JSON => "json", + .Text => "text", + }; + } + + pub fn set_responseType(self: *XMLHttpRequest, rtype: []const u8) !void { + if (self.state == LOADING or self.state == DONE) return DOMError.InvalidState; + + if (std.mem.eql(u8, rtype, "")) { + self.response_type = .Empty; + return; + } + if (std.mem.eql(u8, rtype, "arraybuffer")) { + self.response_type = .ArrayBuffer; + return; + } + if (std.mem.eql(u8, rtype, "blob")) { + self.response_type = .Blob; + return; + } + if (std.mem.eql(u8, rtype, "document")) { + self.response_type = .Document; + return; + } + if (std.mem.eql(u8, rtype, "json")) { + self.response_type = .JSON; + return; + } + if (std.mem.eql(u8, rtype, "text")) { + self.response_type = .Text; + return; + } + } + + // TODO retrieve the redirected url + pub fn get_responseURL(self: *XMLHttpRequest) ?[]const u8 { + return self.url; + } + + pub fn get_responseXML(self: *XMLHttpRequest, alloc: std.mem.Allocator) !?Response { + if (self.response_type != .Empty and self.response_type != .Document) { + return DOMError.InvalidState; + } + + if (self.state != DONE) return null; + + // fastpath if response is previously parsed. + if (self.response_obj) |obj| { + return switch (obj) { + .Failure => null, + .Document => |v| .{ .Document = v }, + .JSON => null, + }; + } + + self.setResponseObjDocument(alloc); + + if (self.response_obj) |obj| { + return switch (obj) { + .Failure => null, + .Document => |v| .{ .Document = v }, + .JSON => null, + }; + } + return null; + } + + // https://xhr.spec.whatwg.org/#the-response-attribute + pub fn get_response(self: *XMLHttpRequest, alloc: std.mem.Allocator) !?Response { + if (self.response_type == .Empty or self.response_type == .Text) { + if (self.state == LOADING or self.state == DONE) return .{ .Text = "" }; + return .{ .Text = try self.get_responseText() }; + } + + // fastpath if response is previously parsed. + if (self.response_obj) |obj| { + return switch (obj) { + .Failure => null, + .Document => |v| .{ .Document = v }, + .JSON => |v| .{ .JSON = v.value }, + }; + } + + if (self.response_type == .ArrayBuffer) { + // TODO If this’s response type is "arraybuffer", then set this’s + // response object to a new ArrayBuffer object representing this’s + // received bytes. If this throws an exception, then set this’s + // response object to failure and return null. + return null; + } + + if (self.response_type == .Blob) { + // TODO Otherwise, if this’s response type is "blob", set this’s + // response object to a new Blob object representing this’s + // received bytes with type set to the result of get a final MIME + // type for this. + return null; + } + + // Otherwise, if this’s response type is "document", set a + // document response for this. + if (self.response_type == .Document) { + self.setResponseObjDocument(alloc); + } + + if (self.response_type == .JSON) { + if (self.response_bytes == null) return null; + + // TODO Let jsonObject be the result of running parse JSON from bytes + // on this’s received bytes. If that threw an exception, then return + // null. + self.setResponseObjJSON(alloc); + } + + if (self.response_obj) |obj| { + return switch (obj) { + .Failure => null, + .Document => |v| .{ .Document = v }, + .JSON => |v| .{ .JSON = v.value }, + }; + } + + return null; + } + + // setResponseObjDocument parses the received bytes as HTML document and + // stores the result into response_obj. + // If the par sing fails, a Failure is stored in response_obj. + // TODO parse XML. + // https://xhr.spec.whatwg.org/#response-object + fn setResponseObjDocument(self: *XMLHttpRequest, alloc: std.mem.Allocator) void { + const isHTML = self.response_mime.eql(Mime.HTML); + + // TODO If finalMIME is not an HTML MIME type or an XML MIME type, then + // return. + if (!isHTML) return; + + const ccharset = alloc.dupeZ(u8, self.response_mime.charset orelse "utf-8") catch { + self.response_obj = .{ .Failure = true }; + return; + }; + defer alloc.free(ccharset); + + var fbs = std.io.fixedBufferStream(self.response_bytes.?); + const doc = parser.documentHTMLParse(fbs.reader(), ccharset) catch { + self.response_obj = .{ .Failure = true }; + return; + }; + + // TODO Set document’s URL to xhr’s response’s URL. + // TODO Set document’s origin to xhr’s relevant settings object’s origin. + + self.response_obj = .{ + .Document = parser.documentHTMLToDocument(doc), + }; + } + + // setResponseObjJSON parses the received bytes as a std.json.Value. + fn setResponseObjJSON(self: *XMLHttpRequest, alloc: std.mem.Allocator) void { + // TODO should we use parseFromSliceLeaky if we expect the allocator is + // already an arena? + const p = std.json.parseFromSlice( + JSONValue, + alloc, + self.response_bytes.?, + .{}, + ) catch |e| { + log.err("parse JSON: {}", .{e}); + self.response_obj = .{ .Failure = true }; + return; + }; + + self.response_obj = .{ .JSON = p }; + } + + pub fn get_responseText(self: *XMLHttpRequest) ![]const u8 { + if (self.response_type != .Empty and self.response_type != .Text) return DOMError.InvalidState; + + return if (self.response_bytes) |v| v else ""; + } + + pub fn _getResponseHeader(self: *XMLHttpRequest, name: []const u8) ?[]const u8 { + return self.response_headers.getFirstValue(name); + } + + // The caller owns the string returned. + // TODO change the return type to express the string ownership and let + // jsruntime free the string once copied to v8. + // see https://github.com/lightpanda-io/jsruntime-lib/issues/195 + pub fn _getAllResponseHeaders(self: *XMLHttpRequest, alloc: std.mem.Allocator) ![]const u8 { + if (self.response_headers.list.items.len == 0) return ""; + self.response_headers.sort(); + + var buf: std.ArrayListUnmanaged(u8) = .{}; + errdefer buf.deinit(alloc); + + const w = buf.writer(alloc); + + for (self.response_headers.list.items) |entry| { + if (entry.value.len == 0) continue; + + try w.writeAll(entry.name); + try w.writeAll(": "); + try w.writeAll(entry.value); + try w.writeAll("\r\n"); + } + + return buf.items; + } + + pub fn get_status(self: *XMLHttpRequest) u16 { + return self.response_status; + } + + pub fn get_statusText(self: *XMLHttpRequest) []const u8 { + if (self.response_status == 0) return ""; + + return std.http.Status.phrase(@enumFromInt(self.response_status)) orelse ""; + } +}; + +pub fn testExecFn( + _: std.mem.Allocator, + js_env: *jsruntime.Env, +) anyerror!void { + var send = [_]Case{ + .{ .src = "var nb = 0; var evt = null; function cbk(event) { nb ++; evt = event; }", .ex = "undefined" }, + .{ .src = "const req = new XMLHttpRequest()", .ex = "undefined" }, + + .{ .src = "req.onload = cbk", .ex = "function cbk(event) { nb ++; evt = event; }" }, + // Getter returning a callback crashes. + // blocked by https://github.com/lightpanda-io/jsruntime-lib/issues/200 + // .{ .src = "req.onload", .ex = "function cbk(event) { nb ++; evt = event; }" }, + //.{ .src = "req.onload = cbk", .ex = "function cbk(event) { nb ++; evt = event; }" }, + + .{ .src = "req.open('GET', 'http://httpbin.io/html')", .ex = "undefined" }, + .{ .src = "req.setRequestHeader('User-Agent', 'lightpanda/1.0')", .ex = "undefined" }, + + // ensure open resets values + .{ .src = "req.status", .ex = "0" }, + .{ .src = "req.statusText", .ex = "" }, + .{ .src = "req.getAllResponseHeaders()", .ex = "" }, + .{ .src = "req.getResponseHeader('Content-Type')", .ex = "null" }, + .{ .src = "req.responseText", .ex = "" }, + + .{ .src = "req.send(); nb", .ex = "0" }, + + // Each case executed waits for all loop callaback calls. + // So the url has been retrieved. + .{ .src = "nb", .ex = "1" }, + .{ .src = "evt.type", .ex = "load" }, + .{ .src = "evt.loaded > 0", .ex = "true" }, + .{ .src = "evt instanceof ProgressEvent", .ex = "true" }, + .{ .src = "req.status", .ex = "200" }, + .{ .src = "req.statusText", .ex = "OK" }, + .{ .src = "req.getResponseHeader('Content-Type')", .ex = "text/html; charset=utf-8" }, + .{ .src = "req.getAllResponseHeaders().length > 64", .ex = "true" }, + .{ .src = "req.responseText.length > 64", .ex = "true" }, + .{ .src = "req.response", .ex = "" }, + .{ .src = "req.responseXML instanceof Document", .ex = "true" }, + }; + try checkCases(js_env, &send); + + var document = [_]Case{ + .{ .src = "const req2 = new XMLHttpRequest()", .ex = "undefined" }, + .{ .src = "req2.open('GET', 'http://httpbin.io/html')", .ex = "undefined" }, + .{ .src = "req2.responseType = 'document'", .ex = "document" }, + + .{ .src = "req2.send()", .ex = "undefined" }, + + // Each case executed waits for all loop callaback calls. + // So the url has been retrieved. + .{ .src = "req2.status", .ex = "200" }, + .{ .src = "req2.statusText", .ex = "OK" }, + .{ .src = "req2.response instanceof Document", .ex = "true" }, + .{ .src = "req2.responseXML instanceof Document", .ex = "true" }, + }; + try checkCases(js_env, &document); + + // var json = [_]Case{ + // .{ .src = "const req3 = new XMLHttpRequest()", .ex = "undefined" }, + // .{ .src = "req3.open('GET', 'http://httpbin.io/json')", .ex = "undefined" }, + // .{ .src = "req3.responseType = 'json'", .ex = "json" }, + + // .{ .src = "req3.send()", .ex = "undefined" }, + + // // Each case executed waits for all loop callaback calls. + // // So the url has been retrieved. + // .{ .src = "req3.status", .ex = "200" }, + // .{ .src = "req3.statusText", .ex = "OK" }, + // .{ .src = "req3.response", .ex = "" }, + // }; + // try checkCases(js_env, &json); + // + var post = [_]Case{ + .{ .src = "const req3 = new XMLHttpRequest()", .ex = "undefined" }, + .{ .src = "req3.open('POST', 'http://httpbin.io/post')", .ex = "undefined" }, + .{ .src = "req3.send('foo')", .ex = "undefined" }, + + // Each case executed waits for all loop callaback calls. + // So the url has been retrieved. + .{ .src = "req3.status", .ex = "200" }, + .{ .src = "req3.statusText", .ex = "OK" }, + .{ .src = "req3.responseText.length > 64", .ex = "true" }, + }; + try checkCases(js_env, &post); +} diff --git a/vendor/jsruntime-lib b/vendor/jsruntime-lib index 8fe165bc..2d7b816f 160000 --- a/vendor/jsruntime-lib +++ b/vendor/jsruntime-lib @@ -1 +1 @@ -Subproject commit 8fe165bc49ddd235701ecb1903f2cff3928e2535 +Subproject commit 2d7b816f48da724036e41a956c996e775f9b226a