public/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2025-10-29 15:13:28 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Switch XHR to new http client

Browse Source 
get puppeteer/cdp.js working again

make test are all passing
This commit is contained in:
Karl Seguin
2025-07-30 17:13:23 +08:00
parent b0fe5d60ab
commit 54ab1326e5
12 changed files with 330 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/browser/ScriptManager.zig
View File

@@ -38,6 +38,10 @@ page: *Page,
// Only once this is true can deferred scripts be run
static_scripts_done: bool,
// when async_count == 0 and static_script_done == true, the document is completed
// loading (i.e. page.documentIsComplete should be called).
async_count: usize,
// Normal scripts (non-deffered & non-async). These must be executed ni order
scripts: OrderList,
@@ -58,6 +62,7 @@ pub fn init(app: *App, page: *Page) ScriptManager {
.page = page,
.scripts = .{},
.deferred = .{},
.async_count = 0,
.allocator = allocator,
.client = app.http_client,
.static_scripts_done = false,
@@ -183,7 +188,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void {
.ctx = pending_script,
.method = .GET,
.start_callback = startCallback,
.header_callback = headerCallback,
.header_done_callback = headerCallback,
.data_callback = dataCallback,
.done_callback = doneCallback,
.error_callback = errorCallback,
@@ -229,7 +234,31 @@ fn evaluate(self: *ScriptManager) void {
pending_script.script.eval(page);
}
// When all scripts (normal and deferred) are done loading, the document
// state changes (this ultimately triggers the DOMContentLoaded event)
page.documentIsLoaded();
if (self.async_count == 0) {
// if we're here, then its like `asyncDone`
// 1 - there are no async scripts pending
// 2 - we checkecked static_scripts_done == true above
// 3 - we drained self.scripts above
// 4 - we drained self.deferred above
page.documentIsComplete();
}
}
fn asyncDone(self: *ScriptManager) void {
self.async_count -= 1;
if (
self.async_count == 0 and // there are no more async scripts
self.static_scripts_done and // and we've finished parsing the HTML to queue all <scripts>
self.scripts.first == null and // and there are no more <script src=> to wait for
self.deferred.first == null // and there are no more <script defer src=> to wait for
) {
// then the document is considered complete
self.page.documentIsComplete();
}
}
fn getList(self: *ScriptManager, script: *const Script) ?*OrderList {
@@ -334,10 +363,13 @@ const PendingScript = struct {
fn doneCallback(self: *PendingScript, transfer: *http.Transfer) void {
log.debug(.http, "script fetch complete", .{ .req = transfer });
const manager = self.manager;
if (self.script.is_async) {
// async script can be evaluated immediately
defer self.deinit();
self.script.eval(self.manager.page);
manager.asyncDone();
} else {
self.complete = true;
self.manager.evaluate();
@@ -348,6 +380,7 @@ const PendingScript = struct {
log.warn(.http, "script fetch error", .{ .req = transfer, .err = err });
self.deinit();
}
};
const Script = struct {

20
src/browser/mime.zig
View File

@@ -96,12 +96,23 @@ pub const Mime = struct {
break;
}
var attribute_value = value;
if (value[0] == '"' and value[value.len - 1] == '"') {
if (value[0] == '"') {
if (value.len < 3 or value[value.len - 1] != '"') {
return error.Invalid;
}
attribute_value = value[1 .. value.len - 1];
}
if (std.ascii.eqlIgnoreCase(attribute_value, "utf-8")) {
charset = "UTF-8";
} else {
// we only care about null (which we default to UTF-8)
// or UTF-8. If this is actually set (i.e. not null)
// and isn't UTF-8, we'll just put a dummy value. If
// we want to capture the actual value, we'll need to
// dupe/allocate it. Since, for now, we don't need that
// we can avoid the allocation.
charset = "lightpanda:UNSUPPORTED";
}
},
}
@@ -289,7 +300,6 @@ test "Mime: invalid " {
"text/html; charset=\"\"",
"text/html; charset=\"",
"text/html; charset=\"\\",
"text/html; charset=\"\\a\"", // invalid to escape non special characters
};
for (invalids) |invalid| {
@@ -351,19 +361,19 @@ test "Mime: parse charset" {
try expect(.{
.content_type = .{ .text_xml = {} },
.charset = "utf-8",
.charset = "UTF-8",
.params = "charset=utf-8",
}, "text/xml; charset=utf-8");
try expect(.{
.content_type = .{ .text_xml = {} },
.charset = "utf-8",
.charset = "UTF-8",
.params = "charset=\"utf-8\"",
}, "text/xml;charset=\"utf-8\"");
try expect(.{
.content_type = .{ .text_xml = {} },
.charset = "\\ \" ",
.charset = "lightpanda:UNSUPPORTED",
.params = "charset=\"\\\\ \\\" \"",
}, "text/xml;charset=\"\\\\ \\\" \" ");
}

40
src/browser/page.zig
View File

@@ -131,6 +131,7 @@ pub const Page = struct {
.messageloop_node = .{ .func = messageLoopCallback },
.keydown_event_node = .{ .func = keydownCallback },
.window_clicked_event_node = .{ .func = windowClicked },
// @newhttp
// .request_factory = browser.http_client.requestFactory(.{
// .notification = browser.notification,
// }),
@@ -233,32 +234,32 @@ pub const Page = struct {
pub fn wait(self: *Page, wait_sec: usize) !void {
switch (self.mode) {
.pre, .html, .raw, .parsed => {
// The HTML page was parsed. We're now either have JS scripts to
// The HTML page was parsed. We now either have JS scripts to
// download, or timeouts to execute, or both.
const cutoff = timestamp() + wait_sec;
var loop = self.session.browser.app.loop;
var try_catch: Env.TryCatch = undefined;
try_catch.init(self.main_context);
defer try_catch.deinit();
var http_client = self.http_client;
var loop = self.session.browser.app.loop;
// @newhttp Not sure about the timing / the order / any of this.
// I think I want to remove the loop. Implement our own timeouts
// and switch the CDP server to blocking. For now, just try this.`
while (timestamp() < cutoff) {
const active = try self.http_client.tick(10); // 10ms
if (active == 0) {
if (!self.loaded) {
// We have no pending HTTP requests and we haven't
// triggered the load event. Trigger the load event.
self.documentIsComplete();
} else if (loop.hasPendingTimeout() == false) {
const has_pending_timeouts = loop.hasPendingTimeout();
if (http_client.active > 0) {
try http_client.tick(10); // 10ms
} else if (self.loaded and self.loaded and !has_pending_timeouts) {
// we have no active HTTP requests, and no timeouts pending
return;
}
if (!has_pending_timeouts) {
continue;
}
// 10ms
@@ -308,11 +309,17 @@ pub const Page = struct {
.ctx = self,
.url = owned_url,
.method = opts.method,
.header_callback = pageHeaderCallback,
.header_done_callback = pageHeaderCallback,
.data_callback = pageDataCallback,
.done_callback = pageDoneCallback,
.error_callback = pageErrorCallback,
});
self.session.browser.notification.dispatch(.page_navigate, &.{
.opts = opts,
.url = owned_url,
.timestamp = timestamp(),
});
}
pub fn documentIsLoaded(self: *Page) void {
@@ -321,11 +328,18 @@ pub const Page = struct {
};
}
fn documentIsComplete(self: *Page) void {
pub fn documentIsComplete(self: *Page) void {
std.debug.assert(self.loaded == false);
self.loaded = true;
self._documentIsComplete() catch |err| {
log.err(.browser, "document is complete", .{ .err = err });
};
self.session.browser.notification.dispatch(.page_navigated, &.{
.url = self.url.raw,
.timestamp = timestamp(),
});
}
fn _documentIsComplete(self: *Page) !void {
try HTMLDocument.documentIsComplete(self.window.document, self);

358
src/browser/xhr/xhr.zig
View File

@@ -31,7 +31,6 @@ const Mime = @import("../mime.zig").Mime;
const parser = @import("../netsurf.zig");
const Page = @import("../page.zig").Page;
const http = @import("../../http/client.zig");
const Loop = @import("../../runtime/loop.zig").Loop;
const CookieJar = @import("../storage/storage.zig").CookieJar;
// XHR interfaces
@@ -80,54 +79,29 @@ const XMLHttpRequestBodyInit = union(enum) {
pub const XMLHttpRequest = struct {
proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
loop: *Loop,
arena: Allocator,
request: ?*http.Request = null,
transfer: ?*http.Transfer = null,
cookie_jar: *CookieJar,
err: ?anyerror = null,
last_dispatch: i64 = 0,
send_flag: bool = false,
method: http.Method,
state: State,
url: ?URL = null,
origin_url: *const URL,
url: ?[:0]const u8 = null,
// request headers
headers: Headers,
sync: bool = true,
err: ?anyerror = null,
last_dispatch: i64 = 0,
withCredentials: bool = false,
headers: std.ArrayListUnmanaged([:0]const u8),
request_body: ?[]const u8 = null,
cookie_jar: *CookieJar,
// the URI of the page where this request is originating from
// TODO uncomment this field causes casting issue with
// XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but
// not sure. see
// https://lightpanda.slack.com/archives/C05TRU6RBM1/p1707819010681019
// upload: ?XMLHttpRequestUpload = null,
// TODO uncomment this field causes casting issue with
// XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but
// not sure. see
// https://lightpanda.slack.com/archives/C05TRU6RBM1/p1707819010681019
// timeout: u32 = 0,
withCredentials: bool = false,
// TODO: response readonly attribute any response;
response_status: u16 = 0,
response_bytes: std.ArrayListUnmanaged(u8) = .{},
response_type: ResponseType = .Empty,
response_headers: Headers,
response_status: u16 = 0,
// TODO uncomment this field causes casting issue with
// XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but
// not sure. see
// https://lightpanda.slack.com/archives/C05TRU6RBM1/p1707819010681019
// response_override_mime_type: ?[]const u8 = null,
response_headers: std.ArrayListUnmanaged([]const u8) = .{},
response_mime: ?Mime = null,
response_obj: ?ResponseObj = null,
send_flag: bool = false,
pub const prototype = *XMLHttpRequestEventTarget;
@@ -158,68 +132,6 @@ pub const XMLHttpRequest = struct {
const JSONValue = std.json.Value;
const Headers = struct {
list: List,
arena: Allocator,
const List = std.ArrayListUnmanaged(std.http.Header);
fn init(arena: Allocator) Headers {
return .{
.arena = arena,
.list = .{},
};
}
fn append(self: *Headers, k: []const u8, v: []const u8) !void {
// duplicate strings
const kk = try self.arena.dupe(u8, k);
const vv = try self.arena.dupe(u8, v);
try self.list.append(self.arena, .{ .name = kk, .value = vv });
}
fn reset(self: *Headers) void {
self.list.clearRetainingCapacity();
}
fn has(self: Headers, k: []const u8) bool {
for (self.list.items) |h| {
if (std.ascii.eqlIgnoreCase(k, h.name)) {
return true;
}
}
return false;
}
fn getFirstValue(self: Headers, k: []const u8) ?[]const u8 {
for (self.list.items) |h| {
if (std.ascii.eqlIgnoreCase(k, h.name)) {
return h.value;
}
}
return null;
}
// replace any existing header with the same key
fn set(self: *Headers, k: []const u8, v: []const u8) !void {
for (self.list.items, 0..) |h, i| {
if (std.ascii.eqlIgnoreCase(k, h.name)) {
_ = self.list.swapRemove(i);
}
}
self.append(k, v);
}
// TODO
fn sort(_: *Headers) void {}
fn all(self: Headers) []std.http.Header {
return self.list.items;
}
};
const Response = union(ResponseType) {
Empty: void,
Text: []const u8,
@@ -254,22 +166,18 @@ pub const XMLHttpRequest = struct {
return .{
.url = null,
.arena = arena,
.loop = page.loop,
.headers = Headers.init(arena),
.response_headers = Headers.init(arena),
.headers = .{},
.method = undefined,
.state = .unsent,
.origin_url = &page.url,
.cookie_jar = page.cookie_jar,
};
}
pub fn destructor(self: *XMLHttpRequest) void {
// @newhttp
// if (self.request) |req| {
// req.abort();
self.request = null;
// }
if (self.transfer) |transfer| {
transfer.abort();
self.transfer = null;
}
}
pub fn reset(self: *XMLHttpRequest) void {
@@ -283,9 +191,8 @@ pub const XMLHttpRequest = struct {
self.response_type = .Empty;
self.response_mime = null;
// TODO should we clearRetainingCapacity instead?
self.headers.reset();
self.response_headers.reset();
self.headers.clearRetainingCapacity();
self.response_headers.clearRetainingCapacity();
self.response_status = 0;
self.send_flag = false;
@@ -325,6 +232,7 @@ pub const XMLHttpRequest = struct {
asyn: ?bool,
username: ?[]const u8,
password: ?[]const u8,
page: *Page,
) !void {
_ = username;
_ = password;
@@ -335,9 +243,7 @@ pub const XMLHttpRequest = struct {
self.reset();
self.method = try validMethod(method);
const arena = self.arena;
self.url = try self.origin_url.resolve(arena, url);
self.url = try URL.stitch(page.arena, url, page.url.raw, .{ .null_terminated = true });
self.sync = if (asyn) |b| !b else false;
self.state = .opened;
@@ -438,9 +344,18 @@ pub const XMLHttpRequest = struct {
}
pub fn _setRequestHeader(self: *XMLHttpRequest, name: []const u8, value: []const u8) !void {
if (self.state != .opened) return DOMError.InvalidState;
if (self.send_flag) return DOMError.InvalidState;
return try self.headers.append(name, value);
if (self.state != .opened) {
return DOMError.InvalidState;
}
if (self.send_flag) {
return DOMError.InvalidState;
}
return self.headers.append(
self.arena,
try std.fmt.allocPrintZ(self.arena, "{s}: {s}", .{name, value}),
);
}
// TODO body can be either a XMLHttpRequestBodyInit or a document
@@ -455,76 +370,67 @@ pub const XMLHttpRequest = struct {
self.request_body = try self.arena.dupe(u8, b);
}
// @newhttp
_ = page;
// try page.request_factory.initAsync(
// page.arena,
// self.method,
// &self.url.?.uri,
// self,
// onHttpRequestReady,
// );
}
fn onHttpRequestReady(ctx: *anyopaque, request: *http.Request) !void {
// on error, our caller will cleanup request
const self: *XMLHttpRequest = @alignCast(@ptrCast(ctx));
for (self.headers.list.items) |hdr| {
try request.addHeader(hdr.name, hdr.value, .{});
}
{
var arr: std.ArrayListUnmanaged(u8) = .{};
try self.cookie_jar.forRequest(&self.url.?.uri, arr.writer(self.arena), .{
.navigation = false,
.origin_uri = &self.origin_url.uri,
.is_http = true,
try page.http_client.request(.{
.ctx = self,
.url = self.url.?,
.method = self.method,
.start_callback = httpStartCallback,
.header_callback = httpHeaderCallback,
.header_done_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,
.done_callback = httpDoneCallback,
.error_callback = httpErrorCallback,
});
if (arr.items.len > 0) {
try request.addHeader("Cookie", arr.items, .{});
}
}
// The body argument provides the request body, if any, and is ignored
// if the request method is GET or HEAD.
// https://xhr.spec.whatwg.org/#the-send()-method
// var used_body: ?XMLHttpRequestBodyInit = null;
fn httpStartCallback(transfer: *http.Transfer) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
for (self.headers.items) |hdr| {
try transfer.addHeader(hdr);
}
// @newhttp
// {
// var arr: std.ArrayListUnmanaged(u8) = .{};
// try self.cookie_jar.forRequest(&self.url.?.uri, arr.writer(self.arena), .{
// .navigation = false,
// .origin_uri = &self.origin_url.uri,
// .is_http = true,
// });
// if (arr.items.len > 0) {
// try request.addHeader("Cookie", arr.items, .{});
// }
// }
if (self.request_body) |b| {
if (self.method != .GET and self.method != .HEAD) {
request.body = b;
try request.addHeader("Content-Type", "text/plain; charset=UTF-8", .{});
try transfer.setBody(b);
try transfer.addHeader("Content-Type: text/plain; charset=UTF-8");
}
}
try request.sendAsync(self, .{});
self.request = request;
self.transfer = transfer;
}
pub fn onHttpResponse(self: *XMLHttpRequest, progress_: anyerror!http.Progress) !void {
const progress = progress_ catch |err| {
// The request has been closed internally by the client, it isn't safe
// for us to keep it around.
self.request = null;
self.onErr(err);
return err;
};
fn httpHeaderCallback(transfer: *http.Transfer, header: []const u8) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
try self.response_headers.append(self.arena, try self.arena.dupe(u8, header));
}
fn httpHeaderDoneCallback(transfer: *http.Transfer) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
const header = &transfer.response_header.?;
if (progress.first) {
const header = progress.header;
log.debug(.http, "request header", .{
.source = "xhr",
.url = self.url,
.status = header.status,
});
for (header.headers.items) |hdr| {
try self.response_headers.append(hdr.name, hdr.value);
}
// extract a mime type from headers.
if (header.get("content-type")) |ct| {
self.response_mime = Mime.parse(self.arena, ct) catch |e| {
if (header.contentType()) |ct| {
self.response_mime = Mime.parse(ct) catch |e| {
return self.onErr(e);
};
}
@@ -541,27 +447,30 @@ pub const XMLHttpRequest = struct {
self.state = .loading;
self.dispatchEvt("readystatechange");
try self.cookie_jar.populateFromResponse(self.request.?.request_uri, &header);
// @newhttp
// try self.cookie_jar.populateFromResponse(self.request.?.request_uri, &header);
}
if (progress.data) |data| {
fn httpDataCallback(transfer: *http.Transfer, data: []const u8) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
try self.response_bytes.appendSlice(self.arena, data);
const now = std.time.milliTimestamp();
if (now - self.last_dispatch < 50) {
// don't send this more than once every 50ms
return;
}
const loaded = self.response_bytes.items.len;
const now = std.time.milliTimestamp();
if (now - self.last_dispatch > 50) {
// don't send this more than once every 50ms
self.dispatchProgressEvent("progress", .{
.total = loaded,
.total = loaded, // TODO, this is wrong? Need the content-type
.loaded = loaded,
});
self.last_dispatch = now;
}
if (progress.done == false) {
return;
}
fn httpDoneCallback(transfer: *http.Transfer) !void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
log.info(.http, "request complete", .{
.source = "xhr",
@@ -569,20 +478,34 @@ pub const XMLHttpRequest = struct {
.status = self.response_status,
});
// Not that the request is done, the http/client will free the request
// Not that the request is done, the http/client will free the transfer
// object. It isn't safe to keep it around.
self.request = null;
self.transfer = null;
self.state = .done;
self.send_flag = false;
self.dispatchEvt("readystatechange");
const loaded = self.response_bytes.items.len;
// dispatch a progress event load.
self.dispatchProgressEvent("load", .{ .loaded = loaded, .total = loaded });
// dispatch a progress event loadend.
self.dispatchProgressEvent("loadend", .{ .loaded = loaded, .total = loaded });
}
fn httpErrorCallback(transfer: *http.Transfer, err: anyerror) void {
const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx));
// http client will close it after an error, it isn't safe to keep around
self.transfer = null;
self.onErr(err);
}
pub fn _abort(self: *XMLHttpRequest) void {
self.onErr(DOMError.Abort);
self.destructor();
}
fn onErr(self: *XMLHttpRequest, err: anyerror) void {
self.send_flag = false;
@@ -610,15 +533,10 @@ pub const XMLHttpRequest = struct {
log.log(.http, level, "error", .{
.url = self.url,
.err = err,
.source = "xhr",
.source = "xhr.OnErr",
});
}
pub fn _abort(self: *XMLHttpRequest) void {
self.onErr(DOMError.Abort);
self.destructor();
}
pub fn get_responseType(self: *XMLHttpRequest) []const u8 {
return switch (self.response_type) {
.Empty => "",
@@ -660,9 +578,8 @@ pub const XMLHttpRequest = struct {
}
// TODO retrieve the redirected url
pub fn get_responseURL(self: *XMLHttpRequest) ?[]const u8 {
const url = &(self.url orelse return null);
return url.raw;
pub fn get_responseURL(self: *XMLHttpRequest) ?[:0]const u8 {
return self.url;
}
pub fn get_responseXML(self: *XMLHttpRequest) !?Response {
@@ -766,18 +683,8 @@ pub const XMLHttpRequest = struct {
return;
}
var ccharset: [:0]const u8 = "utf-8";
if (mime.charset) |rc| {
if (std.mem.eql(u8, rc, "utf-8") == false) {
ccharset = self.arena.dupeZ(u8, rc) catch {
self.response_obj = .{ .Failure = {} };
return;
};
}
}
var fbs = std.io.fixedBufferStream(self.response_bytes.items);
const doc = parser.documentHTMLParse(fbs.reader(), ccharset) catch {
const doc = parser.documentHTMLParse(fbs.reader(), mime.charset orelse "UTF-8") catch {
self.response_obj = .{ .Failure = {} };
return;
};
@@ -814,26 +721,27 @@ pub const XMLHttpRequest = struct {
}
pub fn _getResponseHeader(self: *XMLHttpRequest, name: []const u8) ?[]const u8 {
return self.response_headers.getFirstValue(name);
for (self.response_headers.items) |entry| {
if (entry.len <= name.len) {
continue;
}
if (std.ascii.eqlIgnoreCase(name, entry[0..name.len]) == false) {
continue;
}
if (entry[name.len] != ':') {
continue;
}
return std.mem.trimLeft(u8, entry[name.len + 1..], " ");
}
return null;
}
// The caller owns the string returned.
// TODO change the return type to express the string ownership and let
// jsruntime free the string once copied to v8.
// see https://github.com/lightpanda-io/jsruntime-lib/issues/195
pub fn _getAllResponseHeaders(self: *XMLHttpRequest) ![]const u8 {
if (self.response_headers.list.items.len == 0) return "";
self.response_headers.sort();
var buf: std.ArrayListUnmanaged(u8) = .{};
const w = buf.writer(self.arena);
for (self.response_headers.list.items) |entry| {
if (entry.value.len == 0) continue;
try w.writeAll(entry.name);
try w.writeAll(": ");
try w.writeAll(entry.value);
for (self.response_headers.items) |entry| {
try w.writeAll(entry);
try w.writeAll("\r\n");
}
@@ -865,8 +773,7 @@ test "Browser.XHR.XMLHttpRequest" {
.{ "req.onload", "function cbk(event) { nb ++; evt = event; }" },
.{ "req.onload = cbk", "function cbk(event) { nb ++; evt = event; }" },
.{ "req.open('GET', 'https://127.0.0.1:9581/xhr')", "undefined" },
.{ "req.setRequestHeader('User-Agent', 'lightpanda/1.0')", "undefined" },
.{ "req.open('GET', 'http://127.0.0.1:9582/xhr')", null },
// ensure open resets values
.{ "req.status ", "0" },
@@ -886,7 +793,12 @@ test "Browser.XHR.XMLHttpRequest" {
.{ "req.status", "200" },
.{ "req.statusText", "OK" },
.{ "req.getResponseHeader('Content-Type')", "text/html; charset=utf-8" },
.{ "req.getAllResponseHeaders().length", "80" },
.{
"req.getAllResponseHeaders()",
"content-length: 100\r\n" ++
"Content-Type: text/html; charset=utf-8\r\n" ++
"Connection: Close\r\n"
},
.{ "req.responseText.length", "100" },
.{ "req.response.length == req.responseText.length", "true" },
.{ "req.responseXML instanceof Document", "true" },
@@ -894,7 +806,7 @@ test "Browser.XHR.XMLHttpRequest" {
try runner.testCases(&.{
.{ "const req2 = new XMLHttpRequest()", "undefined" },
.{ "req2.open('GET', 'https://127.0.0.1:9581/xhr')", "undefined" },
.{ "req2.open('GET', 'http://127.0.0.1:9582/xhr')", "undefined" },
.{ "req2.responseType = 'document'", "document" },
.{ "req2.send()", "undefined" },
@@ -909,7 +821,7 @@ test "Browser.XHR.XMLHttpRequest" {
try runner.testCases(&.{
.{ "const req3 = new XMLHttpRequest()", "undefined" },
.{ "req3.open('GET', 'https://127.0.0.1:9581/xhr/json')", "undefined" },
.{ "req3.open('GET', 'http://127.0.0.1:9582/xhr/json')", "undefined" },
.{ "req3.responseType = 'json'", "json" },
.{ "req3.send()", "undefined" },
@@ -923,7 +835,7 @@ test "Browser.XHR.XMLHttpRequest" {
try runner.testCases(&.{
.{ "const req4 = new XMLHttpRequest()", "undefined" },
.{ "req4.open('POST', 'https://127.0.0.1:9581/xhr')", "undefined" },
.{ "req4.open('POST', 'http://127.0.0.1:9582/xhr')", "undefined" },
.{ "req4.send('foo')", "undefined" },
// Each case executed waits for all loop callaback calls.
@@ -935,7 +847,7 @@ test "Browser.XHR.XMLHttpRequest" {
try runner.testCases(&.{
.{ "const req5 = new XMLHttpRequest()", "undefined" },
.{ "req5.open('GET', 'https://127.0.0.1:9581/xhr')", "undefined" },
.{ "req5.open('GET', 'http://127.0.0.1:9582/xhr')", "undefined" },
.{ "var status = 0; req5.onload = function () { status = this.status };", "function () { status = this.status }" },
.{ "req5.send()", "undefined" },
@@ -956,7 +868,7 @@ test "Browser.XHR.XMLHttpRequest" {
,
null,
},
.{ "req6.open('GET', 'https://127.0.0.1:9581/xhr')", null },
.{ "req6.open('GET', 'http://127.0.0.1:9582/xhr')", null },
.{ "req6.send()", null },
.{ "readyStates.length", "4" },
.{ "readyStates[0] === XMLHttpRequest.OPENED", "true" },

9
src/cdp/domains/page.zig
View File

@@ -155,6 +155,7 @@ fn navigate(cmd: anytype) !void {
.reason = .address_bar,
.cdp_id = cmd.input.id,
});
try page.wait(5);
}
pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.PageNavigate) !void {
@@ -189,13 +190,13 @@ pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.Pa
.frameId = target_id,
.delay = 0,
.reason = reason,
.url = event.url.raw,
.url = event.url,
}, .{ .session_id = session_id });
try cdp.sendEvent("Page.frameRequestedNavigation", .{
.frameId = target_id,
.reason = reason,
.url = event.url.raw,
.url = event.url,
.disposition = "currentTab",
}, .{ .session_id = session_id });
}
@@ -203,7 +204,7 @@ pub fn pageNavigate(arena: Allocator, bc: anytype, event: *const Notification.Pa
// frameStartedNavigating event
try cdp.sendEvent("Page.frameStartedNavigating", .{
.frameId = target_id,
.url = event.url.raw,
.url = event.url,
.loaderId = loader_id,
.navigationType = "differentDocument",
}, .{ .session_id = session_id });
@@ -306,7 +307,7 @@ pub fn pageNavigated(bc: anytype, event: *const Notification.PageNavigated) !voi
.type = "Navigation",
.frame = Frame{
.id = target_id,
.url = event.url.raw,
.url = event.url,
.loaderId = bc.loader_id,
.securityOrigin = bc.security_origin,
.secureContextType = bc.secure_context_type,

72
src/http/client.zig
View File

@@ -23,6 +23,7 @@ pub const c = @cImport({
const ENABLE_DEBUG = false;
const std = @import("std");
const log = @import("../log.zig");
const builtin = @import("builtin");
const errors = @import("errors.zig");
@@ -36,14 +37,11 @@ pub fn init() !void {
}
}
const LogFn = *const fn(ctx: []const u8, err: anyerror) void;
pub fn deinit() void {
c.curl_global_cleanup();
}
pub const Client = struct {
log: LogFn,
active: usize,
multi: *c.CURLM,
handles: Handles,
@@ -57,7 +55,6 @@ pub const Client = struct {
const RequestQueue = std.DoublyLinkedList(Request);
const Opts = struct {
log: ?LogFn = null,
timeout_ms: u31 = 0,
max_redirects: u8 = 10,
connect_timeout_ms: u31 = 5000,
@@ -80,7 +77,6 @@ pub const Client = struct {
errdefer _ = c.curl_multi_cleanup(multi);
client.* = .{
.log = opts.log orelse noopLog,
.queue = .{},
.active = 0,
.multi = multi,
@@ -101,7 +97,7 @@ pub const Client = struct {
self.allocator.destroy(self);
}
pub fn tick(self: *Client, timeout_ms: usize) !usize {
pub fn tick(self: *Client, timeout_ms: usize) !void {
var handles = &self.handles.available;
while (true) {
if (handles.first == null) {
@@ -116,7 +112,6 @@ pub const Client = struct {
}
try self.perform(@intCast(timeout_ms));
return self.active;
}
pub fn request(self: *Client, req: Request) !void {
@@ -144,16 +139,11 @@ pub const Client = struct {
.OPTIONS => try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_CUSTOMREQUEST, "options")),
}
const header_list = c.curl_slist_append(null, "User-Agent: lightpanda/1");
const header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0");
errdefer c.curl_slist_free_all(header_list);
try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list));
//try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(STRING.len))));
//try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDS, STRING.ptr));
break :blk header_list;
};
@@ -345,8 +335,9 @@ pub const Request = struct {
// arbitrary data that can be associated with this request
ctx: *anyopaque = undefined,
start_callback: ?*const fn(req: *Transfer) anyerror!void = noopStart,
header_callback: *const fn (req: *Transfer) anyerror!void,
start_callback: ?*const fn(req: *Transfer) anyerror!void = null,
header_callback: ?*const fn (req: *Transfer, header: []const u8) anyerror!void = null ,
header_done_callback: *const fn (req: *Transfer) anyerror!void,
data_callback: *const fn(req: *Transfer, data: []const u8) anyerror!void,
done_callback: *const fn(req: *Transfer) anyerror!void,
error_callback: *const fn(req: *Transfer, err: anyerror) void,
@@ -366,6 +357,12 @@ pub const Transfer = struct {
// needs to be freed when we're done
_request_header_list: ?*c.curl_slist = null,
fn deinit(self: *Transfer) void {
if (self._request_header_list) |list| {
c.curl_slist_free_all(list);
}
}
pub fn format(self: *const Transfer, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void {
const req = self.req;
return writer.print("[{d}] {s} {s}", .{self.id, @tagName(req.method), req.url});
@@ -375,10 +372,23 @@ pub const Transfer = struct {
self.req.error_callback(self, err);
}
fn deinit(self: *Transfer) void {
if (self._request_header_list) |list| {
c.curl_slist_free_all(list);
pub fn setBody(self: *Transfer, body: []const u8) !void {
const easy = self.handle.easy;
try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDSIZE, @as(c_long, @intCast(body.len))));
try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDS, body.ptr));
}
pub fn addHeader(self: *Transfer, value: [:0]const u8) !void {
self._request_header_list = c.curl_slist_append(self._request_header_list, value);
}
pub fn abort(self: *Transfer) void {
var client = self.handle.client;
errorMCheck(c.curl_multi_remove_handle(client.multi, self.handle.easy)) catch |err| {
log.err(.http, "Failed to abort", .{.err = err});
};
client.active -= 1;
self.deinit();
}
fn headerCallback(buffer: [*]const u8, header_count: usize, buf_len: usize, data: *anyopaque) callconv(.c) usize {
@@ -387,11 +397,13 @@ pub const Transfer = struct {
const handle: *Handle = @alignCast(@ptrCast(data));
var transfer = fromEasy(handle.easy) catch |err| {
handle.client.log("retrieve private info", err);
log.err(.http, "retrive private info", .{.err = err});
return 0;
};
const header = buffer[0..buf_len];
std.debug.assert(std.mem.endsWith(u8, buffer[0..buf_len], "\r\n"));
const header = buffer[0..buf_len - 2];
if (transfer.response_header == null) {
if (buf_len < 13 or std.mem.startsWith(u8, header, "HTTP/") == false) {
@@ -439,8 +451,7 @@ pub const Transfer = struct {
if (buf_len > CONTENT_TYPE_LEN) {
if (std.ascii.eqlIgnoreCase(header[0..CONTENT_TYPE_LEN], "content-type:")) {
const value = std.mem.trimLeft(u8, header[CONTENT_TYPE_LEN..], " ");
// -2 to trim the trailing \r\n
const len = @min(value.len - 2, hdr._content_type.len);
const len = @min(value.len, hdr._content_type.len);
hdr._content_type_len = len;
@memcpy(hdr._content_type[0..len], value[0..len]);
}
@@ -448,10 +459,14 @@ pub const Transfer = struct {
}
if (buf_len == 2) {
transfer.req.header_callback(transfer) catch {
transfer.req.header_done_callback(transfer) catch {
// returning < buf_len terminates the request
return 0;
};
} else {
if (transfer.req.header_callback) |cb| {
cb(transfer, header) catch return 0;
}
}
return buf_len;
}
@@ -462,8 +477,8 @@ pub const Transfer = struct {
const handle: *Handle = @alignCast(@ptrCast(data));
var transfer = fromEasy(handle.easy) catch |err| {
handle.client.log("retrieve private info", err);
return 0;
log.err(.http, "retrive private info", .{.err = err});
return c.CURL_WRITEFUNC_ERROR;
};
if (transfer._redirecting) {
@@ -535,10 +550,3 @@ pub const ProxyAuth = union(enum) {
bearer: struct { token: []const u8 },
};
fn noopLog(ctx: []const u8, _: anyerror) void {
_ = ctx;
}
fn noopStart(transfer: *Transfer) !void {
_ = transfer;
}

80
src/main.zig
View File

@@ -667,88 +667,42 @@ fn serveHTTP(address: std.net.Address) !void {
test_wg.finish();
var read_buffer: [1024]u8 = undefined;
ACCEPT: while (true) {
defer _ = arena.reset(.{ .free_all = {} });
const aa = arena.allocator();
while (true) {
var conn = try listener.accept();
defer conn.stream.close();
var http_server = std.http.Server.init(conn, &read_buffer);
var connect_headers: std.ArrayListUnmanaged(std.http.Header) = .{};
REQUEST: while (true) {
var request = http_server.receiveHead() catch |err| switch (err) {
error.HttpConnectionClosing => continue :ACCEPT,
error.HttpConnectionClosing => continue,
else => {
std.debug.print("Test HTTP Server error: {}\n", .{err});
return err;
},
};
if (request.head.method == .CONNECT) {
try request.respond("", .{ .status = .ok });
// Proxy headers and destination headers are separated in the case of a CONNECT proxy
// We store the CONNECT headers, then continue with the request for the destination
var it = request.iterateHeaders();
while (it.next()) |hdr| {
try connect_headers.append(aa, .{
.name = try std.fmt.allocPrint(aa, "__{s}", .{hdr.name}),
.value = try aa.dupe(u8, hdr.value),
});
}
continue :REQUEST;
}
const path = request.head.target;
if (std.mem.eql(u8, path, "/loader")) {
try request.respond("Hello!", .{
.extra_headers = &.{.{ .name = "Connection", .value = "close" }},
});
} else if (std.mem.eql(u8, path, "/http_client/simple")) {
try request.respond("", .{
.extra_headers = &.{.{ .name = "Connection", .value = "close" }},
});
} else if (std.mem.eql(u8, path, "/http_client/redirect")) {
try request.respond("", .{
.status = .moved_permanently,
} else if (std.mem.eql(u8, path, "/xhr")) {
try request.respond("1234567890" ** 10, .{
.extra_headers = &.{
.{ .name = "Connection", .value = "close" },
.{ .name = "LOCATION", .value = "../http_client/echo" },
.{ .name = "Content-Type", .value = "text/html; charset=utf-8" },
.{ .name = "Connection", .value = "Close" },
},
});
} else if (std.mem.eql(u8, path, "/http_client/redirect/secure")) {
try request.respond("", .{
.status = .moved_permanently,
.extra_headers = &.{ .{ .name = "Connection", .value = "close" }, .{ .name = "LOCATION", .value = "https://127.0.0.1:9581/http_client/body" } },
} else if (std.mem.eql(u8, path, "/xhr/json")) {
try request.respond("{\"over\":\"9000!!!\"}", .{
.extra_headers = &.{
.{ .name = "Content-Type", .value = "application/json" },
.{ .name = "Connection", .value = "Close" },
},
});
} else if (std.mem.eql(u8, path, "/http_client/gzip")) {
const body = &.{ 0x1f, 0x8b, 0x08, 0x08, 0x01, 0xc6, 0x19, 0x68, 0x00, 0x03, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x00, 0x73, 0x54, 0xc8, 0x4b, 0x2d, 0x57, 0x48, 0x2a, 0xca, 0x2f, 0x2f, 0x4e, 0x2d, 0x52, 0x48, 0x2a, 0xcd, 0xcc, 0x29, 0x51, 0x48, 0xcb, 0x2f, 0x52, 0xc8, 0x4d, 0x4c, 0xce, 0xc8, 0xcc, 0x4b, 0x2d, 0xe6, 0x02, 0x00, 0xe7, 0xc3, 0x4b, 0x27, 0x21, 0x00, 0x00, 0x00 };
try request.respond(body, .{
.extra_headers = &.{ .{ .name = "Connection", .value = "close" }, .{ .name = "Content-Encoding", .value = "gzip" } },
});
} else if (std.mem.eql(u8, path, "/http_client/echo")) {
var headers: std.ArrayListUnmanaged(std.http.Header) = .{};
var it = request.iterateHeaders();
while (it.next()) |hdr| {
try headers.append(aa, .{
.name = try std.fmt.allocPrint(aa, "_{s}", .{hdr.name}),
.value = hdr.value,
});
}
if (connect_headers.items.len > 0) {
try headers.appendSlice(aa, connect_headers.items);
connect_headers.clearRetainingCapacity();
}
try headers.append(aa, .{ .name = "Connection", .value = "Close" });
try request.respond("over 9000!", .{
.status = .created,
.extra_headers = headers.items,
});
}
continue :ACCEPT;
} else {
// should not have an unknown path
unreachable;
}
}
}

4
src/notification.zig
View File

@@ -81,13 +81,13 @@ pub const Notification = struct {
pub const PageNavigate = struct {
timestamp: u32,
url: *const URL,
url: []const u8,
opts: page.NavigateOpts,
};
pub const PageNavigated = struct {
timestamp: u32,
url: *const URL,
url: []const u8,
};
pub const RequestStart = struct {

8
src/server.zig
View File

@@ -1052,7 +1052,15 @@ pub fn run(
// infinite loop on I/O events, either:
// - cmd from incoming connection on server socket
// - JS callbacks events from scripts
// var http_client = app.http_client;
while (true) {
// // @newhttp
// // This is a temporary hack for the newhttp work. The issue is that we
// // now have 2 event loops.
// if (http_client.active > 0) {
// _ = try http_client.tick(10);
// }
try loop.io.run_for_ns(10 * std.time.ns_per_ms);
}
}

2
src/telemetry/lightpanda.zig
View File

@@ -35,7 +35,7 @@ pub const LightPanda = struct {
.thread = null,
.running = true,
.allocator = allocator,
.client = &app.http_client,
.client = app.http_client,
.uri = std.Uri.parse(URL) catch unreachable,
.node_pool = std.heap.MemoryPool(List.Node).init(allocator),
};

2
src/telemetry/telemetry.zig
View File

@@ -79,7 +79,7 @@ fn TelemetryT(comptime P: type) type {
const self: *Self = @alignCast(@ptrCast(ctx));
self.record(.{ .navigate = .{
.proxy = false,
.tls = std.ascii.eqlIgnoreCase(data.url.scheme(), "https"),
.tls = std.ascii.startsWithIgnoreCase(data.url, "https://"),
} });
}
};

6
src/testing.zig
View File

@@ -409,6 +409,10 @@ pub const JsRunner = struct {
const html_doc = try parser.documentHTMLParseFromStr(opts.html);
try page.setDocument(html_doc);
// after the page is considered loaded, page.wait can exit early if
// there's no IO/timeouts. So setting this speeds up our tests
page.loaded = true;
return .{
.app = app,
.page = page,
@@ -441,7 +445,7 @@ pub const JsRunner = struct {
}
return err;
};
try self.page.loop.run(std.time.ns_per_ms * 200);
try self.page.wait(1);
@import("root").js_runner_duration += std.time.Instant.since(try std.time.Instant.now(), start);
if (case.@"1") |expected| {