mirror of
https://github.com/lightpanda-io/browser.git
synced 2025-10-29 23:23:28 +00:00
Merge branch 'main' into cdp_struct
This commit is contained in:
107
.github/workflows/e2e-test.yml
vendored
Normal file
107
.github/workflows/e2e-test.yml
vendored
Normal file
@@ -0,0 +1,107 @@
|
|||||||
|
name: e2e-test
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- "build.zig"
|
||||||
|
- "src/**/*.zig"
|
||||||
|
- "src/*.zig"
|
||||||
|
- "vendor/zig-js-runtime"
|
||||||
|
- ".github/**"
|
||||||
|
- "vendor/**"
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
# By default GH trigger on types opened, synchronize and reopened.
|
||||||
|
# see https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
|
||||||
|
# Since we skip the job when the PR is in draft state, we want to force CI
|
||||||
|
# running when the PR is marked ready_for_review w/o other change.
|
||||||
|
# see https://github.com/orgs/community/discussions/25722#discussioncomment-3248917
|
||||||
|
types: [opened, synchronize, reopened, ready_for_review]
|
||||||
|
|
||||||
|
paths:
|
||||||
|
- ".github/**"
|
||||||
|
- "build.zig"
|
||||||
|
- "src/**/*.zig"
|
||||||
|
- "src/*.zig"
|
||||||
|
- "vendor/**"
|
||||||
|
- ".github/**"
|
||||||
|
- "vendor/**"
|
||||||
|
# Allows you to run this workflow manually from the Actions tab
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
zig-build-release:
|
||||||
|
name: zig build release
|
||||||
|
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
# fetch submodules recusively, to get zig-js-runtime submodules also.
|
||||||
|
submodules: recursive
|
||||||
|
|
||||||
|
- uses: ./.github/actions/install
|
||||||
|
|
||||||
|
- name: zig build release
|
||||||
|
run: zig build -Doptimize=ReleaseSafe -Dengine=v8
|
||||||
|
|
||||||
|
- name: upload artifact
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: lightpanda-build-release
|
||||||
|
path: |
|
||||||
|
zig-out/bin/lightpanda
|
||||||
|
retention-days: 1
|
||||||
|
|
||||||
|
puppeteer:
|
||||||
|
name: puppeteer
|
||||||
|
needs: zig-build-release
|
||||||
|
|
||||||
|
env:
|
||||||
|
MAX_MEMORY: 25000
|
||||||
|
MAX_AVG_DURATION: 24
|
||||||
|
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
repository: 'lightpanda-io/demo'
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
- run: npm install
|
||||||
|
|
||||||
|
- name: download artifact
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: lightpanda-build-release
|
||||||
|
|
||||||
|
- run: chmod a+x ./lightpanda
|
||||||
|
|
||||||
|
- name: run puppeteer
|
||||||
|
run: |
|
||||||
|
python3 -m http.server 1234 -d ./public & echo $! > PYTHON.pid
|
||||||
|
./lightpanda & echo $! > LPD.pid
|
||||||
|
RUNS=100 npm run bench-puppeteer-cdp > puppeteer.out || exit 1
|
||||||
|
cat /proc/`cat LPD.pid`/status |grep VmHWM|grep -oP '\d+' > LPD.VmHWM
|
||||||
|
kill `cat LPD.pid` `cat PYTHON.pid`
|
||||||
|
|
||||||
|
- name: puppeteer result
|
||||||
|
run: cat puppeteer.out
|
||||||
|
|
||||||
|
- name: memory regression
|
||||||
|
run: |
|
||||||
|
export LPD_VmHWM=`cat LPD.VmHWM`
|
||||||
|
echo "Peak resident set size: $LPD_VmHWM"
|
||||||
|
test "$LPD_VmHWM" -le "$MAX_MEMORY"
|
||||||
|
|
||||||
|
- name: duration regression
|
||||||
|
run: |
|
||||||
|
export PUPPETEER_AVG_DURATION=`cat puppeteer.out|grep 'avg run'|sed 's/avg run duration (ms) //'`
|
||||||
|
echo "puppeteer avg duration: $PUPPETEER_AVG_DURATION"
|
||||||
|
test "$PUPPETEER_AVG_DURATION" -le "$MAX_AVG_DURATION"
|
||||||
|
|
||||||
47
.github/workflows/zig-test.yml
vendored
47
.github/workflows/zig-test.yml
vendored
@@ -66,26 +66,6 @@ jobs:
|
|||||||
zig-out/bin/lightpanda
|
zig-out/bin/lightpanda
|
||||||
retention-days: 1
|
retention-days: 1
|
||||||
|
|
||||||
zig-build-release:
|
|
||||||
name: zig build release
|
|
||||||
|
|
||||||
# Don't run the CI on PR
|
|
||||||
if: github.event_name != 'pull_request'
|
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
# fetch submodules recusively, to get zig-js-runtime submodules also.
|
|
||||||
submodules: recursive
|
|
||||||
|
|
||||||
- uses: ./.github/actions/install
|
|
||||||
|
|
||||||
- name: zig build release
|
|
||||||
run: zig build -Doptimize=ReleaseSafe -Dengine=v8
|
|
||||||
|
|
||||||
zig-test:
|
zig-test:
|
||||||
name: zig test
|
name: zig test
|
||||||
|
|
||||||
@@ -141,30 +121,3 @@ jobs:
|
|||||||
|
|
||||||
- name: format and send json result
|
- name: format and send json result
|
||||||
run: /perf-fmt bench-browser ${{ github.sha }} bench.json
|
run: /perf-fmt bench-browser ${{ github.sha }} bench.json
|
||||||
|
|
||||||
demo-puppeteer:
|
|
||||||
name: demo-puppeteer
|
|
||||||
needs: zig-build-dev
|
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
repository: 'lightpanda-io/demo'
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- run: npm install
|
|
||||||
|
|
||||||
- name: download artifact
|
|
||||||
uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
name: lightpanda-build-dev
|
|
||||||
|
|
||||||
- run: chmod a+x ./lightpanda
|
|
||||||
|
|
||||||
- name: run puppeteer
|
|
||||||
run: |
|
|
||||||
python3 -m http.server 1234 -d ./public &
|
|
||||||
./lightpanda &
|
|
||||||
RUNS=2 npm run bench-puppeteer-cdp
|
|
||||||
|
|||||||
2
Makefile
2
Makefile
@@ -144,7 +144,7 @@ _install-netsurf: clean-netsurf
|
|||||||
BUILDDIR=$(BC_NS)/build/libdom make install && \
|
BUILDDIR=$(BC_NS)/build/libdom make install && \
|
||||||
printf "\e[33mRunning libdom example...\e[0m\n" && \
|
printf "\e[33mRunning libdom example...\e[0m\n" && \
|
||||||
cd examples && \
|
cd examples && \
|
||||||
zig cc \
|
$(ZIG) cc \
|
||||||
-I$(ICONV)/include \
|
-I$(ICONV)/include \
|
||||||
-I$(BC_NS)/include \
|
-I$(BC_NS)/include \
|
||||||
-L$(ICONV)/lib \
|
-L$(ICONV)/lib \
|
||||||
|
|||||||
109
README.md
109
README.md
@@ -8,33 +8,35 @@
|
|||||||
|
|
||||||
<div align="center">
|
<div align="center">
|
||||||
|
|
||||||
[](https://github.com/lightpanda-io/browser/commits/main)
|
|
||||||
[](https://github.com/lightpanda-io/browser/blob/main/LICENSE)
|
[](https://github.com/lightpanda-io/browser/blob/main/LICENSE)
|
||||||
[](https://twitter.com/lightpanda_io)
|
[](https://twitter.com/lightpanda_io)
|
||||||
[](https://github.com/lightpanda-io/browser)
|
[](https://github.com/lightpanda-io/browser)
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div align="center">
|
|
||||||
|
|
||||||
<a href="https://trendshift.io/repositories/12815" target="_blank"><img src="https://trendshift.io/api/badge/repositories/12815" alt="lightpanda-io%2Fbrowser | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
|
||||||
|
|
||||||
</div>
|
|
||||||
|
|
||||||
Lightpanda is the open-source browser made for headless usage:
|
Lightpanda is the open-source browser made for headless usage:
|
||||||
|
|
||||||
- Javascript execution
|
- Javascript execution
|
||||||
- Support of Web APIs (partial, WIP)
|
- Support of Web APIs (partial, WIP)
|
||||||
- Compatible with Playwright, Puppeteer through CDP (WIP)
|
- Compatible with Playwright, Puppeteer through CDP (WIP)
|
||||||
|
|
||||||
Fast web automation for AI agents, LLM training, scraping and testing with minimal memory footprint:
|
Fast web automation for AI agents, LLM training, scraping and testing:
|
||||||
|
|
||||||
- Ultra-low memory footprint (9x less than Chrome)
|
- Ultra-low memory footprint (9x less than Chrome)
|
||||||
- Exceptionally fast execution (11x faster than Chrome) & instant startup
|
- Exceptionally fast execution (11x faster than Chrome)
|
||||||
|
- Instant startup
|
||||||
|
<br />
|
||||||
|
<br />
|
||||||
|
|
||||||
<img width=500px src="https://cdn.lightpanda.io/assets/images/benchmark_2024-12-04.png">
|
[<img width="350px" src="https://cdn.lightpanda.io/assets/images/github/execution-time.svg">
|
||||||
|
](https://github.com/lightpanda-io/demo)
|
||||||
|
 
|
||||||
|
[<img width="350px" src="https://cdn.lightpanda.io/assets/images/github/memory-frame.svg">
|
||||||
|
](https://github.com/lightpanda-io/demo)
|
||||||
|
</div>
|
||||||
|
|
||||||
See [benchmark details](https://github.com/lightpanda-io/demo).
|
_Puppeteer requesting 100 pages from a local website on a AWS EC2 m5.large instance.
|
||||||
|
See [benchmark details](https://github.com/lightpanda-io/demo)._
|
||||||
|
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
@@ -44,29 +46,24 @@ You can download the last binary from the [nightly
|
|||||||
builds](https://github.com/lightpanda-io/browser/releases/tag/nightly) for
|
builds](https://github.com/lightpanda-io/browser/releases/tag/nightly) for
|
||||||
Linux x86_64 and MacOS aarch64.
|
Linux x86_64 and MacOS aarch64.
|
||||||
|
|
||||||
|
*For linux*
|
||||||
```console
|
```console
|
||||||
# Download the binary
|
curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/nightly/lightpanda-x86_64-linux && \
|
||||||
$ wget https://github.com/lightpanda-io/browser/releases/download/nightly/lightpanda-x86_64-linux
|
chmod a+x ./lightpanda
|
||||||
$ chmod a+x ./lightpanda-x86_64-linux
|
```
|
||||||
$ ./lightpanda-x86_64-linux -h
|
|
||||||
usage: ./lightpanda-x86_64-linux [options] [URL]
|
|
||||||
|
|
||||||
start Lightpanda browser
|
*For MacOS*
|
||||||
|
```console
|
||||||
* if an url is provided the browser will fetch the page and exit
|
curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/nightly/lightpanda-aarch64-macos && \
|
||||||
* otherwhise the browser starts a CDP server
|
chmod a+x ./lightpanda
|
||||||
|
|
||||||
-h, --help Print this help message and exit.
|
|
||||||
--host Host of the CDP server (default "127.0.0.1")
|
|
||||||
--port Port of the CDP server (default "9222")
|
|
||||||
--timeout Timeout for incoming connections of the CDP server (in seconds, default "3")
|
|
||||||
--dump Dump document in stdout (fetch mode only)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Dump an URL
|
### Dump an URL
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ ./lightpanda-x86_64-linux --dump https://lightpanda.io
|
./lightpanda --dump https://lightpanda.io
|
||||||
|
```
|
||||||
|
```console
|
||||||
info(browser): GET https://lightpanda.io/ http.Status.ok
|
info(browser): GET https://lightpanda.io/ http.Status.ok
|
||||||
info(browser): fetch script https://api.website.lightpanda.io/js/script.js: http.Status.ok
|
info(browser): fetch script https://api.website.lightpanda.io/js/script.js: http.Status.ok
|
||||||
info(browser): eval remote https://api.website.lightpanda.io/js/script.js: TypeError: Cannot read properties of undefined (reading 'pushState')
|
info(browser): eval remote https://api.website.lightpanda.io/js/script.js: TypeError: Cannot read properties of undefined (reading 'pushState')
|
||||||
@@ -76,7 +73,9 @@ info(browser): eval remote https://api.website.lightpanda.io/js/script.js: TypeE
|
|||||||
### Start a CDP server
|
### Start a CDP server
|
||||||
|
|
||||||
```console
|
```console
|
||||||
$ ./lightpanda-x86_64-linux --host 127.0.0.1 --port 9222
|
./lightpanda --host 127.0.0.1 --port 9222
|
||||||
|
```
|
||||||
|
```console
|
||||||
info(websocket): starting blocking worker to listen on 127.0.0.1:9222
|
info(websocket): starting blocking worker to listen on 127.0.0.1:9222
|
||||||
info(server): accepting new conn...
|
info(server): accepting new conn...
|
||||||
```
|
```
|
||||||
@@ -98,12 +97,44 @@ const browser = await puppeteer.connect({
|
|||||||
const context = await browser.createBrowserContext();
|
const context = await browser.createBrowserContext();
|
||||||
const page = await context.newPage();
|
const page = await context.newPage();
|
||||||
|
|
||||||
|
// Dump all the links from the page.
|
||||||
await page.goto('https://wikipedia.com/');
|
await page.goto('https://wikipedia.com/');
|
||||||
|
|
||||||
|
const links = await page.evaluate(() => {
|
||||||
|
return Array.from(document.querySelectorAll('a')).map(row => {
|
||||||
|
return row.getAttribute('href');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(links);
|
||||||
|
|
||||||
await page.close();
|
await page.close();
|
||||||
await context.close();
|
await context.close();
|
||||||
|
await browser.disconnect();
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Lightpanda is still a work in progress and is currently at a Beta stage.
|
||||||
|
|
||||||
|
:warning: You should expect most websites to fail or crash.
|
||||||
|
|
||||||
|
Here are the key features we have implemented:
|
||||||
|
|
||||||
|
- [x] HTTP loader
|
||||||
|
- [x] HTML parser and DOM tree (based on Netsurf libs)
|
||||||
|
- [x] Javascript support (v8)
|
||||||
|
- [x] Basic DOM APIs
|
||||||
|
- [x] Ajax
|
||||||
|
- [x] XHR API
|
||||||
|
- [x] Fetch API
|
||||||
|
- [x] DOM dump
|
||||||
|
- [x] Basic CDP/websockets server
|
||||||
|
|
||||||
|
NOTE: There are hundreds of Web APIs. Developing a browser (even just for headless mode) is a huge task. Coverage will increase over time.
|
||||||
|
|
||||||
|
You can also follow the progress of our Javascript support in our dedicated [zig-js-runtime](https://github.com/lightpanda-io/zig-js-runtime#development) project.
|
||||||
|
|
||||||
## Build from sources
|
## Build from sources
|
||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
@@ -260,25 +291,3 @@ If we want both Javascript and performance in a true headless browser, we need t
|
|||||||
- Not based on Chromium, Blink or WebKit
|
- Not based on Chromium, Blink or WebKit
|
||||||
- Low-level system programming language (Zig) with optimisations in mind
|
- Low-level system programming language (Zig) with optimisations in mind
|
||||||
- Opinionated: without graphical rendering
|
- Opinionated: without graphical rendering
|
||||||
|
|
||||||
## Status
|
|
||||||
|
|
||||||
Lightpanda is still a work in progress and is currently at a Beta stage.
|
|
||||||
|
|
||||||
:warning: You should expect most websites to fail or crash.
|
|
||||||
|
|
||||||
Here are the key features we have implemented:
|
|
||||||
|
|
||||||
- [x] HTTP loader
|
|
||||||
- [x] HTML parser and DOM tree (based on Netsurf libs)
|
|
||||||
- [x] Javascript support (v8)
|
|
||||||
- [x] Basic DOM APIs
|
|
||||||
- [x] Ajax
|
|
||||||
- [x] XHR API
|
|
||||||
- [x] Fetch API
|
|
||||||
- [x] DOM dump
|
|
||||||
- [x] Basic CDP/websockets server
|
|
||||||
|
|
||||||
NOTE: There are hundreds of Web APIs. Developing a browser (even just for headless mode) is a huge task. Coverage will increase over time.
|
|
||||||
|
|
||||||
You can also follow the progress of our Javascript support in our dedicated [zig-js-runtime](https://github.com/lightpanda-io/zig-js-runtime#development) project.
|
|
||||||
|
|||||||
@@ -267,7 +267,12 @@ pub const Page = struct {
|
|||||||
|
|
||||||
// add global objects
|
// add global objects
|
||||||
log.debug("setup global env", .{});
|
log.debug("setup global env", .{});
|
||||||
try self.session.env.bindGlobal(&self.session.window);
|
|
||||||
|
if (comptime builtin.is_test == false) {
|
||||||
|
// By not loading this during tests, we aren't required to load
|
||||||
|
// all of the interfaces into zig-js-runtime.
|
||||||
|
try self.session.env.bindGlobal(&self.session.window);
|
||||||
|
}
|
||||||
|
|
||||||
// load polyfills
|
// load polyfills
|
||||||
try polyfill.load(self.arena.allocator(), self.session.env);
|
try polyfill.load(self.arena.allocator(), self.session.env);
|
||||||
|
|||||||
24
src/main.zig
24
src/main.zig
@@ -202,23 +202,13 @@ pub fn main() !void {
|
|||||||
|
|
||||||
// allocator
|
// allocator
|
||||||
// - in Debug mode we use the General Purpose Allocator to detect memory leaks
|
// - in Debug mode we use the General Purpose Allocator to detect memory leaks
|
||||||
// - in Release mode we use the page allocator
|
// - in Release mode we use the c allocator
|
||||||
var alloc: std.mem.Allocator = undefined;
|
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
||||||
var _gpa: ?std.heap.GeneralPurposeAllocator(.{}) = null;
|
const alloc = if (builtin.mode == .Debug) gpa.allocator() else std.heap.c_allocator;
|
||||||
if (builtin.mode == .Debug) {
|
|
||||||
_gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
defer if (builtin.mode == .Debug) {
|
||||||
alloc = _gpa.?.allocator();
|
_ = gpa.detectLeaks();
|
||||||
} else {
|
};
|
||||||
alloc = std.heap.page_allocator;
|
|
||||||
}
|
|
||||||
defer {
|
|
||||||
if (_gpa) |*gpa| {
|
|
||||||
switch (gpa.deinit()) {
|
|
||||||
.ok => std.debug.print("No memory leaks\n", .{}),
|
|
||||||
.leak => @panic("Memory leak"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// args
|
// args
|
||||||
var args: std.process.ArgIterator = undefined;
|
var args: std.process.ArgIterator = undefined;
|
||||||
|
|||||||
@@ -336,12 +336,6 @@ test {
|
|||||||
std.testing.refAllDecls(queryTest);
|
std.testing.refAllDecls(queryTest);
|
||||||
|
|
||||||
std.testing.refAllDecls(@import("generate.zig"));
|
std.testing.refAllDecls(@import("generate.zig"));
|
||||||
|
|
||||||
// Don't use refAllDecls, as this will pull in the entire project
|
|
||||||
// and break the test build.
|
|
||||||
// We should fix this. See this branch & the commit message for details:
|
|
||||||
// https://github.com/karlseguin/browser/commit/193ab5ceab3d3758ea06db04f7690460d79eb79e
|
|
||||||
_ = @import("server.zig");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn testJSRuntime(alloc: std.mem.Allocator) !void {
|
fn testJSRuntime(alloc: std.mem.Allocator) !void {
|
||||||
|
|||||||
119
src/server.zig
119
src/server.zig
@@ -184,6 +184,13 @@ const Server = struct {
|
|||||||
client.close(null);
|
client.close(null);
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
if (size == 0) {
|
||||||
|
if (self.client != null) {
|
||||||
|
self.client = null;
|
||||||
|
}
|
||||||
|
self.queueAccept();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const more = client.processData(size) catch |err| {
|
const more = client.processData(size) catch |err| {
|
||||||
log.err("Client Processing Error: {any}\n", .{err});
|
log.err("Client Processing Error: {any}\n", .{err});
|
||||||
@@ -970,14 +977,6 @@ pub fn run(
|
|||||||
timeout: u64,
|
timeout: u64,
|
||||||
loop: *jsruntime.Loop,
|
loop: *jsruntime.Loop,
|
||||||
) !void {
|
) !void {
|
||||||
if (comptime builtin.is_test) {
|
|
||||||
// There's bunch of code that won't compiler in a test build (because
|
|
||||||
// it relies on a global root.Types). So we fight the compiler and make
|
|
||||||
// sure it doesn't include any of that code. Hopefully one day we can
|
|
||||||
// remove all this.
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// create socket
|
// create socket
|
||||||
const flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | posix.SOCK.NONBLOCK;
|
const flags = posix.SOCK.STREAM | posix.SOCK.CLOEXEC | posix.SOCK.NONBLOCK;
|
||||||
const listener = try posix.socket(address.any.family, flags, posix.IPPROTO.TCP);
|
const listener = try posix.socket(address.any.family, flags, posix.IPPROTO.TCP);
|
||||||
@@ -1555,6 +1554,49 @@ test "server: mask" {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "server: 404" {
|
||||||
|
var c = try createTestClient();
|
||||||
|
defer c.deinit();
|
||||||
|
|
||||||
|
const res = try c.httpRequest("GET /unknown HTTP/1.1\r\n\r\n");
|
||||||
|
try testing.expectEqualStrings("HTTP/1.1 404 \r\n" ++
|
||||||
|
"Connection: Close\r\n" ++
|
||||||
|
"Content-Length: 9\r\n\r\n" ++
|
||||||
|
"Not found", res);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "server: get /json/version" {
|
||||||
|
const expected_response =
|
||||||
|
"HTTP/1.1 200 OK\r\n" ++
|
||||||
|
"Content-Length: 48\r\n" ++
|
||||||
|
"Content-Type: application/json; charset=UTF-8\r\n\r\n" ++
|
||||||
|
"{\"webSocketDebuggerUrl\": \"ws://127.0.0.1:9583/\"}";
|
||||||
|
|
||||||
|
{
|
||||||
|
// twice on the same connection
|
||||||
|
var c = try createTestClient();
|
||||||
|
defer c.deinit();
|
||||||
|
|
||||||
|
const res1 = try c.httpRequest("GET /json/version HTTP/1.1\r\n\r\n");
|
||||||
|
try testing.expectEqualStrings(expected_response, res1);
|
||||||
|
|
||||||
|
const res2 = try c.httpRequest("GET /json/version HTTP/1.1\r\n\r\n");
|
||||||
|
try testing.expectEqualStrings(expected_response, res2);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
// again on a new connection
|
||||||
|
var c = try createTestClient();
|
||||||
|
defer c.deinit();
|
||||||
|
|
||||||
|
const res1 = try c.httpRequest("GET /json/version HTTP/1.1\r\n\r\n");
|
||||||
|
try testing.expectEqualStrings(expected_response, res1);
|
||||||
|
|
||||||
|
const res2 = try c.httpRequest("GET /json/version HTTP/1.1\r\n\r\n");
|
||||||
|
try testing.expectEqualStrings(expected_response, res2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn assertHTTPError(
|
fn assertHTTPError(
|
||||||
expected_error: anyerror,
|
expected_error: anyerror,
|
||||||
comptime expected_status: u16,
|
comptime expected_status: u16,
|
||||||
@@ -1680,6 +1722,7 @@ const MockServer = struct {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
const MockCDP = struct {
|
const MockCDP = struct {
|
||||||
messages: std.ArrayListUnmanaged([]const u8) = .{},
|
messages: std.ArrayListUnmanaged([]const u8) = .{},
|
||||||
|
|
||||||
@@ -1705,3 +1748,63 @@ const MockCDP = struct {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
fn createTestClient() !TestClient {
|
||||||
|
const address = std.net.Address.initIp4([_]u8{ 127, 0, 0, 1 }, 9583);
|
||||||
|
const stream = try std.net.tcpConnectToAddress(address);
|
||||||
|
|
||||||
|
const timeout = std.mem.toBytes(posix.timeval{
|
||||||
|
.tv_sec = 2,
|
||||||
|
.tv_usec = 0,
|
||||||
|
});
|
||||||
|
try posix.setsockopt(stream.handle, posix.SOL.SOCKET, posix.SO.RCVTIMEO, &timeout);
|
||||||
|
try posix.setsockopt(stream.handle, posix.SOL.SOCKET, posix.SO.SNDTIMEO, &timeout);
|
||||||
|
return .{ .stream = stream };
|
||||||
|
}
|
||||||
|
|
||||||
|
const TestClient = struct {
|
||||||
|
stream: std.net.Stream,
|
||||||
|
buf: [1024]u8 = undefined,
|
||||||
|
|
||||||
|
fn deinit(self: *TestClient) void {
|
||||||
|
self.stream.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn httpRequest(self: *TestClient, req: []const u8) ![]const u8 {
|
||||||
|
try self.stream.writeAll(req);
|
||||||
|
|
||||||
|
var pos: usize = 0;
|
||||||
|
var total_length: ?usize = null;
|
||||||
|
while (true) {
|
||||||
|
pos += try self.stream.read(self.buf[pos..]);
|
||||||
|
const response = self.buf[0..pos];
|
||||||
|
if (total_length == null) {
|
||||||
|
const header_end = std.mem.indexOf(u8, response, "\r\n\r\n") orelse continue;
|
||||||
|
const header = response[0 .. header_end + 4];
|
||||||
|
|
||||||
|
const cl_header = "Content-Length: ";
|
||||||
|
const start = (std.mem.indexOf(u8, header, cl_header) orelse {
|
||||||
|
return error.MissingContentLength;
|
||||||
|
}) + cl_header.len;
|
||||||
|
|
||||||
|
const end = std.mem.indexOfScalarPos(u8, header, start, '\r') orelse {
|
||||||
|
return error.InvalidContentLength;
|
||||||
|
};
|
||||||
|
const cl = std.fmt.parseInt(usize, header[start..end], 10) catch {
|
||||||
|
return error.InvalidContentLength;
|
||||||
|
};
|
||||||
|
|
||||||
|
total_length = cl + header.len;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (total_length) |tl| {
|
||||||
|
if (pos == tl) {
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
if (pos > tl) {
|
||||||
|
return error.DataExceedsContentLength;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|||||||
@@ -18,10 +18,17 @@
|
|||||||
|
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
const builtin = @import("builtin");
|
const builtin = @import("builtin");
|
||||||
|
const parser = @import("netsurf");
|
||||||
|
|
||||||
const Allocator = std.mem.Allocator;
|
const Allocator = std.mem.Allocator;
|
||||||
|
|
||||||
|
const jsruntime = @import("jsruntime");
|
||||||
|
pub const Types = jsruntime.reflect(@import("generate.zig").Tuple(.{}){});
|
||||||
|
pub const UserContext = @import("user_context.zig").UserContext;
|
||||||
|
// pub const IO = @import("asyncio").Wrapper(jsruntime.Loop);
|
||||||
|
|
||||||
pub const std_options = std.Options{
|
pub const std_options = std.Options{
|
||||||
|
.log_level = .err,
|
||||||
.http_disable_tls = true,
|
.http_disable_tls = true,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -31,10 +38,14 @@ const BORDER = "=" ** 80;
|
|||||||
var current_test: ?[]const u8 = null;
|
var current_test: ?[]const u8 = null;
|
||||||
|
|
||||||
pub fn main() !void {
|
pub fn main() !void {
|
||||||
var mem: [8192]u8 = undefined;
|
try parser.init();
|
||||||
var fba = std.heap.FixedBufferAllocator.init(&mem);
|
defer parser.deinit();
|
||||||
|
|
||||||
const allocator = fba.allocator();
|
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
||||||
|
const allocator = gpa.allocator();
|
||||||
|
|
||||||
|
var loop = try jsruntime.Loop.init(allocator);
|
||||||
|
defer loop.deinit();
|
||||||
|
|
||||||
const env = Env.init(allocator);
|
const env = Env.init(allocator);
|
||||||
defer env.deinit(allocator);
|
defer env.deinit(allocator);
|
||||||
@@ -47,12 +58,20 @@ pub fn main() !void {
|
|||||||
var skip: usize = 0;
|
var skip: usize = 0;
|
||||||
var leak: usize = 0;
|
var leak: usize = 0;
|
||||||
|
|
||||||
const address = try std.net.Address.parseIp("127.0.0.1", 9582);
|
const http_thread = blk: {
|
||||||
var listener = try address.listen(.{ .reuse_address = true });
|
const address = try std.net.Address.parseIp("127.0.0.1", 9582);
|
||||||
defer listener.deinit();
|
const thread = try std.Thread.spawn(.{}, serveHTTP, .{address});
|
||||||
const http_thread = try std.Thread.spawn(.{}, serverHTTP, .{&listener});
|
break :blk thread;
|
||||||
|
};
|
||||||
defer http_thread.join();
|
defer http_thread.join();
|
||||||
|
|
||||||
|
const cdp_thread = blk: {
|
||||||
|
const address = try std.net.Address.parseIp("127.0.0.1", 9583);
|
||||||
|
const thread = try std.Thread.spawn(.{}, serveCDP, .{ allocator, address, &loop });
|
||||||
|
break :blk thread;
|
||||||
|
};
|
||||||
|
defer cdp_thread.join();
|
||||||
|
|
||||||
const printer = Printer.init();
|
const printer = Printer.init();
|
||||||
printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line
|
printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line
|
||||||
|
|
||||||
@@ -98,7 +117,9 @@ pub fn main() !void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (result) |_| {
|
if (result) |_| {
|
||||||
pass += 1;
|
if (is_unnamed_test == false) {
|
||||||
|
pass += 1;
|
||||||
|
}
|
||||||
} else |err| switch (err) {
|
} else |err| switch (err) {
|
||||||
error.SkipZigTest => {
|
error.SkipZigTest => {
|
||||||
skip += 1;
|
skip += 1;
|
||||||
@@ -117,11 +138,13 @@ pub fn main() !void {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
if (env.verbose) {
|
if (is_unnamed_test == false) {
|
||||||
const ms = @as(f64, @floatFromInt(ns_taken)) / 1_000_000.0;
|
if (env.verbose) {
|
||||||
printer.status(status, "{s} ({d:.2}ms)\n", .{ friendly_name, ms });
|
const ms = @as(f64, @floatFromInt(ns_taken)) / 1_000_000.0;
|
||||||
} else {
|
printer.status(status, "{s} ({d:.2}ms)\n", .{ friendly_name, ms });
|
||||||
printer.status(status, ".", .{});
|
} else {
|
||||||
|
printer.status(status, ".", .{});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,7 +317,10 @@ fn isUnnamed(t: std.builtin.TestFn) bool {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn serverHTTP(listener: *std.net.Server) !void {
|
fn serveHTTP(address: std.net.Address) !void {
|
||||||
|
var listener = try address.listen(.{ .reuse_address = true });
|
||||||
|
defer listener.deinit();
|
||||||
|
|
||||||
var read_buffer: [1024]u8 = undefined;
|
var read_buffer: [1024]u8 = undefined;
|
||||||
ACCEPT: while (true) {
|
ACCEPT: while (true) {
|
||||||
var conn = try listener.accept();
|
var conn = try listener.accept();
|
||||||
@@ -320,6 +346,14 @@ fn serverHTTP(listener: *std.net.Server) !void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn serveCDP(allocator: Allocator, address: std.net.Address, loop: *jsruntime.Loop) !void {
|
||||||
|
const server = @import("server.zig");
|
||||||
|
server.run(allocator, address, std.time.ns_per_s * 2, loop) catch |err| {
|
||||||
|
std.debug.print("CDP server error: {}", .{err});
|
||||||
|
return err;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const Response = struct {
|
const Response = struct {
|
||||||
body: []const u8 = "",
|
body: []const u8 = "",
|
||||||
status: std.http.Status = .ok,
|
status: std.http.Status = .ok,
|
||||||
|
|||||||
@@ -103,7 +103,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
ctx: ?Client.Ctx = null,
|
ctx: ?Client.Ctx = null,
|
||||||
|
|
||||||
method: std.http.Method,
|
method: std.http.Method,
|
||||||
state: u16,
|
state: State,
|
||||||
url: ?[]const u8,
|
url: ?[]const u8,
|
||||||
uri: std.Uri,
|
uri: std.Uri,
|
||||||
// request headers
|
// request headers
|
||||||
@@ -150,11 +150,13 @@ pub const XMLHttpRequest = struct {
|
|||||||
pub const prototype = *XMLHttpRequestEventTarget;
|
pub const prototype = *XMLHttpRequestEventTarget;
|
||||||
pub const mem_guarantied = true;
|
pub const mem_guarantied = true;
|
||||||
|
|
||||||
pub const UNSENT: u16 = 0;
|
const State = enum(u16) {
|
||||||
pub const OPENED: u16 = 1;
|
unsent = 0,
|
||||||
pub const HEADERS_RECEIVED: u16 = 2;
|
opened = 1,
|
||||||
pub const LOADING: u16 = 3;
|
headers_received = 2,
|
||||||
pub const DONE: u16 = 4;
|
loading = 3,
|
||||||
|
done = 4,
|
||||||
|
};
|
||||||
|
|
||||||
// https://xhr.spec.whatwg.org/#response-type
|
// https://xhr.spec.whatwg.org/#response-type
|
||||||
const ResponseType = enum {
|
const ResponseType = enum {
|
||||||
@@ -297,7 +299,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
.method = undefined,
|
.method = undefined,
|
||||||
.url = null,
|
.url = null,
|
||||||
.uri = undefined,
|
.uri = undefined,
|
||||||
.state = UNSENT,
|
.state = .unsent,
|
||||||
.cli = userctx.httpClient,
|
.cli = userctx.httpClient,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -347,7 +349,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_readyState(self: *XMLHttpRequest) u16 {
|
pub fn get_readyState(self: *XMLHttpRequest) u16 {
|
||||||
return self.state;
|
return @intFromEnum(self.state);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_timeout(_: *XMLHttpRequest) u32 {
|
pub fn get_timeout(_: *XMLHttpRequest) u32 {
|
||||||
@@ -367,7 +369,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_withCredentials(self: *XMLHttpRequest, withCredentials: bool) !void {
|
pub fn set_withCredentials(self: *XMLHttpRequest, withCredentials: bool) !void {
|
||||||
if (self.state != OPENED and self.state != UNSENT) return DOMError.InvalidState;
|
if (self.state != .opened and self.state != .unsent) return DOMError.InvalidState;
|
||||||
if (self.send_flag) return DOMError.InvalidState;
|
if (self.send_flag) return DOMError.InvalidState;
|
||||||
|
|
||||||
self.withCredentials = withCredentials;
|
self.withCredentials = withCredentials;
|
||||||
@@ -401,7 +403,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
log.debug("open url ({s})", .{self.url.?});
|
log.debug("open url ({s})", .{self.url.?});
|
||||||
self.sync = if (asyn) |b| !b else false;
|
self.sync = if (asyn) |b| !b else false;
|
||||||
|
|
||||||
self.state = OPENED;
|
self.state = .opened;
|
||||||
self.dispatchEvt("readystatechange");
|
self.dispatchEvt("readystatechange");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -477,14 +479,14 @@ pub const XMLHttpRequest = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn _setRequestHeader(self: *XMLHttpRequest, name: []const u8, value: []const u8) !void {
|
pub fn _setRequestHeader(self: *XMLHttpRequest, name: []const u8, value: []const u8) !void {
|
||||||
if (self.state != OPENED) return DOMError.InvalidState;
|
if (self.state != .opened) return DOMError.InvalidState;
|
||||||
if (self.send_flag) return DOMError.InvalidState;
|
if (self.send_flag) return DOMError.InvalidState;
|
||||||
return try self.headers.append(name, value);
|
return try self.headers.append(name, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO body can be either a XMLHttpRequestBodyInit or a document
|
// TODO body can be either a XMLHttpRequestBodyInit or a document
|
||||||
pub fn _send(self: *XMLHttpRequest, alloc: std.mem.Allocator, body: ?[]const u8) !void {
|
pub fn _send(self: *XMLHttpRequest, alloc: std.mem.Allocator, body: ?[]const u8) !void {
|
||||||
if (self.state != OPENED) return DOMError.InvalidState;
|
if (self.state != .opened) return DOMError.InvalidState;
|
||||||
if (self.send_flag) return DOMError.InvalidState;
|
if (self.send_flag) return DOMError.InvalidState;
|
||||||
|
|
||||||
// The body argument provides the request body, if any, and is ignored
|
// The body argument provides the request body, if any, and is ignored
|
||||||
@@ -554,7 +556,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
|
|
||||||
// TODO handle override mime type
|
// TODO handle override mime type
|
||||||
|
|
||||||
self.state = HEADERS_RECEIVED;
|
self.state = .headers_received;
|
||||||
self.dispatchEvt("readystatechange");
|
self.dispatchEvt("readystatechange");
|
||||||
|
|
||||||
self.response_status = @intFromEnum(self.req.?.response.status);
|
self.response_status = @intFromEnum(self.req.?.response.status);
|
||||||
@@ -592,7 +594,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
if (prev_dispatch != null and now.since(prev_dispatch.?) < min_delay) continue;
|
if (prev_dispatch != null and now.since(prev_dispatch.?) < min_delay) continue;
|
||||||
defer prev_dispatch = now;
|
defer prev_dispatch = now;
|
||||||
|
|
||||||
self.state = LOADING;
|
self.state = .loading;
|
||||||
self.dispatchEvt("readystatechange");
|
self.dispatchEvt("readystatechange");
|
||||||
|
|
||||||
// dispatch a progress event progress.
|
// dispatch a progress event progress.
|
||||||
@@ -604,7 +606,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
self.response_bytes = buf.items;
|
self.response_bytes = buf.items;
|
||||||
self.send_flag = false;
|
self.send_flag = false;
|
||||||
|
|
||||||
self.state = DONE;
|
self.state = .done;
|
||||||
self.dispatchEvt("readystatechange");
|
self.dispatchEvt("readystatechange");
|
||||||
|
|
||||||
// dispatch a progress event load.
|
// dispatch a progress event load.
|
||||||
@@ -666,7 +668,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
self.priv_state = .done;
|
self.priv_state = .done;
|
||||||
|
|
||||||
self.err = err;
|
self.err = err;
|
||||||
self.state = DONE;
|
self.state = .done;
|
||||||
self.send_flag = false;
|
self.send_flag = false;
|
||||||
self.dispatchEvt("readystatechange");
|
self.dispatchEvt("readystatechange");
|
||||||
self.dispatchProgressEvent("error", .{});
|
self.dispatchProgressEvent("error", .{});
|
||||||
@@ -697,7 +699,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn set_responseType(self: *XMLHttpRequest, rtype: []const u8) !void {
|
pub fn set_responseType(self: *XMLHttpRequest, rtype: []const u8) !void {
|
||||||
if (self.state == LOADING or self.state == DONE) return DOMError.InvalidState;
|
if (self.state == .loading or self.state == .done) return DOMError.InvalidState;
|
||||||
|
|
||||||
if (std.mem.eql(u8, rtype, "")) {
|
if (std.mem.eql(u8, rtype, "")) {
|
||||||
self.response_type = .Empty;
|
self.response_type = .Empty;
|
||||||
@@ -735,7 +737,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
return DOMError.InvalidState;
|
return DOMError.InvalidState;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (self.state != DONE) return null;
|
if (self.state != .done) return null;
|
||||||
|
|
||||||
// fastpath if response is previously parsed.
|
// fastpath if response is previously parsed.
|
||||||
if (self.response_obj) |obj| {
|
if (self.response_obj) |obj| {
|
||||||
@@ -761,7 +763,7 @@ pub const XMLHttpRequest = struct {
|
|||||||
// https://xhr.spec.whatwg.org/#the-response-attribute
|
// https://xhr.spec.whatwg.org/#the-response-attribute
|
||||||
pub fn get_response(self: *XMLHttpRequest, alloc: std.mem.Allocator) !?Response {
|
pub fn get_response(self: *XMLHttpRequest, alloc: std.mem.Allocator) !?Response {
|
||||||
if (self.response_type == .Empty or self.response_type == .Text) {
|
if (self.response_type == .Empty or self.response_type == .Text) {
|
||||||
if (self.state == LOADING or self.state == DONE) {
|
if (self.state == .loading or self.state == .done) {
|
||||||
return .{ .Text = try self.get_responseText() };
|
return .{ .Text = try self.get_responseText() };
|
||||||
}
|
}
|
||||||
return .{ .Text = "" };
|
return .{ .Text = "" };
|
||||||
|
|||||||
Reference in New Issue
Block a user