Merge pull request #1236 from lightpanda-io/v8-build-with-zig-gclient-ci

V8 build with zig gclient ci

.github/actions/install/action.yml

@@ -2,10 +2,6 @@ name: "Browsercore install"
 description: "Install deps for the project browsercore"
 
 inputs:
-  zig:
-    description: 'Zig version to install'
-    required: false
-    default: '0.15.2'
   arch:
     description: 'CPU arch used to select the v8 lib'
     required: false
@@ -26,10 +22,6 @@ inputs:
     description: 'cache dir to use'
     required: false
     default: '~/.cache'
-  mode:
-    description: 'debug or release'
-    required: false
-    default: 'debug'
 
 runs:
   using: "composite"
@@ -42,9 +34,8 @@ runs:
         sudo apt-get update
         sudo apt-get install -y wget xz-utils python3 ca-certificates git pkg-config libglib2.0-dev gperf libexpat1-dev cmake clang
 
-    - uses: mlugg/setup-zig@v2.0.5
+    # Zig version used from the `minimum_zig_version` field in build.zig.zon
-      with:
+    - uses: mlugg/setup-zig@v2
-        version: ${{ inputs.zig }}
 
     - name: Cache v8
       id: cache-v8
@@ -62,26 +53,34 @@ runs:
 
         wget -O ${{ inputs.cache-dir }}/v8/libc_v8.a https://github.com/lightpanda-io/zig-v8-fork/releases/download/${{ inputs.zig-v8 }}/libc_v8_${{ inputs.v8 }}_${{ inputs.os }}_${{ inputs.arch }}.a
 
-    - name: install v8 release
+    - name: install v8
-      if: ${{ inputs.mode == 'release' }}
       shell: bash
       run: |
-        mkdir -p v8/out/${{ inputs.os }}/release/obj/zig/
+        mkdir -p v8
-        ln -s ${{ inputs.cache-dir }}/v8/libc_v8.a v8/out/${{ inputs.os }}/release/obj/zig/libc_v8.a
+        ln -s ${{ inputs.cache-dir }}/v8/libc_v8.a v8/libc_v8.a
 
-    - name: install v8 debug
+    - name: Cache libiconv
-      if: ${{ inputs.mode == 'debug' }}
+      id: cache-libiconv
-      shell: bash
+      uses: actions/cache@v4
-      run: |
+      env:
-        mkdir -p v8/out/${{ inputs.os }}/debug/obj/zig/
+        cache-name: cache-libiconv
-        ln -s ${{ inputs.cache-dir }}/v8/libc_v8.a v8/out/${{ inputs.os }}/debug/obj/zig/libc_v8.a
+      with:
+        path: ${{ inputs.cache-dir }}/libiconv
+        key: vendor/libiconv/libiconv-1.17
 
-    - name: hmtl5ever release
+    - name: download libiconv
-      if: ${{ inputs.mode == 'release' }}
+      if: ${{ steps.cache-libiconv.outputs.cache-hit != 'true' }}
       shell: bash
-      run: zig build -Doptimize=ReleaseFast html5ever
+      run: make download-libiconv
 
-    - name: hmtl5ever debug
+    - name: build libiconv
-      if: ${{ inputs.mode == 'debug' }}
       shell: bash
-      run: zig build html5ever
+      run: make build-libiconv
+
+    - name: build mimalloc
+      shell: bash
+      run: make install-mimalloc
+
+    - name: build netsurf
+      shell: bash
+      run: make install-netsurf

.github/workflows/build.yml

@@ -39,7 +39,7 @@ jobs:
           mode: 'release'
 
       - name: zig build
-        run: zig build --release=safe -Doptimize=ReleaseSafe -Dcpu=x86_64 -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
+        run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseSafe -Dcpu=x86_64 -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
 
       - name: Rename binary
         run: mv zig-out/bin/lightpanda lightpanda-${{ env.ARCH }}-${{ env.OS }}
@@ -78,7 +78,7 @@ jobs:
           mode: 'release'
 
       - name: zig build
-        run: zig build --release=safe -Doptimize=ReleaseSafe -Dcpu=generic -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
+        run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseSafe -Dcpu=generic -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
 
       - name: Rename binary
         run: mv zig-out/bin/lightpanda lightpanda-${{ env.ARCH }}-${{ env.OS }}
@@ -119,7 +119,7 @@ jobs:
           mode: 'release'
 
       - name: zig build
-        run: zig build --release=safe -Doptimize=ReleaseSafe -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
+        run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseSafe -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
 
       - name: Rename binary
         run: mv zig-out/bin/lightpanda lightpanda-${{ env.ARCH }}-${{ env.OS }}
@@ -163,7 +163,7 @@ jobs:
           mode: 'release'
 
       - name: zig build
-        run: zig build --release=safe -Doptimize=ReleaseSafe -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
+        run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseSafe -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
 
       - name: Rename binary
         run: mv zig-out/bin/lightpanda lightpanda-${{ env.ARCH }}-${{ env.OS }}

.github/workflows/e2e-test.yml

@@ -60,7 +60,7 @@ jobs:
           mode: 'release'
 
       - name: zig build release
-        run: zig build -Doptimize=ReleaseFast -Dcpu=x86_64 -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
+        run: zig build -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64 -Dgit_commit=$(git rev-parse --short ${{ github.sha }})
 
       - name: upload artifact
         uses: actions/upload-artifact@v4

.github/workflows/zig-test.yml

@@ -56,7 +56,7 @@ jobs:
       - uses: ./.github/actions/install
 
       - name: zig build debug
-        run: zig build
+        run: zig build -Dprebuilt_v8_path=v8/libc_v8.a
 
       - name: upload artifact
         uses: actions/upload-artifact@v4
@@ -104,7 +104,7 @@ jobs:
       - uses: ./.github/actions/install
 
       - name: zig build test
-        run: zig build test -- --json > bench.json
+        run: zig build  -Dprebuilt_v8_path=v8/libc_v8.a test -- --json > bench.json
 
       - name: write commit
         run: |

.gitignore

@@ -1,5 +1,9 @@
+zig-cache
 /.zig-cache/
-/zig-out/
+/.lp-cache/
+zig-out
+/vendor/netsurf/out
+/vendor/libiconv/
 lightpanda.id
 /v8/
 /build/

Dockerfile

@@ -1,10 +1,9 @@
-FROM debian:stable
+FROM debian:stable-slim
 
 ARG MINISIG=0.12
-ARG ZIG=0.15.2
 ARG ZIG_MINISIG=RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U
 ARG V8=14.0.365.4
-ARG ZIG_V8=v0.1.35
+ARG ZIG_V8=v0.1.34
 ARG TARGETPLATFORM
 
 RUN apt-get update -yq && \
@@ -17,30 +16,32 @@ RUN apt-get update -yq && \
 
 # install minisig
 RUN curl --fail -L -O https://github.com/jedisct1/minisign/releases/download/${MINISIG}/minisign-${MINISIG}-linux.tar.gz && \
-    tar xvzf minisign-${MINISIG}-linux.tar.gz
+    tar xvzf minisign-${MINISIG}-linux.tar.gz -C /
+
+# clone lightpanda
+RUN git clone https://github.com/lightpanda-io/browser.git
+WORKDIR /browser
 
 # install zig
-RUN case $TARGETPLATFORM in \
+RUN ZIG=$(grep '\.minimum_zig_version = "' "build.zig.zon" | cut -d'"' -f2) && \
+    case $TARGETPLATFORM in \
       "linux/arm64") ARCH="aarch64" ;; \
       *) ARCH="x86_64" ;; \
     esac && \
     curl --fail -L -O https://ziglang.org/download/${ZIG}/zig-${ARCH}-linux-${ZIG}.tar.xz && \
     curl --fail -L -O https://ziglang.org/download/${ZIG}/zig-${ARCH}-linux-${ZIG}.tar.xz.minisig && \
-    minisign-linux/${ARCH}/minisign -Vm zig-${ARCH}-linux-${ZIG}.tar.xz -P ${ZIG_MINISIG} && \
+    /minisign-linux/${ARCH}/minisign -Vm zig-${ARCH}-linux-${ZIG}.tar.xz -P ${ZIG_MINISIG} && \
     tar xvf zig-${ARCH}-linux-${ZIG}.tar.xz && \
     mv zig-${ARCH}-linux-${ZIG} /usr/local/lib && \
     ln -s /usr/local/lib/zig-${ARCH}-linux-${ZIG}/zig /usr/local/bin/zig
 
-# clone lightpanda
-RUN git clone https://github.com/lightpanda-io/browser.git
-
-WORKDIR /browser
-
 # install deps
 RUN git submodule init && \
     git submodule update --recursive
 
-RUN zig build -Doptimize=ReleaseFast html5ever
+RUN make install-libiconv && \
+    make install-netsurf && \
+    make install-mimalloc
 
 # download and install v8
 RUN case $TARGETPLATFORM in \
@@ -48,11 +49,16 @@ RUN case $TARGETPLATFORM in \
     *) ARCH="x86_64" ;; \
     esac && \
     curl --fail -L -o libc_v8.a https://github.com/lightpanda-io/zig-v8-fork/releases/download/${ZIG_V8}/libc_v8_${V8}_linux_${ARCH}.a && \
-    mkdir -p v8/out/linux/release/obj/zig/ && \
+    mkdir -p v8/ && \
-    mv libc_v8.a v8/out/linux/release/obj/zig/libc_v8.a
+    mv libc_v8.a v8/libc_v8.a
 
 # build release
-RUN make build
+RUN zig build -Doptimize=ReleaseSafe -Dprebuilt_v8_path=v8/libc_v8.a -Dgit_commit=$$(git rev-parse --short HEAD)
+
+FROM debian:stable-slim
+
+RUN apt-get update -yq && \
+    apt-get install -yq tini
 
 FROM debian:stable-slim
 
@@ -60,7 +66,12 @@ FROM debian:stable-slim
 COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 
 COPY --from=0 /browser/zig-out/bin/lightpanda /bin/lightpanda
+COPY --from=1 /usr/bin/tini /usr/bin/tini
 
 EXPOSE 9222/tcp
 
-CMD ["/bin/lightpanda", "serve", "--host", "0.0.0.0", "--port", "9222"]
+# Lightpanda install only some signal handlers, and PID 1 doesn't have a default SIGTERM signal handler.
+# Using "tini" as PID1 ensures that signals work as expected, so e.g. "docker stop" will not hang.
+# (See https://github.com/krallin/tini#why-tini).
+ENTRYPOINT ["/usr/bin/tini", "--"]
+CMD ["/bin/lightpanda", "serve", "--host", "0.0.0.0", "--port", "9222", "--log_level", "info"]

Makefile

@@ -47,7 +47,7 @@ help:
 
 # $(ZIG) commands
 # ------------
-.PHONY: build build-dev run run-release shell test bench download-zig wpt data get-v8 build-v8 build-v8-dev
+.PHONY: build build-dev run run-release shell test bench download-zig wpt data
 .PHONY: end2end
 
 zig_version = $(shell grep 'recommended_zig_version = "' "vendor/zig-js-runtime/build.zig" | cut -d'"' -f2)
@@ -112,19 +112,6 @@ end2end:
 	@test -d ../demo
 	cd ../demo && go run runner/main.go
 
-## v8
-get-v8:
-	@printf "\e[36mGetting v8 source...\e[0m\n"
-	@$(ZIG) build get-v8
-
-build-v8-dev:
-	@printf "\e[36mBuilding v8 (dev)...\e[0m\n"
-	@$(ZIG) build build-v8
-
-build-v8:
-	@printf "\e[36mBuilding v8...\e[0m\n"
-	@$(ZIG) build -Doptimize=ReleaseSafe build-v8
-
 # Install and build required dependencies commands
 # ------------
 .PHONY: install install-dev

README.md

@@ -225,22 +225,6 @@ zig build html5ever
 
 For a release build, use `zig build -Doptimize=ReleaseFast html5ever`.
 
-**v8**
-
-First, get the tools necessary for building V8, as well as the V8 source code:
-
-```
-make get-v8
-```
-
-Next, build v8. This build task is very long and cpu consuming, as you will build v8 from sources.
-
-```
-make build-v8
-```
-
-For dev env, use `make build-v8-dev`.
-
 ## Test
 
 ### Unit Tests

build.zig

@@ -21,33 +21,18 @@ const builtin = @import("builtin");
 
 const Build = std.Build;
 
-/// Do not rename this constant. It is scanned by some scripts to determine
-/// which zig version to install.
-const recommended_zig_version = "0.15.2";
-
 pub fn build(b: *Build) !void {
-    switch (comptime builtin.zig_version.order(std.SemanticVersion.parse(recommended_zig_version) catch unreachable)) {
-        .eq => {},
-        .lt => {
-            @compileError("The minimum version of Zig required to compile is '" ++ recommended_zig_version ++ "', found '" ++ builtin.zig_version_string ++ "'.");
-        },
-        .gt => {
-            std.debug.print(
-                "WARNING: Recommended Zig version '{s}', but found '{s}', build may fail...\n\n",
-                .{ recommended_zig_version, builtin.zig_version_string },
-            );
-        },
-    }
-
     const target = b.standardTargetOptions(.{});
     const optimize = b.standardOptimizeOption(.{});
 
+    const manifest = Manifest.init(b);
+
+    const git_commit = b.option([]const u8, "git_commit", "Current git commit");
+    const prebuilt_v8_path = b.option([]const u8, "prebuilt_v8_path", "Path to prebuilt libc_v8.a");
+
     var opts = b.addOptions();
-    opts.addOption(
+    opts.addOption([]const u8, "version", manifest.version);
-        []const u8,
+    opts.addOption([]const u8, "git_commit", git_commit orelse "dev");
-        "git_commit",
-        b.option([]const u8, "git_commit", "Current git commit") orelse "dev",
-    );
 
     // Build step to install html5ever dependency.
     const html5ever_argv = blk: {
@@ -88,7 +73,7 @@ pub fn build(b: *Build) !void {
             .sanitize_thread = enable_tsan,
         });
 
-        try addDependencies(b, mod, opts);
+        try addDependencies(b, mod, opts, prebuilt_v8_path);
 
         break :blk mod;
     };
@@ -191,33 +176,17 @@ pub fn build(b: *Build) !void {
         const run_step = b.step("wpt", "Run WPT tests");
         run_step.dependOn(&run_cmd.step);
     }
-
-    {
-        // get v8
-        // -------
-        const v8 = b.dependency("v8", .{ .target = target, .optimize = optimize });
-        const get_v8 = b.addRunArtifact(v8.artifact("get-v8"));
-        const get_step = b.step("get-v8", "Get v8");
-        get_step.dependOn(&get_v8.step);
-    }
-
-    {
-        // build v8
-        // -------
-        const v8 = b.dependency("v8", .{ .target = target, .optimize = optimize });
-        const build_v8 = b.addRunArtifact(v8.artifact("build-v8"));
-        const build_step = b.step("build-v8", "Build v8");
-        build_step.dependOn(&build_v8.step);
-    }
 }
 
-fn addDependencies(b: *Build, mod: *Build.Module, opts: *Build.Step.Options) !void {
+fn addDependencies(b: *Build, mod: *Build.Module, opts: *Build.Step.Options, prebuilt_v8_path: ?[]const u8) !void {
     mod.addImport("build_config", opts.createModule());
 
     const target = mod.resolved_target.?;
     const dep_opts = .{
         .target = target,
         .optimize = mod.optimize.?,
+        .prebuilt_v8_path = prebuilt_v8_path,
+        .cache_root = b.pathFromRoot(".lp-cache"),
     };
 
     mod.addIncludePath(b.path("vendor/lightpanda"));
@@ -230,36 +199,6 @@ fn addDependencies(b: *Build, mod: *Build.Module, opts: *Build.Step.Options) !vo
         const v8_mod = b.dependency("v8", dep_opts).module("v8");
         v8_mod.addOptions("default_exports", v8_opts);
         mod.addImport("v8", v8_mod);
-
-        const release_dir = if (mod.optimize.? == .Debug) "debug" else "release";
-        const os = switch (target.result.os.tag) {
-            .linux => "linux",
-            .macos => "macos",
-            else => return error.UnsupportedPlatform,
-        };
-        var lib_path = try std.fmt.allocPrint(
-            mod.owner.allocator,
-            "v8/out/{s}/{s}/obj/zig/libc_v8.a",
-            .{ os, release_dir },
-        );
-        std.fs.cwd().access(lib_path, .{}) catch {
-            // legacy path
-            lib_path = try std.fmt.allocPrint(
-                mod.owner.allocator,
-                "v8/out/{s}/obj/zig/libc_v8.a",
-                .{release_dir},
-            );
-        };
-        mod.addObjectFile(mod.owner.path(lib_path));
-
-        switch (target.result.os.tag) {
-            .macos => {
-                // v8 has a dependency, abseil-cpp, which, on Mac, uses CoreFoundation
-                mod.addSystemFrameworkPath(.{ .cwd_relative = "/System/Library/Frameworks" });
-                mod.linkFramework("CoreFoundation", .{});
-            },
-            else => {},
-        }
     }
 
     {
@@ -747,3 +686,28 @@ fn buildCurl(b: *Build, m: *Build.Module) !void {
         },
     });
 }
+
+const Manifest = struct {
+    version: []const u8,
+    minimum_zig_version: []const u8,
+
+    fn init(b: *std.Build) Manifest {
+        const input = @embedFile("build.zig.zon");
+
+        var diagnostics: std.zon.parse.Diagnostics = .{};
+        defer diagnostics.deinit(b.allocator);
+
+        return std.zon.parse.fromSlice(Manifest, b.allocator, input, &diagnostics, .{
+            .free_on_error = true,
+            .ignore_unknown_fields = true,
+        }) catch |err| {
+            switch (err) {
+                error.OutOfMemory => @panic("OOM"),
+                error.ParseZon => {
+                    std.debug.print("Parse diagnostics:\n{f}\n", .{diagnostics});
+                    std.process.exit(1);
+                },
+            }
+        };
+    }
+};

build.zig.zon

@@ -3,10 +3,11 @@
     .paths = .{""},
     .version = "0.0.0",
     .fingerprint = 0xda130f3af836cea0,
+    .minimum_zig_version = "0.15.2",
     .dependencies = .{
         .v8 = .{
-            .url = "https://github.com/lightpanda-io/zig-v8-fork/archive/0d19781ccec829640e4f07591cbc166fa7dbe139.tar.gz",
+            .url = "https://github.com/lightpanda-io/zig-v8-fork/archive/e047d2a4d5af5783763f0f6a652fab8982a08603.tar.gz",
-            .hash = "v8-0.0.0-xddH6wTgAwALFCYoZbUIqtsRyP6mr69N7aKT_cySHKN2",
+            .hash = "v8-0.0.0-xddH65gMBACRBQMM7EwmVgfi94FJyyX-0jpe5KhXYhfv",
         },
         //.v8 = .{ .path = "../zig-v8-fork" }
         .@"boringssl-zig" = .{