Merge pull request #199 from lightpanda-io/c_alloc

setCAllocator

.github/actions/install/action.yml

@@ -19,6 +19,10 @@ runs:
       run: |
         ln -s /usr/local/lib/libiconv vendor/libiconv
 
+    - name: build mimalloc
+      shell: bash
+      run: make install-mimalloc
+
     - name: build netsurf
       shell: bash
       run: make install-netsurf

.gitmodules

@@ -3,10 +3,10 @@
 	url = git@github.com:lightpanda-io/jsruntime-lib.git
 [submodule "vendor/netsurf/libwapcaplet"]
 	path = vendor/netsurf/libwapcaplet
-	url = https://source.netsurf-browser.org/libwapcaplet.git
+	url = git@github.com:lightpanda-io/libwapcaplet.git
 [submodule "vendor/netsurf/libparserutils"]
 	path = vendor/netsurf/libparserutils
-	url = https://source.netsurf-browser.org/libparserutils.git
+	url = git@github.com:lightpanda-io/libparserutils.git
 [submodule "vendor/netsurf/libdom"]
 	path = vendor/netsurf/libdom
 	url = git@github.com:lightpanda-io/libdom.git
@@ -15,7 +15,10 @@
 	url = https://source.netsurf-browser.org/buildsystem.git
 [submodule "vendor/netsurf/libhubbub"]
 	path = vendor/netsurf/libhubbub
-	url = https://source.netsurf-browser.org/libhubbub.git
+	url = git@github.com:lightpanda-io/libhubbub.git
 [submodule "tests/wpt"]
 	path = tests/wpt
 	url = https://github.com/lightpanda-io/wpt
+[submodule "vendor/mimalloc"]
+	path = vendor/mimalloc
+	url = git@github.com:microsoft/mimalloc.git

Makefile

@@ -95,13 +95,14 @@ test:
 .PHONY: install-submodule
 .PHONY: install-jsruntime install-jsruntime-dev install-libiconv
 .PHONY: _install-netsurf install-netsurf clean-netsurf test-netsurf install-netsurf-dev
+.PHONY: install-mimalloc install-mimalloc-dev clean-mimalloc
 .PHONY: install-dev install
 
 ## Install and build dependencies for release
-install: install-submodule install-jsruntime install-netsurf
+install: install-submodule install-jsruntime install-netsurf install-mimalloc
 
 ## Install and build dependencies for dev
-install-dev: install-submodule install-jsruntime-dev install-netsurf-dev
+install-dev: install-submodule install-jsruntime-dev install-netsurf-dev install-mimalloc-dev
 
 install-netsurf-dev: _install-netsurf
 install-netsurf-dev: OPTCFLAGS := -O0 -g -DNDEBUG
@@ -185,6 +186,26 @@ install-jsruntime:
 	@cd vendor/jsruntime-lib && \
 	make install
 
+.PHONY: _build_mimalloc
+_build_mimalloc:
+	@cd vendor/mimalloc && \
+		mkdir -p out/include && \
+		cp include/mimalloc.h out/include/ && \
+		cd out && \
+		cmake -DMI_BUILD_SHARED=OFF -DMI_BUILD_OBJECT=OFF -DMI_BUILD_TESTS=OFF -DMI_OVERRIDE=OFF $(OPTS) .. && \
+		make
+
+install-mimalloc-dev: _build_mimalloc
+install-mimalloc-dev: OPTS=-DCMAKE_BUILD_TYPE=Debug
+install-mimalloc-dev:
+	@cd vendor/mimalloc/out && \
+		mv libmimalloc-debug.a libmimalloc.a
+
+install-mimalloc: _build_mimalloc
+
+clean-mimalloc:
+	@rm -fr vendor/mimalloc/lib/*
+
 ## Init and update git submodule
 install-submodule:
 	@git submodule init && \

README.md

@@ -8,8 +8,9 @@ Browsercore is written with [Zig](https://ziglang.org/) `0.12`. You have to
 install it with the right version in order to build the project.
 
 Browsercore also depends on
-[js-runtimelib](https://github.com/francisbouvier/jsruntime-lib/) and
+[js-runtimelib](https://github.com/francisbouvier/jsruntime-lib/),
-[Netsurf libs](https://www.netsurf-browser.org/) libs.
+[Netsurf libs](https://www.netsurf-browser.org/) and
+[Mimalloc](https://microsoft.github.io/mimalloc) libs.
 
 To be able to build the v8 engine for js-runtimelib, you have to install some libs:
 
@@ -36,11 +37,26 @@ make install-submodule
 
 ### Build Netsurf
 
-The command `make install-netsurf` will build netsurf libs used by browsercore.
+The command `make install-netsurf` will build Netsurf libs used by browsercore.
 ```
 make install-netsurf
 ```
 
+For dev env, use `make install-netsurf-dev`.
+
+### Build Mimalloc
+
+The command `make install-mimalloc` will build Mimalloc lib used by browsercore.
+```
+make install-mimalloc
+```
+
+For dev env, use `make install-mimalloc-dev`.
+
+Note, when Mimalloc is built in dev mode, you can dump memory stats with the
+env var `MIMALLOC_SHOW_STATS=1`. See
+https://microsoft.github.io/mimalloc/environment.html
+
 ### Build jsruntime-lib
 
 The command `make install-jsruntime-dev` uses jsruntime-lib's `zig-v8` dependency to build v8 engine lib.

build.zig

@@ -152,6 +152,10 @@ fn common(
 ) !void {
     try jsruntime_pkgs.add(step, options);
     linkNetSurf(step);
+
+    // link mimalloc
+    step.addObjectFile(.{ .path = "vendor/mimalloc/out/libmimalloc.a" });
+    step.addIncludePath(.{ .path = "vendor/mimalloc/out/include" });
 }
 
 fn linkNetSurf(step: *std.build.LibExeObjStep) void {

src/browser/browser.zig

@@ -134,6 +134,9 @@ pub const Page = struct {
         self.session.env.stop();
         // TODO unload document: https://html.spec.whatwg.org/#unloading-documents
 
+        // clear netsurf memory arena.
+        parser.deinit();
+
         _ = self.arena.reset(.free_all);
     }
 
@@ -211,6 +214,9 @@ pub const Page = struct {
     fn loadHTMLDoc(self: *Page, reader: anytype, charset: []const u8) !void {
         const alloc = self.arena.allocator();
 
+        // start netsurf memory arena.
+        try parser.init();
+
         log.debug("parse html with charset {s}", .{charset});
 
         const ccharset = try alloc.dupeZ(u8, charset);

src/main.zig

@@ -56,6 +56,9 @@ pub fn main() !void {
     var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
     defer arena.deinit();
 
+    try parser.init();
+    defer parser.deinit();
+
     // document
     const file = try std.fs.cwd().openFile("test.html", .{});
     defer file.close();

src/main_get.zig

@@ -65,8 +65,10 @@ pub fn main() !void {
     defer browser.deinit();
 
     var page = try browser.currentSession().createPage();
-    defer page.end();
+    defer page.deinit();
+
     try page.navigate(url);
+    defer page.end();
 
     if (dump) {
         try page.dump(std.io.getStdOut());

src/main_shell.zig

@@ -37,6 +37,9 @@ pub fn main() !void {
     var arena = std.heap.ArenaAllocator.init(gpa.allocator());
     defer arena.deinit();
 
+    try parser.init();
+    defer parser.deinit();
+
     // document
     const file = try std.fs.cwd().openFile("test.html", .{});
     defer file.close();

src/mimalloc.zig

@@ -0,0 +1,58 @@
+// This file makes the glue between mimalloc heap allocation and libdom memory
+// management.
+// We replace the libdom default usage of allocations with mimalloc heap
+// allocation to be able to free all memory used at once, like an arena usage.
+
+const std = @import("std");
+const c = @cImport({
+    @cInclude("mimalloc.h");
+});
+
+const Error = error{
+    HeapNotNull,
+    HeapNull,
+};
+
+var heap: ?*c.mi_heap_t = null;
+
+pub fn create() Error!void {
+    if (heap != null) return Error.HeapNotNull;
+    heap = c.mi_heap_new();
+    if (heap == null) return Error.HeapNull;
+}
+
+pub fn destroy() void {
+    if (heap == null) return;
+    c.mi_heap_destroy(heap.?);
+    heap = null;
+}
+
+pub export fn m_alloc(size: usize) callconv(.C) ?*anyopaque {
+    if (heap == null) return null;
+    return c.mi_heap_malloc(heap.?, size);
+}
+
+pub export fn re_alloc(ptr: ?*anyopaque, size: usize) callconv(.C) ?*anyopaque {
+    if (heap == null) return null;
+    return c.mi_heap_realloc(heap.?, ptr, size);
+}
+
+pub export fn c_alloc(nmemb: usize, size: usize) callconv(.C) ?*anyopaque {
+    if (heap == null) return null;
+    return c.mi_heap_calloc(heap.?, nmemb, size);
+}
+
+pub export fn str_dup(s: [*c]const u8) callconv(.C) [*c]u8 {
+    if (heap == null) return null;
+    return c.mi_heap_strdup(heap.?, s);
+}
+
+pub export fn strn_dup(s: [*c]const u8, size: usize) callconv(.C) [*c]u8 {
+    if (heap == null) return null;
+    return c.mi_heap_strndup(heap.?, s, size);
+}
+
+// NOOP, use destroy to clear all the memory allocated at once.
+pub export fn f_ree(_: ?*anyopaque) callconv(.C) void {
+    return;
+}

src/netsurf.zig

@@ -8,9 +8,29 @@ const c = @cImport({
     @cInclude("events/event.h");
 });
 
+const mimalloc = @import("mimalloc.zig");
+
 const Callback = @import("jsruntime").Callback;
 const EventToInterface = @import("events/event.zig").Event.toInterface;
 
+// init initializes netsurf lib.
+// init starts a mimalloc heap arena for the netsurf session. The caller must
+// call deinit() to free the arena memory.
+pub fn init() !void {
+    try mimalloc.create();
+}
+
+// deinit frees the mimalloc heap arena memory.
+// It also clean dom namespaces and lwc strings.
+pub fn deinit() void {
+    _ = c.dom_namespace_finalise();
+
+    // destroy all lwc strings.
+    c.lwc_deinit_strings();
+
+    mimalloc.destroy();
+}
+
 // Vtable
 // ------
 

src/run_tests.zig

@@ -38,6 +38,9 @@ fn testExecFn(
     js_env: *jsruntime.Env,
     comptime execFn: jsruntime.ContextExecFn,
 ) anyerror!void {
+    try parser.init();
+    defer parser.deinit();
+
     // start JS env
     try js_env.start(alloc);
     defer js_env.stop();
@@ -155,6 +158,9 @@ pub fn main() !void {
     if (run == .all or run == .unit) {
         std.debug.print("\n", .{});
         for (builtin.test_functions) |test_fn| {
+            try parser.init();
+            defer parser.deinit();
+
             try test_fn.func();
             std.debug.print("{s}\tOK\n", .{test_fn.name});
         }

src/wpt/run.zig

@@ -17,6 +17,8 @@ const Types = @import("../main_wpt.zig").Types;
 // It loads first the js libs files.
 pub fn run(arena: *std.heap.ArenaAllocator, comptime dir: []const u8, f: []const u8, loader: *FileLoader) !jsruntime.JSResult {
     const alloc = arena.allocator();
+    try parser.init();
+    defer parser.deinit();
 
     // document
     const file = try std.fs.cwd().openFile(f, .{});