fuzzer and performance optimisations

tools/fuzz.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""build and run the libFuzzer harness for the .rc config parser.
+
+usage:
+    python3 tools/fuzz.py [--seconds N] [--build-only] [--corpus DIR]
+                          [--dict PATH|--no-dict] [--no-detect-leaks]
+                          [--quarantine-mb N]
+
+defaults to 60 seconds of fuzzing against a small seed corpus. Seed corpus
+files land in build/fuzz-corpus (recreated only when empty so you can add your
+own); crashing inputs are saved under build/fuzz- and reported at the end.
+Leak detection is on by default (tools/lsan.supp silences a torsocks
+LD_PRELOAD false positive on the dev host); --no-detect-leaks disables it.
+ASan's freed-memory quarantine (the fuzz peak-RSS driver) defaults to 64MiB;
+override with --quarantine-mb.
+
+requires clang++ (libFuzzer's -fsanitize=fuzzer is a clang feature).
+"""
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+TESTS = ROOT / "tests"
+BUILD = ROOT / "build"
+OUT = BUILD / "fuzz_config"
+CORPUS = BUILD / "fuzz-corpus"
+DICT = BUILD / "fuzz.dict"
+
+# restart wildcard entries are not allowed in a dict; keep tokens plain ASCII.
+SEED_DICT = r"""# bajia rc grammar: libFuzzer dictionary (-dict), regenerated only if missing
+"service"
+"on"
+"@import"
+"oneshot"
+"disabled"
+"console"
+"user"
+"group"
+"class"
+"respawn"
+"crash-threshold"
+"crash-window"
+"setenv"
+"cwd"
+"seclabel"
+"="
+"start"
+"stop"
+"restart"
+"exec"
+"mkdir"
+"chmod"
+"chown"
+"write"
+"symlink"
+"mount"
+"log"
+"early-init"
+"init"
+"boot"
+"shutdown"
+"always"
+"never"
+"on-failure"
+"root"
+"nobody"
+"daemon"
+"/bin/true"
+"/bin/false"
+"/bin/sh"
+"/usr/sbin/sshd"
+"/etc/bajia/init.rc"
+"/proc"
+"/sys"
+"/dev"
+"/tmp"
+"0"
+"0755"
+"#"
+";"
+# __bajia_seed_dict__
+"""
+
+SEEDS = {
+    "tiny.rc": b"on boot\n    log hi\n",
+    "service.rc": b"service sshd /usr/sbin/sshd -D\n    user = root\n    respawn = always\n    console\n",
+    "alloptions.rc": (
+        b"service s /bin/true\n"
+        b"    user = nobody\n    group = nobody daemon\n"
+        b"    class = main\n    oneshot\n    disabled\n    console\n"
+        b"    respawn = never\n    crash-threshold = 4\n    crash-window = 30\n"
+        b"    setenv = FOO=bar\n    cwd = /tmp\n    seclabel = system_u:object_r:root_t:s0\n"
+    ),
+    "quotes.rc": b"write /tmp/x \"a b\\\"c\"\non boot\n    exec /bin/sh -c \"echo quoted\"\n",
+    "import.rc": b"@import /etc/bajia/init.rc\non init\n    log imported\n",
+    "nested.rc": (
+        b"on early-init\n    mount proc /proc proc\n    mount sysfs /sys sysfs\n"
+        b"on init\n    write /proc/sys/kernel/hostname bajia\n"
+        b"on boot\n    start console-serial\n"
+    ),
+}
+
+def find_clang() -> str | None:
+    explicit = os.environ.get("BAJIA_FUZZ_CXX")
+    if explicit and shutil.which(explicit):
+        return explicit
+    if shutil.which("clang++"):
+        return "clang++"
+    return None
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--seconds", type=int, default=60, help="fuzz time budget")
+    ap.add_argument("--corpus", type=Path,
+                    help="corpus dir (default: build/fuzz-corpus)")
+    ap.add_argument("--dict", type=Path,
+                    help="custom fuzzer dictionary (default: build/fuzz.dict, "
+                         "seeded from the rc grammar on first use)")
+    ap.add_argument("--no-dict", action="store_true",
+                    help="run without a dictionary")
+    ap.add_argument("--build-only", action="store_true",
+                    help="build the harness and stop")
+    ap.add_argument("--detect-leaks", action="store_true",
+                    help=argparse.SUPPRESS)  # historical no-op; already default
+    ap.add_argument("--no-detect-leaks", action="store_true",
+                    help="disable leak detection (default on: exits nonzero on "
+                         "leaks; tools/lsan.supp silences a torsocks "
+                         "LD_PRELOAD false positive on the dev host)")
+    ap.add_argument("--quarantine-mb", type=int, default=64, metavar="N",
+                    help="ASan freed-memory quarantine in MiB (default 64; "
+                         "ASan's own default of 256 dominates fuzz peak RSS. "
+                         "0 disables quarantine, which recycles freed memory "
+                         "immediately but weakens use-after-free detection)")
+    args = ap.parse_args()
+
+    clang = find_clang()
+    if not clang:
+        sys.exit("clang++ not found: libFuzzer (-fsanitize=fuzzer) requires clang")
+
+    BUILD.mkdir(exist_ok=True)
+    subprocess.run([
+        clang, "-std=c++20", "-O1", "-g",
+        "-fsanitize=fuzzer,address,undefined", "-fno-omit-frame-pointer",
+        str(TESTS / "fuzz_config.cpp"), "-o", str(OUT),
+    ], check=True)
+    print("built:", OUT)
+    if args.build_only:
+        return 0
+
+    corpus = args.corpus or CORPUS
+    if corpus.is_dir() and any(corpus.iterdir()):
+        print("using existing corpus:", corpus)
+    else:
+        corpus.mkdir(parents=True, exist_ok=True)
+        for name, blob in SEEDS.items():
+            (corpus / name).write_bytes(blob)
+        print("seeded corpus:", corpus)
+
+    prefix = BUILD / "fuzz-"
+    prefix.mkdir(exist_ok=True)
+    flags = [f"-artifact_prefix={prefix}/", "-print_final_stats=1"]
+    if not args.no_dict:
+        if args.dict:
+            dict_path = args.dict
+        else:
+            if not DICT.is_file() or "__bajia_seed_dict__" not in DICT.read_text(errors="ignore"):
+                DICT.write_text(SEED_DICT)  # (re)seed a missing or stale copy
+            dict_path = DICT
+        if not dict_path.is_file():
+            sys.exit(f"--dict: file not found: {dict_path}")
+        flags.append(f"-dict={dict_path}")
+        print("dictionary:", dict_path)
+    env = dict(os.environ)
+    asan = f"quarantine_size_mb={args.quarantine_mb}"
+    if args.no_detect_leaks:
+        flags.append("-detect_leaks=0")
+        asan = "detect_leaks=0:" + asan
+    else:
+        asan = "detect_leaks=1:" + asan
+        supp = ROOT / "tools" / "lsan.supp"
+        env["LSAN_OPTIONS"] = f"suppressions={supp}" + (":" + env["LSAN_OPTIONS"]
+                                                        if env.get("LSAN_OPTIONS") else "")
+        print("leak detection on (suppression:", supp, ")")
+    env["ASAN_OPTIONS"] = asan
+    cmd = [str(OUT), f"-max_total_time={args.seconds}"] + flags + [str(corpus)]
+    print("$", " ".join(cmd))
+    r = subprocess.run(cmd, env=env)
+
+    crashes = sorted(glob.glob(str(prefix) + "crash-*"))
+    leaks = sorted(glob.glob(str(prefix) + "leak-*"))
+    timeouts = sorted(glob.glob(str(prefix) + "timeout-*"))
+    if crashes or leaks or timeouts:
+        for art in crashes + leaks + timeouts:
+            print("artifact:", art)
+        print("!! fuzzer found problems (see artifact files above)")
+        return 1
+    print("no crashes in", args.seconds, "seconds")
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/lsan.supp

@@ -0,0 +1,2 @@
+# LeakSanitizer suppressions for bajia dev/test tooling.
+leak:tsocks_once

tools/run_vm.py

@@ -273,7 +273,8 @@ on boot
 """
 
 def build_initramfs(init: Path, busybox: Path, rc_text: str, root_password: str | None,
-                    selinux: bool, keep: bool) -> Path:
+                    selinux: bool, keep: bool,
+                    bundles: list[tuple[str, Path]] | None = None) -> Path:
     if not shutil.which("cpio"):
         sys.exit("cpio not found (install cpio)")
     root = Path(tempfile.mkdtemp(prefix="bajia-root-"))
@@ -320,6 +321,12 @@ def build_initramfs(init: Path, busybox: Path, rc_text: str, root_password: str
             "nobody:x:65534:\n"
             "daemon:x:1:\n")
 
+        for rel, src in (bundles or []):
+            dst = root / rel.lstrip("/")
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(src, dst)
+            dst.chmod(0o644)
+
         # The staging tree is owned by the host user and mkdtemp makes the
         # top dir 0700; GNU cpio preserves both, so without this the guest's
         # "/" would be mode 0700 owned by uid 1000 -- fine for root services,
@@ -380,6 +387,11 @@ def main() -> int:
                          "build it static, cache in ~/.cache/bajia")
     ap.add_argument("--config", type=Path,
                     help="use this init.rc instead of the bundled test config")
+    ap.add_argument("--bundle", action="append", default=[],
+                    metavar="REL=HOSTPATH",
+                    help="copy HOSTPATH into the initramfs at absolute REL "
+                         "(repeatable; e.g. --bundle "
+                         "etc/bajia/extra.rc=/tmp/extra.rc for @import tests)")
     ap.add_argument("--root-password", default=None,
                     help="password for the root account in the guest "
                          "(default: passwordless login)")
@@ -435,8 +447,20 @@ def main() -> int:
               "safest inside an initramfs")
 
     rc_text = args.config.read_text() if args.config else DEFAULT_RC
+    bundles: list[tuple[str, Path]] = []
+    for spec in args.bundle:
+        rel, _, path = spec.partition("=")
+        src = Path(path)
+        if not src.is_file():
+            sys.exit(f"--bundle: host file not found: {src}")
+        if not rel.startswith("/"):
+            sys.exit(f"--bundle: REL must be absolute, got: {rel!r}")
+        if ".." in [c for c in Path(rel).parts]:
+            sys.exit(f"--bundle: REL must not contain '..': {rel}")
+        bundles.append((rel, src))
     initrd = build_initramfs(init, busybox, rc_text, args.root_password,
-                             selinux=args.selinux, keep=args.keep_initramfs)
+                             selinux=args.selinux, keep=args.keep_initramfs,
+                             bundles=bundles)
     print("initramfs:", initrd, f"({initrd.stat().st_size / 1024:.0f} KB)")
 
     cmd = qemu_command(kernel, initrd, args)