From 48e39e44dbe267f8945e9d597e61fd8aa3dfb376 Mon Sep 17 00:00:00 2001 From: Tor Andersson Date: Mon, 28 Apr 2025 22:09:29 +0200 Subject: Improved fuzzing. --- bin/rtt-fuzz | 13 +- bin/rtt-fuzz-rand | 32 ++++ bin/rtt-fuzz-rand-n | 12 ++ bin/rtt-help | 1 + bin/rtt-import | 3 +- bin/rtt-patch | 10 +- bin/rtt-run | 2 +- bin/rtt-show-game | 2 +- bin/rtt-tm-unban-tick | 4 +- bin/rtt-update-elo | 1 + docs/module/fuzzer.md | 30 ++-- docs/module/rules.md | 5 + docs/overview/architecture.md | 14 +- docs/server/production.md | 6 +- docs/server/toolbox.md | 5 + package.json | 3 - tools/fuzz.js | 344 +++++++++++++++++++++++------------------- 17 files changed, 290 insertions(+), 197 deletions(-) create mode 100755 bin/rtt-fuzz-rand create mode 100755 bin/rtt-fuzz-rand-n diff --git a/bin/rtt-fuzz b/bin/rtt-fuzz index d7f2aef..464ff34 100755 --- a/bin/rtt-fuzz +++ b/bin/rtt-fuzz @@ -1,6 +1,6 @@ #!/bin/bash -TITLE=$1 +export TITLE=$1 shift if [ ! -f ./public/$TITLE/rules.js ] @@ -9,8 +9,11 @@ then exit 1 fi -mkdir -p fuzzer/corpus-$TITLE +if [ -z $(npm ls -p jsfuzz) ] +then + echo Installing "jsfuzz" package. + npm install -s --no-save jsfuzz +fi -RULES=../public/$TITLE/rules.js \ - npx jazzer tools/fuzz.js --sync fuzzer/corpus-$TITLE "$@" -- -exact_artifact_path=/dev/null | \ - tee fuzzer/log-$TITLE.txt +mkdir -p fuzzer/corpus-$TITLE +npx jsfuzz tools/fuzz.js fuzzer/corpus-$TITLE --exact-artifact-path=/dev/null | tee fuzzer/log-$TITLE.txt diff --git a/bin/rtt-fuzz-rand b/bin/rtt-fuzz-rand new file mode 100755 index 0000000..38878c3 --- /dev/null +++ b/bin/rtt-fuzz-rand @@ -0,0 +1,32 @@ +#!/usr/bin/env -S node + +"use strict" + +const fs = require("fs") +const crypto = require("crypto") + +if (process.argv.length < 3) { + console.error("rtt-fuzz-rand TITLE") + process.exit(1) +} + +process.env.TITLE = process.argv[2] + +const { fuzz } = require("../tools/fuzz.js") + +fs.mkdir("fuzzer", ()=>{}) + +if (process.argv.length > 3) { + fuzz(parseInt(process.argv[3])) +} else { + // run for an hour-ish + var i, n, a, b + for (i = 0; i < 3600; ++i) { + a = b = Date.now() + for (n = 0; b < a + 5_000; ++n) { + fuzz(crypto.randomInt(1, 2**48)) + b = Date.now() + } + console.log("# " + Math.round( (1000 * n) / (b-a) ) + " runs/second") + } +} diff --git a/bin/rtt-fuzz-rand-n b/bin/rtt-fuzz-rand-n new file mode 100755 index 0000000..7d3c969 --- /dev/null +++ b/bin/rtt-fuzz-rand-n @@ -0,0 +1,12 @@ +#!/bin/bash + +# count number of actual cores +CORES=$(lscpu --all --parse=SOCKET,CORE | grep -v '^#' | sort -u | wc -l) +# CORES=$(nproc) + +for P in $(seq $CORES) +do + ./bin/rtt-fuzz-rand $1 & +done + +wait $(jobs -p) diff --git a/bin/rtt-help b/bin/rtt-help index 5f6815c..57e42d1 100755 --- a/bin/rtt-help +++ b/bin/rtt-help @@ -16,6 +16,7 @@ game management module development foreach -- run a command for each module fuzz -- fuzz test a module + fuzz-rand -- fuzz test a module (random) game debugging show-chat -- show game chat (for moderation) diff --git a/bin/rtt-import b/bin/rtt-import index 2e4295b..53be292 100755 --- a/bin/rtt-import +++ b/bin/rtt-import @@ -1,4 +1,5 @@ #!/usr/bin/env -S node +"use strict" const fs = require("fs") const sqlite3 = require("better-sqlite3") @@ -30,7 +31,7 @@ for (let file of input) { game.setup.notice = options.notice if (game.setup.notice === undefined) - game.setup.notice = "" + game.setup.notice = file if (game.setup.options === undefined) game.setup.options = "{}" diff --git a/bin/rtt-patch b/bin/rtt-patch index ec0b795..2abb431 100755 --- a/bin/rtt-patch +++ b/bin/rtt-patch @@ -1,5 +1,7 @@ #!/usr/bin/env -S node +"use strict" + const sqlite3 = require("better-sqlite3") let db = new sqlite3("db") @@ -7,7 +9,7 @@ let db = new sqlite3("db") let select_game = db.prepare("select * from games where game_id=?") let select_replay = db.prepare("select * from game_replay where game_id=?") -let delete_replay = db.prepare("delete from game_replay where game_id=? and replay_id>?") +let delete_replay = db.prepare("delete from game_replay where game_id=?") let insert_replay = db.prepare("insert into game_replay (game_id,replay_id,role,action,arguments) values (?,?,?,?,?)") let select_last_snap = db.prepare("select max(snap_id) from game_snap where game_id=?").pluck() @@ -260,8 +262,8 @@ function patch_game(game_id, {validate_actions=true, save_snaps=true, delete_und db.exec("begin") if (need_to_rewrite) { - delete_replay.run(game_id, skip_replay_id) - for (item of replay) + delete_replay.run(game_id) + for (let item of replay) if (!item.remove) insert_replay.run(game_id, item.replay_id, item.role, item.action, item.arguments) } @@ -269,7 +271,7 @@ function patch_game(game_id, {validate_actions=true, save_snaps=true, delete_und if (save_snaps) { delete_snap.run(game_id, start_snap_id) let snap_id = start_snap_id - for (item of replay) + for (let item of replay) if (item.save) insert_snap.run(game_id, ++snap_id, item.replay_id, item.state) } diff --git a/bin/rtt-run b/bin/rtt-run index 6873e2f..b0f9b3e 100755 --- a/bin/rtt-run +++ b/bin/rtt-run @@ -1,7 +1,7 @@ #!/bin/bash while true do - nodemon --exitcrash server.js + npx nodemon --exitcrash server.js echo echo "Restarting soon!" echo "Hit Ctl-C to exit." diff --git a/bin/rtt-show-game b/bin/rtt-show-game index ece9e49..f055449 100755 --- a/bin/rtt-show-game +++ b/bin/rtt-show-game @@ -3,5 +3,5 @@ if [ -n "$1" ] then sqlite3 db "select json_remove(json_remove(state, '$.undo'), '$.log') from game_state where game_id = $1" else - echo "usage: rtt-show-state GAME" + echo "usage: rtt-show-game GAME" fi diff --git a/bin/rtt-tm-unban-tick b/bin/rtt-tm-unban-tick index 7294e81..d98a8d8 100755 --- a/bin/rtt-tm-unban-tick +++ b/bin/rtt-tm-unban-tick @@ -12,7 +12,7 @@ create temporary view tm_lift_ban_view as select user_id, name, - date(timeout_last), + date(timeout_last) as timeout_date, timeout_total, games_since_timeout, (games_since_timeout > timeout_total) and (julianday() > julianday(timeout_last)+14) as lift_ban @@ -23,7 +23,7 @@ create temporary view tm_lift_ban_view as order by lift_ban desc, timeout_last asc ; -select * from tm_lift_ban_view; +select name, timeout_date, timeout_total, games_since_timeout from tm_lift_ban_view where lift_ban; delete from tm_banned where user_id in (select user_id from tm_lift_ban_view where lift_ban) returning user_id; diff --git a/bin/rtt-update-elo b/bin/rtt-update-elo index 538964a..9e04224 100755 --- a/bin/rtt-update-elo +++ b/bin/rtt-update-elo @@ -1,4 +1,5 @@ #!/usr/bin/env -S node +"use strict" // Recompute Elo ratings from scratch! diff --git a/docs/module/fuzzer.md b/docs/module/fuzzer.md index d69b992..d576693 100644 --- a/docs/module/fuzzer.md +++ b/docs/module/fuzzer.md @@ -1,9 +1,4 @@ -# Fuzzing the Troops! - -We use [Jazzer.js](https://github.com/CodeIntelligenceTesting/jazzer.js/) -as a coverage-guided fuzzer for automatic testing of module rules. - -## What is fuzzing? +# Fuzz the Troops! Fuzzing or fuzz testing is an automated software testing technique that involves providing invalid, unexpected, or random data as inputs to a computer @@ -16,35 +11,34 @@ The fuzzer can detect the following types of errors: * Dead-end game states where no other actions are available (besides "undo"). * A game taking an excessive number of steps. This could indicate infinite loops and other logical flaws in the rules. -Work files are written to the "fuzzer" directory. +Crash dumps are written to the "fuzzer" directory. ## Running -Start the fuzzer: - - bash tools/fuzz.sh title [ jazzer options... ] +There are two fuzzers available: -This will run jazzer until you stop it or it has found too many errors. +A fuzzer that uses the "jsfuzz" package. +With this fuzzer every title gets its own "fuzzer/corpus-title" sub-directory. +The corpus helps the fuzzer find interesting game states in future runs. -To keep an eye on the crashes, you can watch the fuzzer/log-title.txt file: + rtt fuzz TITLE - tail -f fuzzer/log-title.txt +A simple fuzzer that plays completely randomly: -Each fuzzed title gets its own "fuzzer/corpus-title" sub-directory. -The corpus helps the fuzzer find interesting game states in future runs. + rtt fuzz-rand TITLE -To create a code coverage report pass the `--cov` option to fuzz.sh. +The fuzzer will run until you stop it or it has found too many errors. ## Debug When the fuzzer finds a crash, it saves the game state and replay log to a JSON file. You can import the crashed game state like so: - node tools/import-game.js fuzzer/dump-title-*.json + rtt import fuzzer/dump-title-*.json The imported games don't have snapshots. You can recreate them with the patch-game tool. - node tools/patch-game.js game_id + rtt patch GAME ## Avoidance diff --git a/docs/module/rules.md b/docs/module/rules.md index ed99188..7f9ae2d 100644 --- a/docs/module/rules.md +++ b/docs/module/rules.md @@ -170,6 +170,11 @@ There's also a global scope for the main game data (via the G namespace). --- +The state stack is implmented as a linked list (G.L is the head of the linked +list, and G.L.L is the next state down the stack, etc.) Invoking call pushes a +new state at the top of the stack; goto replaces the current top of the stack, +and end pops the stack. + ## States The "states" where we wait for user input are kept in the S table. diff --git a/docs/overview/architecture.md b/docs/overview/architecture.md index 5865de8..7601d04 100644 --- a/docs/overview/architecture.md +++ b/docs/overview/architecture.md @@ -32,24 +32,24 @@ The following files contain the code and styling for the client display: ## Tools -The "tools" directory holds a number of other useful scripts for administrating the server and debugging modules. +The "rtt" command (in bin/rtt) is useful for administrating the server and debugging modules. - bash tools/export-game.sh game_id > file.json + rtt export game_id > file.json Export full game state to a JSON file. - node tools/import-game.js file.json + rtt import file.json Import a game from an export JSON file. - node tools/patchgame.js game_id + rtt patch game_id Patch game state for one game (by replaying the action log). - node tools/patchgame.js title_id + rtt patch title_id Patch game state for all active games of one module. - bash tools/undo.sh game_id + rtt undo game_id Undo an action by removing the last entry in the replay log and running patchgame.js - bash tools/showgame.sh game_id + rtt show-game game_id Print game state JSON object for debugging.