#!/usr/bin/env sh # VCN #45 Bench - get a machine ready to write an eval for a coding agent. # Doctor first, then scaffold. Idempotent and safe: it never deletes anything, # never touches global config, and never installs a system package for you (it # prints the one line to run instead). # # Run: curl -fsSL https://vcn-45-bench.vercel.app/wire.sh | sh # Doctor only, changes nothing: # curl -fsSL https://vcn-45-bench.vercel.app/wire.sh | sh -s -- --check # # Options (append after -s --): # --check doctor only, change nothing # --dir

where to scaffold (default ./bench) # -y assume yes, no prompts # # There are NO credentials in this file. It never prints your key. ASCII only. # Uninstall: delete the directory it created. Nothing else was touched. set -u BASE="https://vcn-45-bench.vercel.app" DIR="bench" CHECK=0 YES=0 while [ $# -gt 0 ]; do case "$1" in --check) CHECK=1 ;; -y) YES=1 ;; --dir) shift; DIR="${1:-bench}" ;; -h|--help) sed -n '2,18p' "$0" 2>/dev/null || echo "see $BASE/wire.sh"; exit 0 ;; *) echo "unknown option: $1 (ignored)" ;; esac shift done ok() { printf ' OK %s\n' "$1"; } warn() { printf ' WARN %s\n' "$1"; } bad() { printf ' FAIL %s\n' "$1"; } have() { command -v "$1" >/dev/null 2>&1; } FAILED=0 echo "" echo "VCN #45 Bench - machine doctor" echo "-----------------------------------------------------------" # --- platform ------------------------------------------------------------- UNAME="$(uname -s 2>/dev/null || echo unknown)" case "$UNAME" in Darwin) ok "platform: macOS" ;; Linux) if [ -f /etc/os-release ] && grep -qi crostini /etc/os-release 2>/dev/null; then ok "platform: ChromeOS (Crostini container)" elif [ -d /mnt/chromeos ]; then ok "platform: ChromeOS (Crostini container)" warn "clone repos into \$HOME, not /mnt/chromeos: that mount is slow" else ok "platform: Linux" fi ;; MINGW*|MSYS*|CYGWIN*) ok "platform: Windows (Git Bash)" warn "in PowerShell use \$env:NAME=\"value\", not export. This shell is fine." ;; *) warn "platform: $UNAME (unrecognised, continuing)" ;; esac # --- node ----------------------------------------------------------------- if have node; then NV="$(node -v 2>/dev/null | tr -d 'v')" NMAJ="${NV%%.*}" if [ "${NMAJ:-0}" -ge 18 ] 2>/dev/null; then ok "node v$NV (>= 18)" else bad "node v$NV is too old, need 18+" case "$UNAME" in Darwin) echo " installer: https://nodejs.org" ;; *) echo " curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - && sudo apt-get install -y nodejs" ;; esac FAILED=1 fi else bad "node not found, need 18+" case "$UNAME" in Darwin) echo " installer: https://nodejs.org" ;; *) echo " curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - && sudo apt-get install -y nodejs" ;; esac FAILED=1 fi have npm && ok "npm $(npm -v 2>/dev/null)" || { bad "npm not found (ships with node)"; FAILED=1; } # --- claude code ---------------------------------------------------------- if have claude; then ok "claude code on PATH" else warn "claude code not found" echo " npm install -g @anthropic-ai/claude-code" echo " if it installs but is still not found, add \`npm bin -g\` to PATH" fi # --- python + pytest (the oracle runtime) -------------------------------- PY="" have python3 && PY=python3 [ -z "$PY" ] && have python && PY=python if [ -n "$PY" ]; then ok "$PY $($PY -c 'import sys;print("%d.%d"%sys.version_info[:2])' 2>/dev/null)" if $PY -c 'import pytest' >/dev/null 2>&1; then ok "pytest importable (oracles can run)" else warn "pytest not installed: your oracle needs SOME test runner" echo " $PY -m pip install pytest" fi else warn "no python found. Fine if your oracle is not python, but the example is." fi have git && ok "git $(git --version 2>/dev/null | awk '{print $3}')" || { bad "git not found: lab 1 uses git worktree"; FAILED=1; } # --- gateway reachability (no key needed, no key sent) ------------------- GW="https://immersivecommons13.tail5da903.ts.net" if have curl; then CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 12 "$GW/" 2>/dev/null || echo 000)" case "$CODE" in 000) bad "workshop gateway unreachable (network/DNS/TLS). Tell the facilitator." ; FAILED=1 ;; *) ok "workshop gateway answers (HTTP $CODE; 404 on / is expected)" ;; esac else warn "curl not found, skipped the gateway check" fi # --- env vars ------------------------------------------------------------- if [ -n "${ANTHROPIC_AUTH_TOKEN:-}" ]; then ok "ANTHROPIC_AUTH_TOKEN is set (value not shown)" [ -n "${ANTHROPIC_BASE_URL:-}" ] && ok "ANTHROPIC_BASE_URL is set" \ || warn "ANTHROPIC_BASE_URL not set: claude will talk to the wrong host" else warn "ANTHROPIC_AUTH_TOKEN not set yet" echo " request a key: https://immersivecommons.com/zai-keys" echo " then see: $BASE/setup.txt" fi echo "-----------------------------------------------------------" if [ "$FAILED" -ne 0 ]; then echo "Some hard requirements are missing. Fix those first." echo "Full setup: $BASE/setup.txt" [ "$CHECK" -eq 1 ] && exit 1 fi [ "$CHECK" -eq 1 ] && { echo "Doctor only, nothing changed."; exit 0; } # --- scaffold ------------------------------------------------------------- echo "" if [ -e "$DIR" ]; then echo "$DIR already exists, leaving it exactly as it is." else if [ "$YES" -ne 1 ]; then printf "Create the bench skeleton in ./%s ? [y/N] " "$DIR" read ANS 2>/dev/null || ANS=n case "${ANS:-n}" in y|Y|yes|YES) ;; *) echo "Skipped."; exit 0 ;; esac fi mkdir -p "$DIR/tasks/001-example" "$DIR/results" || { echo "could not create $DIR"; exit 1; } cat > "$DIR/tasks/001-example/prompt.md" <<'EOF' Replace this with one real task from your own repo. A good first task is a bug you have ALREADY fixed, so you know the answer: 1. find the fix commit in your history 2. check out its parent into this task's repo/ directory 3. write one sentence here describing what to fix EOF cat > "$DIR/tasks/001-example/oracle.sh" <<'EOF' #!/usr/bin/env bash # The oracle: exit 0 is pass, anything else is fail. No opinions, no LLM. set -euo pipefail cd "$(dirname "$0")/repo" # Point this at the ONE test that decides this task: pytest tests/test_example.py -q EOF chmod +x "$DIR/tasks/001-example/oracle.sh" 2>/dev/null || true cat > "$DIR/probe_oracle.sh" <<'EOF' #!/usr/bin/env bash # Probe an oracle for determinism BEFORE trusting it. # Two runs on unchanged code must give the SAME exit code. set -u O="${1:?usage: probe_oracle.sh tasks//oracle.sh}" bash "$O" >/dev/null 2>&1; A=$? bash "$O" >/dev/null 2>&1; B=$? echo "run1=$A run2=$B" if [ "$A" -eq "$B" ]; then echo "DETERMINISTIC (verdict $A). Safe to trust this oracle." else echo "FLAKY ORACLE. Fix this before blaming the agent:" echo " every number you compute from it is noise." exit 1 fi EOF chmod +x "$DIR/probe_oracle.sh" 2>/dev/null || true ok "scaffolded ./$DIR (tasks/001-example, probe_oracle.sh, results/)" echo "" echo "You write runner.py during lab 3. The code is on the slides:" echo " $BASE (slide 17)" echo "Prose for your agent: $BASE/llms.txt" fi echo "" echo "Doors 19:00, walkthrough 19:30, Frontier Tower Floor 10. See you there."