#!/usr/bin/env sh # VCN #45 Bench - get a machine ready to write an eval for a coding agent. # Doctor first, then scaffold. Idempotent and safe: it never deletes anything, # never touches global config, and never installs a system package for you (it # prints the one line to run instead). # # Run: curl -fsSL https://vcn-45-bench.vercel.app/wire.sh | sh # Doctor only, changes nothing: # curl -fsSL https://vcn-45-bench.vercel.app/wire.sh | sh -s -- --check # # Options (append after -s --): # --check doctor only, change nothing # --dir
where to scaffold (default ./bench)
# -y assume yes, no prompts
#
# There are NO credentials in this file. It never prints your key. ASCII only.
# Uninstall: delete the directory it created. Nothing else was touched.
set -u
BASE="https://vcn-45-bench.vercel.app"
DIR="bench"
CHECK=0
YES=0
while [ $# -gt 0 ]; do
case "$1" in
--check) CHECK=1 ;;
-y) YES=1 ;;
--dir) shift; DIR="${1:-bench}" ;;
-h|--help) sed -n '2,18p' "$0" 2>/dev/null || echo "see $BASE/wire.sh"; exit 0 ;;
*) echo "unknown option: $1 (ignored)" ;;
esac
shift
done
ok() { printf ' OK %s\n' "$1"; }
warn() { printf ' WARN %s\n' "$1"; }
bad() { printf ' FAIL %s\n' "$1"; }
have() { command -v "$1" >/dev/null 2>&1; }
FAILED=0
echo ""
echo "VCN #45 Bench - machine doctor"
echo "-----------------------------------------------------------"
# --- platform -------------------------------------------------------------
UNAME="$(uname -s 2>/dev/null || echo unknown)"
case "$UNAME" in
Darwin) ok "platform: macOS" ;;
Linux)
if [ -f /etc/os-release ] && grep -qi crostini /etc/os-release 2>/dev/null; then
ok "platform: ChromeOS (Crostini container)"
elif [ -d /mnt/chromeos ]; then
ok "platform: ChromeOS (Crostini container)"
warn "clone repos into \$HOME, not /mnt/chromeos: that mount is slow"
else
ok "platform: Linux"
fi ;;
MINGW*|MSYS*|CYGWIN*)
ok "platform: Windows (Git Bash)"
warn "in PowerShell use \$env:NAME=\"value\", not export. This shell is fine." ;;
*) warn "platform: $UNAME (unrecognised, continuing)" ;;
esac
# --- node -----------------------------------------------------------------
if have node; then
NV="$(node -v 2>/dev/null | tr -d 'v')"
NMAJ="${NV%%.*}"
if [ "${NMAJ:-0}" -ge 18 ] 2>/dev/null; then
ok "node v$NV (>= 18)"
else
bad "node v$NV is too old, need 18+"
case "$UNAME" in
Darwin) echo " installer: https://nodejs.org" ;;
*) echo " curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - && sudo apt-get install -y nodejs" ;;
esac
FAILED=1
fi
else
bad "node not found, need 18+"
case "$UNAME" in
Darwin) echo " installer: https://nodejs.org" ;;
*) echo " curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - && sudo apt-get install -y nodejs" ;;
esac
FAILED=1
fi
have npm && ok "npm $(npm -v 2>/dev/null)" || { bad "npm not found (ships with node)"; FAILED=1; }
# --- claude code ----------------------------------------------------------
if have claude; then
ok "claude code on PATH"
else
warn "claude code not found"
echo " npm install -g @anthropic-ai/claude-code"
echo " if it installs but is still not found, add \`npm bin -g\` to PATH"
fi
# --- python + pytest (the oracle runtime) --------------------------------
PY=""
have python3 && PY=python3
[ -z "$PY" ] && have python && PY=python
if [ -n "$PY" ]; then
ok "$PY $($PY -c 'import sys;print("%d.%d"%sys.version_info[:2])' 2>/dev/null)"
if $PY -c 'import pytest' >/dev/null 2>&1; then
ok "pytest importable (oracles can run)"
else
warn "pytest not installed: your oracle needs SOME test runner"
echo " $PY -m pip install pytest"
fi
else
warn "no python found. Fine if your oracle is not python, but the example is."
fi
have git && ok "git $(git --version 2>/dev/null | awk '{print $3}')" || { bad "git not found: lab 1 uses git worktree"; FAILED=1; }
# --- gateway reachability (no key needed, no key sent) -------------------
GW="https://immersivecommons13.tail5da903.ts.net"
if have curl; then
CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 12 "$GW/" 2>/dev/null || echo 000)"
case "$CODE" in
000) bad "workshop gateway unreachable (network/DNS/TLS). Tell the facilitator." ; FAILED=1 ;;
*) ok "workshop gateway answers (HTTP $CODE; 404 on / is expected)" ;;
esac
else
warn "curl not found, skipped the gateway check"
fi
# --- env vars -------------------------------------------------------------
if [ -n "${ANTHROPIC_AUTH_TOKEN:-}" ]; then
ok "ANTHROPIC_AUTH_TOKEN is set (value not shown)"
[ -n "${ANTHROPIC_BASE_URL:-}" ] && ok "ANTHROPIC_BASE_URL is set" \
|| warn "ANTHROPIC_BASE_URL not set: claude will talk to the wrong host"
else
warn "ANTHROPIC_AUTH_TOKEN not set yet"
echo " request a key: https://immersivecommons.com/zai-keys"
echo " then see: $BASE/setup.txt"
fi
echo "-----------------------------------------------------------"
if [ "$FAILED" -ne 0 ]; then
echo "Some hard requirements are missing. Fix those first."
echo "Full setup: $BASE/setup.txt"
[ "$CHECK" -eq 1 ] && exit 1
fi
[ "$CHECK" -eq 1 ] && { echo "Doctor only, nothing changed."; exit 0; }
# --- scaffold -------------------------------------------------------------
echo ""
if [ -e "$DIR" ]; then
echo "$DIR already exists, leaving it exactly as it is."
else
if [ "$YES" -ne 1 ]; then
printf "Create the bench skeleton in ./%s ? [y/N] " "$DIR"
read ANS 2>/dev/null || ANS=n
case "${ANS:-n}" in y|Y|yes|YES) ;; *) echo "Skipped."; exit 0 ;; esac
fi
mkdir -p "$DIR/tasks/001-example" "$DIR/results" || { echo "could not create $DIR"; exit 1; }
cat > "$DIR/tasks/001-example/prompt.md" <<'EOF'
Replace this with one real task from your own repo.
A good first task is a bug you have ALREADY fixed, so you know the answer:
1. find the fix commit in your history
2. check out its parent into this task's repo/ directory
3. write one sentence here describing what to fix
EOF
cat > "$DIR/tasks/001-example/oracle.sh" <<'EOF'
#!/usr/bin/env bash
# The oracle: exit 0 is pass, anything else is fail. No opinions, no LLM.
set -euo pipefail
cd "$(dirname "$0")/repo"
# Point this at the ONE test that decides this task:
pytest tests/test_example.py -q
EOF
chmod +x "$DIR/tasks/001-example/oracle.sh" 2>/dev/null || true
cat > "$DIR/probe_oracle.sh" <<'EOF'
#!/usr/bin/env bash
# Probe an oracle for determinism BEFORE trusting it.
# Two runs on unchanged code must give the SAME exit code.
set -u
O="${1:?usage: probe_oracle.sh tasks/