BENCH.

# 1. Node 18+ (installer from nodejs.org works on Apple Silicon and Intel)
node -v                 # want v18 or higher

# 2. Claude Code
npm install -g @anthropic-ai/claude-code

# 3. Point it at the workshop gateway (paste your approved key)
export ANTHROPIC_BASE_URL=https://immersivecommons13.tail5da903.ts.net
export ANTHROPIC_AUTH_TOKEN=agt_your_key_here
export ANTHROPIC_MODEL=glm-4.6
export ANTHROPIC_SMALL_FAST_MODEL=glm-4.5-air

claude                  # you are live

# 1. Node 18+. Do NOT trust `apt install nodejs` on older LTS, it ships v12.
curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
sudo apt-get install -y nodejs
node -v                 # want v18 or higher

# 2. Claude Code
sudo npm install -g @anthropic-ai/claude-code

# 3. Point it at the workshop gateway (paste your approved key)
export ANTHROPIC_BASE_URL=https://immersivecommons13.tail5da903.ts.net
export ANTHROPIC_AUTH_TOKEN=agt_your_key_here
export ANTHROPIC_MODEL=glm-4.6
export ANTHROPIC_SMALL_FAST_MODEL=glm-4.5-air

claude                  # you are live

# PowerShell. NOTE: `export` does not exist here, use $env: instead.
# 1. Node 18+ (installer from nodejs.org)
node -v                 # want v18 or higher

# 2. Claude Code
npm install -g @anthropic-ai/claude-code

# 3. Point it at the workshop gateway (paste your approved key)
$env:ANTHROPIC_BASE_URL = "https://immersivecommons13.tail5da903.ts.net"
$env:ANTHROPIC_AUTH_TOKEN = "agt_your_key_here"
$env:ANTHROPIC_MODEL = "glm-4.6"
$env:ANTHROPIC_SMALL_FAST_MODEL = "glm-4.5-air"

claude                  # you are live

# Prefer bash? Git Bash takes the macOS/Linux block verbatim.
# Chaining: `&&` fails in PowerShell 5.1. Use `;` or `cmd1; if ($?) { cmd2 }`.

# ChromeOS runs this in the Linux container (Crostini). Turn it on first:
#   Settings -> Advanced -> Developers -> Linux development environment -> On
# Then open Terminal. Inside, you are on Debian, so this is the Linux path.

sudo apt update
curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
sudo apt-get install -y nodejs
node -v                 # want v18 or higher

sudo npm install -g @anthropic-ai/claude-code

export ANTHROPIC_BASE_URL=https://immersivecommons13.tail5da903.ts.net
export ANTHROPIC_AUTH_TOKEN=agt_your_key_here
export ANTHROPIC_MODEL=glm-4.6
export ANTHROPIC_SMALL_FAST_MODEL=glm-4.5-air

claude                  # you are live

# Low on RAM? Do not try to host a model locally. Use the Nebius
# hosted fallback from slide 3. That is what the credits are for.

cat > tasks/001-fix-null-deref/oracle.sh <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/repo"
pytest tests/test_parse_header.py -q    # the ONE test that decides this task
EOF
chmod +x tasks/001-fix-null-deref/oracle.sh

# PROVE it is deterministic BEFORE you trust it: same verdict twice.
./tasks/001-fix-null-deref/oracle.sh; echo "run1=$?"
./tasks/001-fix-null-deref/oracle.sh; echo "run2=$?"

cat > tasks/001-fix-null-deref/oracle.sh <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/repo"
pytest tests/test_parse_header.py -q    # the ONE test that decides this task
EOF
chmod +x tasks/001-fix-null-deref/oracle.sh

# PROVE it is deterministic BEFORE you trust it: same verdict twice.
./tasks/001-fix-null-deref/oracle.sh; echo "run1=$?"
./tasks/001-fix-null-deref/oracle.sh; echo "run2=$?"

# PowerShell-native oracle. A .sh will NOT run here without Git Bash.
Set-Content -Encoding utf8 tasks\001-fix-null-deref\oracle.ps1 @'
$ErrorActionPreference = "Stop"
Set-Location "$PSScriptRoot\repo"
pytest tests/test_parse_header.py -q
exit $LASTEXITCODE
'@

# PROVE it is deterministic BEFORE you trust it: same verdict twice.
powershell -File tasks\001-fix-null-deref\oracle.ps1; "run1=$LASTEXITCODE"
powershell -File tasks\001-fix-null-deref\oracle.ps1; "run2=$LASTEXITCODE"

# Writing oracle.sh on Windows instead? Save it LF, not CRLF, or bash
# reports: bad interpreter: /usr/bin/env bash^M

# Crostini is Debian, so the bash oracle applies unchanged.
cat > tasks/001-fix-null-deref/oracle.sh <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/repo"
pytest tests/test_parse_header.py -q
EOF
chmod +x tasks/001-fix-null-deref/oracle.sh

./tasks/001-fix-null-deref/oracle.sh; echo "run1=$?"
./tasks/001-fix-null-deref/oracle.sh; echo "run2=$?"

# No pytest yet?  sudo apt install -y python3-pip && pip3 install pytest

# runner.py, the whole idea. Agent patches, ORACLE decides.
import json, pathlib, subprocess, sys

def attempt(task: pathlib.Path) -> bool:
    prompt = (task / "prompt.md").read_text()
    # non-interactive agent run, scoped to the task's repo copy
    subprocess.run(["claude", "-p", prompt], cwd=task / "repo", timeout=600)
    oracle = task / "oracle.sh"
    rc = subprocess.run(["bash", str(oracle)], cwd=task).returncode
    return rc == 0            # exit code is the ONLY signal we read

tasks = sorted(p for p in pathlib.Path("tasks").iterdir() if p.is_dir())
rows = {t.name: attempt(t) for t in tasks}
pathlib.Path("results/run1.json").write_text(json.dumps(rows, indent=2))
print(rows)

# runner.py, the whole idea. Agent patches, ORACLE decides.
import json, pathlib, subprocess, sys

def attempt(task: pathlib.Path) -> bool:
    prompt = (task / "prompt.md").read_text()
    subprocess.run(["claude", "-p", prompt], cwd=task / "repo", timeout=600)
    oracle = task / "oracle.sh"
    rc = subprocess.run(["bash", str(oracle)], cwd=task).returncode
    return rc == 0            # exit code is the ONLY signal we read

tasks = sorted(p for p in pathlib.Path("tasks").iterdir() if p.is_dir())
rows = {t.name: attempt(t) for t in tasks}
pathlib.Path("results/run1.json").write_text(json.dumps(rows, indent=2))
print(rows)

# runner.py. Identical Python, but call the PowerShell oracle.
import json, pathlib, subprocess

def attempt(task: pathlib.Path) -> bool:
    prompt = (task / "prompt.md").read_text()
    subprocess.run(["claude", "-p", prompt], cwd=task / "repo", timeout=600)
    oracle = task / "oracle.ps1"
    rc = subprocess.run(
        ["powershell", "-NoProfile", "-File", str(oracle)], cwd=task
    ).returncode
    return rc == 0            # exit code is the ONLY signal we read

tasks = sorted(p for p in pathlib.Path("tasks").iterdir() if p.is_dir())
rows = {t.name: attempt(t) for t in tasks}
pathlib.Path("results/run1.json").write_text(json.dumps(rows, indent=2))
print(rows)

# Run it:  python runner.py     (not py -3 runner.py if python is on PATH)

# Crostini: identical to Linux.
import json, pathlib, subprocess

def attempt(task: pathlib.Path) -> bool:
    prompt = (task / "prompt.md").read_text()
    subprocess.run(["claude", "-p", prompt], cwd=task / "repo", timeout=600)
    rc = subprocess.run(["bash", str(task / "oracle.sh")], cwd=task).returncode
    return rc == 0

tasks = sorted(p for p in pathlib.Path("tasks").iterdir() if p.is_dir())
rows = {t.name: attempt(t) for t in tasks}
pathlib.Path("results/run1.json").write_text(json.dumps(rows, indent=2))
print(rows)

# Container feels slow? The agent is fine, it is your disk. Keep the
# task repos small, and lean on Nebius for any model inference.

# wrap the single run in a k-loop, fresh repo copy per attempt
for k in 1 2 3 4 5; do
  rm -rf .work && cp -r tasks .work        # pristine start every attempt
  python runner.py --tasks .work --out "results/run$k.json"
done

# score it: per task, over k runs
#   pass_at_k     = any(r.passed for r in runs)    # optimistic
#   pass_caret_k  = all(r.passed for r in runs)    # reliability

# wrap the single run in a k-loop, fresh repo copy per attempt
for k in 1 2 3 4 5; do
  rm -rf .work && cp -r tasks .work        # pristine start every attempt
  python runner.py --tasks .work --out "results/run$k.json"
done

# score it: per task, over k runs
#   pass_at_k     = any(r.passed for r in runs)    # optimistic
#   pass_caret_k  = all(r.passed for r in runs)    # reliability

# PowerShell loop. Note: `&&` does not chain in PowerShell 5.1.
foreach ($k in 1..5) {
  Remove-Item -Recurse -Force .work -ErrorAction SilentlyContinue
  Copy-Item -Recurse tasks .work           # pristine start every attempt
  python runner.py --tasks .work --out "results\run$k.json"
}

# score it: per task, over k runs
#   pass_at_k     = any(...)   # optimistic
#   pass_caret_k  = all(...)   # reliability

# Crostini: identical to Linux.
for k in 1 2 3 4 5; do
  rm -rf .work && cp -r tasks .work
  python3 runner.py --tasks .work --out "results/run$k.json"
done

# 5 runs x 3 tasks x an agent call each. If the container is slow,
# drop to k=3 to stay inside the lab window, and SAY that your k was 3
# when you report a number.