{
  "model_label": "z1",
  "served_model_verified": true,
  "date_utc": "2026-10-13",
  "endpoint": "https://ninetyfive-decisions.numerata.lupine.run",
  "status": "jevbench_complete_jevman_running",
  "note": "Self-reported evaluation of the endpoint-served decision-medium model branded z1. JevBench covers the 231 public tasks only, not its full/sealed suite. Timings include network and post-restart warm-up. Cost was not measured. Initial failed runs are preserved under attempts/.",
  "jevbench": {
    "repository": "https://github.com/fstandhartinger/jevbench",
    "revision": "7625ecb1b075cfe4d09d34141d353325825eea1d",
    "scope": "231 public tasks: 72 original, 48 easy, 111 hard; excludes private/sealed tasks",
    "command": "python -m jevbench.cli run --tasks datasets/public/original.jsonl,datasets/public/easy.jsonl,datasets/public/hard.jsonl --adapter typesafe --endpoint https://ninetyfive-decisions.numerata.lupine.run --model z1 --key-env '' --cost-basis no_billable_account_public_endpoint --reserve-usd 0 --results RUN/results.jsonl --raw-dir RUN/raw --ledger RUN/ledger.jsonl --manifest RUN/manifest.json",
    "attempted": 231,
    "successful": 231,
    "unattempted": 0,
    "exit_code": 0,
    "stop_reason": null,
    "model_quality_score": null,
    "performance_score": null,
    "accuracy": 0.8831168831168831,
    "correct": 204,
    "schema_validity": 1.0,
    "ece": 0.07994470880686899,
    "brier_mean": 0.17388059732042813,
    "latency": {
      "n": 231,
      "p50_s": 0.21858379500008596,
      "p95_s": 1.5345631805002995
    },
    "cohorts": {
      "original": {
        "tasks": 72,
        "correct": 71
      },
      "easy": {
        "tasks": 48,
        "correct": 48
      },
      "hard": {
        "tasks": 111,
        "correct": 85
      }
    },
    "timing_method": "Serial, one preliminary two-question request before run; includes network overhead and asynchronous post-restart runtime warm-up.",
    "summary": "jevbench/summary.json",
    "manifest": "jevbench/manifest.json",
    "results": "jevbench/results.jsonl"
  },
  "jevman": {
    "benchmark": "JevMan",
    "repository": "https://github.com/opper-ai/jevman-benchmark",
    "revision": "1c8b77ac61bafa10956c601b1c9b339dcd277836",
    "date_utc": "2026-10-13",
    "endpoint": "https://ninetyfive-decisions.numerata.lupine.run/v1/systemone",
    "command": "node --import tsx scripts/bench.ts --endpoint https://ninetyfive-decisions.numerata.lupine.run/v1/systemone --games 24 --max 300 --parallel 4 --submit submissions/z1 --name z1 --by numerataz --url https://z1.numerata.lupine.run",
    "exit_code": null,
    "status": "retry_pending_h100_capacity",
    "configured_games": 24,
    "completed_games": null,
    "max_seconds": 300,
    "parallel": 4,
    "score": null,
    "stdout": null,
    "stderr": null,
    "error": "Prior completed 24-game run was interrupted by a platform runtime rollback; retained under attempts/jevman-runtime-interrupted. A fresh run is waiting for H100 capacity."
  },
  "served_model": "decision-medium"
}