{
  "version": "0.1-draft",
  "stage": "planning",
  "updatedOn": "2026-10-16",
  "displayDate": "October 16, 2026 (Pacific)",
  "frozenOn": null,
  "goal": "Learn new tasks while preserving previously learned capabilities.",
  "state": "No results have been measured for this proposed evaluation. The environment, budget, and targets are provisional; pilot checks will inform a versioned protocol before formal evaluation. Architecture choices remain open.",
  "environment": {
    "name": "Custom four-task navigation diagnostic",
    "summary": "A controlled, custom diagnostic to measure learning and forgetting cheaply before broader agent evaluations.",
    "caveat": "This is a provisional custom diagnostic, not an established benchmark or a commitment to a game-focused research program. Success would justify broader testing, not demonstrate general continual learning or AGI.",
    "gridSize": 9,
    "episodeActionLimit": 100,
    "tasks": [
      {
        "id": "goal",
        "name": "Navigate to a goal"
      },
      {
        "id": "key",
        "name": "Collect a key, then reach a locked goal"
      },
      {
        "id": "hazards",
        "name": "Avoid hazards while reaching a goal"
      },
      {
        "id": "combined",
        "name": "Combine keys, locks, and hazards"
      }
    ],
    "interface": "Four procedurally generated gridworld task families share the same fully observed interface and actions, with visible task cues and sequential training."
  },
  "comparisons": [
    {
      "name": "Sequential training without replay",
      "description": "The forgetting baseline."
    },
    {
      "name": "Fixed-capacity uniform replay",
      "description": "A memory-based comparison with disclosed capacity and contents."
    },
    {
      "name": "Candidate method",
      "description": "The approach to evaluate; architecture choices remain open."
    }
  ],
  "proposedBudget": {
    "interactionsPerTask": 100000,
    "fairness": "Match environment-interaction and gradient-update budgets across the main comparisons, and count replay updates.",
    "disclosures": [
      "Model size",
      "Memory capacity and contents",
      "Tuning effort",
      "Runtime and compute"
    ],
    "optionalReferences": "Optional joint-training and task-specialist references would be labeled separately because they have different data-access assumptions."
  },
  "metrics": [
    {
      "id": "new-task",
      "label": "Immediate new-task success",
      "proposedTarget": "At least 70% held-out success on each task immediately after training it.",
      "kind": "gate",
      "measured": null
    },
    {
      "id": "retention",
      "label": "Average retention loss",
      "proposedTarget": "At most 10 percentage points on earlier tasks at the end of the stream.",
      "kind": "gate",
      "measured": null
    },
    {
      "id": "sequential-comparison",
      "label": "Final average vs. sequential no-replay",
      "proposedTarget": "At least 10 percentage points higher final average success.",
      "kind": "gate",
      "measured": null
    },
    {
      "id": "replay-adaptation",
      "label": "Immediate new-task average vs. replay",
      "proposedTarget": "No more than 5 percentage points below the replay baseline.",
      "kind": "gate",
      "measured": null
    },
    {
      "id": "replay-final",
      "label": "Final average vs. replay",
      "proposedTarget": "Report the comparison. Do not claim an improvement over replay unless evidence supports it.",
      "kind": "reporting",
      "measured": null
    }
  ],
  "methods": [
    {
      "title": "New-task success",
      "text": "Measure success on held-out layouts immediately after each task phase."
    },
    {
      "title": "Retention loss",
      "text": "For each of the first three tasks, calculate max(0, immediate success − end-of-stream success), then take the mean. Improvements on one task cannot conceal forgetting on another."
    },
    {
      "title": "Full reporting",
      "text": "Report per-task final success and learning curves alongside aggregate metrics, including final average success versus replay."
    }
  ],
  "safeguards": [
    "Use separate training, validation, and final-test generation seeds, with duplicate-layout checks.",
    "Freeze the task order, hyperparameters, budgets, thresholds, and evaluator before final evaluation.",
    "Allow no model updates or replay writes from test episodes."
  ],
  "calibration": "The numbers are proposed targets, not achieved scores or calibrated promises. If pilot baselines show ceiling effects or little forgetting, revise the diagnostic using development data before freezing it, rather than claim success on an uninformative test.",
  "seeds": {
    "preliminary": 3,
    "later": 10,
    "taskOrders": 3,
    "description": "Use at least 3 independent training seeds for initial preliminary evidence and report all of them. Later validation targets 10 seeds over 3 predeclared task orders, with uncertainty intervals."
  },
  "outsideScope": [
    "Hidden task boundaries",
    "Partial observability",
    "Real-world transfer",
    "General agent ability"
  ],
  "milestones": [
    {
      "id": "protocol",
      "title": "Draft and freeze protocol",
      "status": "planning",
      "note": "Draft proposed; freeze after pilot calibration and before formal evaluation.",
      "completedOn": null
    },
    {
      "id": "environment",
      "title": "Implement and test environment",
      "status": "planned",
      "completedOn": null
    },
    {
      "id": "baselines",
      "title": "Establish sequential and replay baselines",
      "status": "planned",
      "completedOn": null
    },
    {
      "id": "candidate",
      "title": "Evaluate candidate",
      "status": "planned",
      "completedOn": null
    },
    {
      "id": "repeat",
      "title": "Repeat across seeds and task orders",
      "status": "planned",
      "completedOn": null
    },
    {
      "id": "external-benchmark",
      "title": "Evaluate on an independently selected established benchmark",
      "status": "planned",
      "note": "Benchmark selection remains pending.",
      "completedOn": null
    }
  ],
  "results": {
    "measuredOn": null,
    "perTask": [
      {
        "taskId": "goal",
        "immediateSuccess": null,
        "finalSuccess": null,
        "learningCurve": null
      },
      {
        "taskId": "key",
        "immediateSuccess": null,
        "finalSuccess": null,
        "learningCurve": null
      },
      {
        "taskId": "hazards",
        "immediateSuccess": null,
        "finalSuccess": null,
        "learningCurve": null
      },
      {
        "taskId": "combined",
        "immediateSuccess": null,
        "finalSuccess": null,
        "learningCurve": null
      }
    ],
    "resourceCosts": null,
    "uncertainty": null,
    "reports": []
  }
}