Loop Engineering: Designing the Cycle of the Work

The 5-Phase Agentic Execution Loop with Checkpointing & Budget Gates

export async function verifyAgentStep(stepOutput: AgentStepOutput): Promise<VerificationResult> {
  // Tier 1: Cheap Deterministic Schema / Regex Check (1ms, $0)
  const schemaCheck = ZodStepSchema.safeParse(stepOutput);
  if (!schemaCheck.success) return { passed: false, tier: 'TIER_1_SCHEMA', error: schemaCheck.error.message };

  // Tier 2: Executable Unit / Integration Test (50ms, $0)
  const testOutcome = await runHeadlessStepTest(stepOutput.actionPayload);
  if (!testOutcome.passed) return { passed: false, tier: 'TIER_2_TEST_SUITE', error: testOutcome.failureReason };

  // Tier 3: Isolated LLM Evaluator Pass (Separate Prompt & Temperature 0.0)
  const judgeScore = await independentLlmJudge.evaluate({
    rubric: 'Verify compliance with store return policy clause 7.1',
    evidence: stepOutput.evidencePayload
  });
  if (judgeScore.verdict !== 'PASS') return { passed: false, tier: 'TIER_3_LLM_JUDGE', error: judgeScore.feedback };

  return { passed: true, tier: 'ALL_VERIFIERS_PASSED' };
}

export interface RunBudget {
  maxSteps: number;       // e.g. 10
  maxTokens: number;      // e.g. 30,000
  maxCostUsd: number;     // e.g. $0.40
  timeoutSeconds: number; // e.g. 60s
}

export function evaluateLoopTermination(state: LoopState, budget: RunBudget): TerminalDecision {
  if (state.isGoalVerified) return { terminate: true, state: 'SUCCESS_VERIFIED' };
  if (state.currentStep >= budget.maxSteps) return { terminate: true, state: 'BUDGET_EXHAUSTED_STEPS' };
  if (state.accumulatedCost >= budget.maxCostUsd) return { terminate: true, state: 'BUDGET_EXHAUSTED_COST' };
  if (state.elapsedSeconds >= budget.timeoutSeconds) return { terminate: true, state: 'BUDGET_EXHAUSTED_TIMEOUT' };
  if (state.fatalErrorEncountered) return { terminate: true, state: 'TERMINAL_ERROR' };
  
  return { terminate: false, state: 'CONTINUE_LOOP' };
}

export async function executeRetryLadder(failedStep: StepContext, attemptNumber: number): Promise<StepExecution> {
  switch (attemptNumber) {
    case 1:
      // Rung 1: Refine parameters and tighten query bounds
      return await executeWithRefinedArgs(failedStep, { similarityThreshold: 0.85 });
    case 2:
      // Rung 2: Switch to alternative data source or secondary tool
      return await executeAlternativeTool(failedStep, 'queryWarehouseBarcodeLogs');
    case 3:
      // Rung 3: Escalate to frontier reasoning model & decompose task
      return await executeWithFrontierModel(failedStep, { model: 'claude-3-5-sonnet', temperature: 0.0 });
    default:
      // Rung 4: Graceful Human Handoff with full diagnostic trace
      return await routeToHumanSupervisor(failedStep.sessionId, failedStep.failureHistory);
  }
}

export async function saveStepCheckpoint(sessionId: string, stepIndex: number, verifiedState: AgentState) {
  await stateStore.set(`checkpoint:${sessionId}:${stepIndex}`, {
    timestamp: Date.now(),
    stepIndex,
    workingScratchpad: verifiedState.scratchpad,
    completedActions: verifiedState.history,
    budgetConsumed: verifiedState.budget
  });
}

export async function restoreLastKnownGoodState(sessionId: string): Promise<AgentState> {
  const latestCheckpoint = await stateStore.getLatestValidCheckpoint(sessionId);
  return latestCheckpoint ?? initializeInitialState(sessionId);
}

export async function convertFailureTraceToRegressionTest(failedSessionTrace: ExecutionTrace) {
  const sanitizedTrace = scrubCustomerPii(failedSessionTrace);
  
  const generatedTestFixture = {
    testName: `Regression: Order ${sanitizedTrace.orderId} - ${sanitizedTrace.failureCategory}`,
    inputTurn: sanitizedTrace.initialPrompt,
    mockContext: sanitizedTrace.retrievedContext,
    expectedHarnessAction: sanitizedTrace.correctTargetAction,
    maxAllowedSteps: 4,
    prohibitedActions: [sanitizedTrace.failedAction]
  };

  await gitHubClient.createPullRequest({
    branch: `test/regression-${Date.now()}`,
    file: 'tests/evals/agent-regressions.json',
    content: JSON.stringify(generatedTestFixture, null, 2)
  });
}

The Progressive Verifier Pyramid & The 4-Rung Retry Ladder

1. Multi-Step Execution with State Checkpointing

2. Autonomous Discrepancy Recovery

3. Automated Regression Test Synthesis