vibn-agent-runner/vibn-frontend/scripts/generate-curated-audit-for-opus.ts

import { Client } from 'pg';
import * as dotenv from 'dotenv';
import * as path from 'path';
import * as fs from 'fs';

// Load env variables
dotenv.config({ path: path.join(__dirname, '../.env.local') });

const connectionString = process.env.DATABASE_URL;

if (!connectionString) {
  console.error("DATABASE_URL is not set in .env.local");
  process.exit(1);
}

// Curation target message IDs representing works and fails:
const curatedMessagePairs = [
  // 1. Success - Initial Analysis and SVG fix
  { user: 'fed45d45-0aab-4615-beb6-9355f03c68c4', assistant: '26333aea-4a71-4fba-a331-93bf87a28e1f', tag: 'WORK: File Edit & Dev Server' },
  // 2. Failure - Syntax error loop (duplicate tag introduced)
  { user: 'bd0fdf63-33b8-4541-a599-f8bfe4a21498', assistant: 'a293a59a-185f-4520-a029-750892bbc7a0', tag: 'FAIL: Syntax Error & Dev Server Crash' },
  // 3. Failure - Typo in filename & loop
  { user: 'ae4f6eb3-6514-45de-9820-01adec83f777', assistant: '784121b4-9db2-4fcc-8e74-94e0e7b6cd01', tag: 'FAIL: Filename Mismatch / Cache Loop' },
  // 4. Failure - Syntax loop repeat
  { user: 'bdb738b6-5f0c-4820-9da8-345efb80d432', assistant: '6d8215c0-293e-4f50-b9f8-7cab7a9c8250', tag: 'FAIL: Text Replacement Duplicate Tag Loop' },
  // 5. Success - Final layout polish & CSS integration
  { user: 'c2be02b2-734e-43ca-ba4b-8d2234bcdba6', assistant: 'c6fd1d75-2280-4c5c-b1f7-025f39f37e74', tag: 'WORK: Design QA & CSS Polish' },
  // 6. Success - App-wide propagation (Footer fix)
  { user: 'd3858ebb-601b-4db4-9663-19abe09718cc', assistant: 'c6dcb697-0d18-47bd-8e78-7a45499d2a82', tag: 'WORK: Global Propagation & Cleanup' }
];

async function main() {
  const client = new Client({ connectionString });
  await client.connect();

  console.log("Connected to PostgreSQL DB...");

  const projectId = 'be169fe8-d381-422b-8e9c-d2e513a8f902';
  const threadId = 'a584c700-7ae2-4fad-a906-b8daf80fcace';

  const turns = [];

  for (const pair of curatedMessagePairs) {
    const userRes = await client.query("SELECT id, created_at, data FROM fs_chat_messages WHERE id = $1", [pair.user]);
    const assistantRes = await client.query("SELECT id, created_at, data FROM fs_chat_messages WHERE id = $1", [pair.assistant]);

    if (userRes.rows.length === 0 || assistantRes.rows.length === 0) {
      console.warn(`Could not find pair: ${pair.user} -> ${pair.assistant}`);
      continue;
    }

    const userMsg = userRes.rows[0];
    const assistantMsg = assistantRes.rows[0];

    const rawToolResults = assistantMsg.data._rawToolResults || [];
    const actionsRun = rawToolResults.map((tr: any) => {
      let stdout = tr.result;
      let ok = true;
      let status = "success";

      try {
        const parsedRes = JSON.parse(tr.result);
        if (parsedRes.ok === false || (parsedRes.errors && parsedRes.errors.length > 0)) {
          ok = false;
          status = "error";
        }
      } catch (e) {}

      return {
        tool_name: tr.name,
        tool_call_id: tr.id || `tc-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
        input_args: tr.args || {},
        execution_outcome: {
          ok,
          status,
          stdout
        }
      };
    });

    turns.push({
      turn_metadata: {
        message_id: userMsg.id,
        timestamp_utc: userMsg.created_at.toISOString(),
        conversation_id: threadId,
        audit_tag: pair.tag
      },
      "1_user_interaction": {
        prompt_text: userMsg.data.content
      },
      "2_payload_sent_to_google": {
        endpoint_url: "https://us-central1-aiplatform.googleapis.com/v1/projects/gen-lang-client-0980079410/locations/us-central1/publishers/google/models/gemini-3.1-pro-preview:generateContent",
        system_instruction: "Configured via VIBN Coder System Prompt (coder.ts / buildSystemPrompt)",
        contents: [
          {
            role: "user",
            parts: [{ text: userMsg.data.content }]
          }
        ]
      },
      "3_payload_received_from_google": {
        timestamp_utc: assistantMsg.created_at.toISOString(),
        raw_candidates: {
          content: {
            role: "model",
            parts: [{ thought: null, text: assistantMsg.data.content }]
          }
        }
      },
      "4_platform_executions_and_telemetry": {
        actions_run: actionsRun
      },
      "5_git_version_control_diffs": []
    });
  }

  const dataset = {
    dataset_metadata: {
      title: "VIBN Agent Telemetry Curated Audit Dataset",
      purpose: "Provide high-fidelity telemetry examples of both successful executions and failure loops to Opus for agent behavior optimization and prompt hardening.",
      source_project: {
        id: projectId,
        name: "GetAcquired 2.0",
        slug: "getacquired-2-0"
      },
      compiled_at: new Date().toISOString(),
      total_turns_audited: turns.length
    },
    turns
  };

  const outputPath = path.join(__dirname, '../../opus_telemetry_audit_dataset.json');
  fs.writeFileSync(outputPath, JSON.stringify(dataset, null, 2));
  console.log(`\n🎉 Curated dataset for Opus successfully written to: ${outputPath}`);

  await client.end();
}

main().catch(console.error);