vibn-agent-runner/vibn-frontend/scripts/generate-qa-audit-for-opus.ts

import { Client } from 'pg';
import * as dotenv from 'dotenv';
import * as path from 'path';
import * as fs from 'fs';

// Load env variables
dotenv.config({ path: path.join(__dirname, '../.env.local') });

const connectionString = process.env.DATABASE_URL;

if (!connectionString) {
  console.error("DATABASE_URL is not set in .env.local");
  process.exit(1);
}

// Curation target message IDs from the live tests:
const curatedMessagePairs = [
  // 1. Success - Hardened fs_edit (Navbar title change)
  {
    user: 'acc489f6-808c-4a1c-9686-7f6145f2fa48',
    assistant: '0c71e628-f6f3-4bce-9ccf-7a1631e7537a',
    tag: 'WORK: Hardened fs_edit (Navbar Title Change)'
  },
  // 2. Success - Port-reaper dev server restart
  {
    user: 'b304d55c-7d8a-4e1a-9823-58d521b1796d',
    assistant: 'a244bc69-b0e7-4977-89fb-d39949a7e7ed',
    tag: 'WORK: Socket-Inode Port Reaper dev_server_start'
  },
  // 3. Failure - Prisma error and AI Success Hallucination
  {
    user: '6c98dd1f-ba1a-4cd8-a654-8e6da4eecace',
    assistant: '46acdaad-3ea7-4fd1-9fde-6aff1d33e896',
    tag: 'FAIL: Prisma DB Error and AI Success Hallucination'
  }
];

async function main() {
  const client = new Client({ connectionString });
  await client.connect();

  console.log("Connected to PostgreSQL DB...");

  const projectId = 'be169fe8-d381-422b-8e9c-d2e513a8f902';
  const threadId = 'a584c700-7ae2-4fad-a906-b8daf80fcace';

  const turns = [];

  for (const pair of curatedMessagePairs) {
    const userRes = await client.query("SELECT id, created_at, data FROM fs_chat_messages WHERE id = $1", [pair.user]);
    const assistantRes = await client.query("SELECT id, created_at, data FROM fs_chat_messages WHERE id = $1", [pair.assistant]);

    if (userRes.rows.length === 0 || assistantRes.rows.length === 0) {
      console.warn(`Could not find pair: ${pair.user} -> ${pair.assistant}`);
      continue;
    }

    const userMsg = userRes.rows[0];
    const assistantMsg = assistantRes.rows[0];

    const rawToolResults = assistantMsg.data._rawToolResults || [];
    const actionsRun = rawToolResults.map((tr: any) => {
      let stdout = tr.result;
      let ok = true;
      let status = "success";

      try {
        const parsedRes = JSON.parse(tr.result);
        if (parsedRes.ok === false || (parsedRes.errors && parsedRes.errors.length > 0) || parsedRes.error) {
          ok = false;
          status = "error";
        }
      } catch (e) {}

      return {
        tool_name: tr.name,
        tool_call_id: tr.id || `tc-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
        input_args: tr.args || {},
        execution_outcome: {
          ok,
          status,
          stdout
        }
      };
    });

    turns.push({
      turn_metadata: {
        message_id: userMsg.id,
        timestamp_utc: userMsg.created_at.toISOString(),
        conversation_id: threadId,
        audit_tag: pair.tag
      },
      "1_user_interaction": {
        prompt_text: userMsg.data.content
      },
      "2_payload_sent_to_google": {
        endpoint_url: "https://us-central1-aiplatform.googleapis.com/v1/projects/gen-lang-client-0980079410/locations/us-central1/publishers/google/models/gemini-3.1-pro-preview:generateContent",
        system_instruction: "Configured via VIBN Coder System Prompt (coder.ts / buildSystemPrompt)",
        contents: [
          {
            role: "user",
            parts: [{ text: userMsg.data.content }]
          }
        ]
      },
      "3_payload_received_from_google": {
        timestamp_utc: assistantMsg.created_at.toISOString(),
        raw_candidates: {
          content: {
            role: "model",
            parts: [{ thought: null, text: assistantMsg.data.content }]
          }
        }
      },
      "4_platform_executions_and_telemetry": {
        actions_run: actionsRun
      },
      "5_git_version_control_diffs": []
    });
  }

  const dataset = {
    dataset_metadata: {
      title: "VIBN Production QA Telemetry Dataset (Hardening Validation)",
      purpose: "Telemetry audit tracking the exact performance of Task-1, Task-3, and Task-4 fixes, plus isolating the database-related AI success-hallucination error.",
      source_project: {
        id: projectId,
        name: "GetAcquired 2.0",
        slug: "getacquired-2-0"
      },
      compiled_at: new Date().toISOString(),
      total_turns_audited: turns.length
    },
    turns
  };

  // We date the file dynamically to track your QA cycles
  const currentDate = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
  const outputPath = path.join(__dirname, `../../opus_telemetry_audit_dataset_${currentDate}.json`);
  fs.writeFileSync(outputPath, JSON.stringify(dataset, null, 2));
  console.log(`\n🎉 New dated QA telemetry dataset written to: ${outputPath}`);

  await client.end();
}

main().catch(console.error);