vibn-frontend/scripts/smoke-chat-tools.ts

/**
 * Smoke test: validate VIBN_TOOL_DEFINITIONS against the live Gemini API.
 *
 * Sends the full tool list with a trivial prompt and checks whether Gemini
 * accepts the schemas. Catches schema validation errors (ARRAY without items,
 * free OBJECT params, etc.) before we deploy.
 *
 * Usage:  GOOGLE_API_KEY=... npx tsx scripts/smoke-chat-tools.ts
 *         (also picks up GOOGLE_API_KEY from .env.local automatically)
 */
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { VIBN_TOOL_DEFINITIONS } from '../lib/ai/vibn-tools';

// Load .env.local manually to avoid needing dotenv as a dep
try {
  const envPath = join(process.cwd(), '.env.local');
  const envText = readFileSync(envPath, 'utf-8');
  for (const line of envText.split('\n')) {
    const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
    if (m && !process.env[m[1]]) process.env[m[1]] = m[2].trim();
  }
} catch {}

const API_KEY = process.env.GOOGLE_API_KEY;
const MODEL = process.env.VIBN_CHAT_MODEL || 'gemini-3.1-pro-preview';

if (!API_KEY) {
  console.error('Missing GOOGLE_API_KEY');
  process.exit(1);
}

async function validateAll() {
  console.log(`Validating ${VIBN_TOOL_DEFINITIONS.length} tool definitions against ${MODEL}...\n`);

  const url = `https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent?key=${API_KEY}`;
  const body = {
    contents: [{ role: 'user', parts: [{ text: 'Show me what is running in my workspace.' }] }],
    tools: [{ functionDeclarations: VIBN_TOOL_DEFINITIONS }],
    generationConfig: { temperature: 0.0, maxOutputTokens: 200 },
  };

  const res = await fetch(url, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify(body),
  });

  const data = await res.json();

  if (!res.ok) {
    console.error(`\n❌ FAIL: HTTP ${res.status}`);
    console.error(JSON.stringify(data, null, 2));

    // Try to identify which tools are bad by sending them one at a time
    console.log('\n🔍 Bisecting to find broken tools...\n');
    const bad: { name: string; error: string }[] = [];
    for (const tool of VIBN_TOOL_DEFINITIONS) {
      const r = await fetch(url, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          contents: [{ role: 'user', parts: [{ text: 'hi' }] }],
          tools: [{ functionDeclarations: [tool] }],
          generationConfig: { temperature: 0.0, maxOutputTokens: 10 },
        }),
      });
      if (!r.ok) {
        const err = await r.json();
        const msg = err?.error?.message || JSON.stringify(err).slice(0, 200);
        bad.push({ name: tool.name, error: msg });
        console.log(`  ❌ ${tool.name}: ${msg.slice(0, 150)}`);
      } else {
        console.log(`  ✓  ${tool.name}`);
      }
    }
    console.log(`\n${bad.length} broken tool(s) out of ${VIBN_TOOL_DEFINITIONS.length}`);
    process.exit(1);
  }

  console.log('✅ All tool definitions accepted by Gemini.');

  const calls = data?.candidates?.[0]?.content?.parts
    ?.filter((p: any) => p.functionCall)
    .map((p: any) => `${p.functionCall.name}(${Object.keys(p.functionCall.args || {}).join(',')})`);
  if (calls?.length) {
    console.log(`\nGemini chose to call: ${calls.join(', ')}`);
  } else {
    console.log('\n(No tool calls produced — model responded with text)');
  }
}

validateAll().catch((e) => {
  console.error('Test crashed:', e);
  process.exit(1);
});