#!/usr/bin/env tsx /** * One-time migration script to process existing knowledge_items into AlloyDB * * This script: * 1. Fetches all knowledge_items from Firestore * 2. For each item, chunks and embeds it into AlloyDB * 3. Shows progress and handles errors gracefully * * Usage: * npx tsx scripts/migrate-existing-knowledge-to-alloydb.ts [projectId] * * - If projectId is provided, processes only that project * - If omitted, processes ALL projects */ import { getAdminDb } from '../lib/firebase/admin'; import { writeKnowledgeChunksForItem, getChunkCountForKnowledgeItem } from '../lib/server/vector-memory'; interface KnowledgeItem { id: string; projectId: string; content: string; sourceMeta?: { sourceType?: string; importance?: 'primary' | 'supporting' | 'irrelevant'; }; } async function getAllKnowledgeItems(projectId?: string): Promise { const adminDb = getAdminDb(); const items: KnowledgeItem[] = []; if (projectId) { // Single project console.log(`[Migration] Fetching knowledge items for project ${projectId}...`); const snapshot = await adminDb .collection('knowledge_items') .where('projectId', '==', projectId) .get(); snapshot.forEach((doc) => { const data = doc.data(); items.push({ id: doc.id, projectId: data.projectId, content: data.content || '', sourceMeta: data.sourceMeta, }); }); } else { // All projects console.log(`[Migration] Fetching ALL knowledge items...`); const snapshot = await adminDb.collection('knowledge_items').get(); snapshot.forEach((doc) => { const data = doc.data(); items.push({ id: doc.id, projectId: data.projectId, content: data.content || '', sourceMeta: data.sourceMeta, }); }); } return items; } async function migrateKnowledgeItems(projectId?: string) { console.log(''); console.log('='.repeat(60)); console.log('🚀 AlloyDB Knowledge Migration'); console.log('='.repeat(60)); console.log(''); try { // Fetch all items const items = await getAllKnowledgeItems(projectId); console.log(`✅ Found ${items.length} knowledge items to process`); console.log(''); if (items.length === 0) { console.log('â„šī¸ No knowledge items found. Nothing to migrate.'); return; } let successCount = 0; let skipCount = 0; let errorCount = 0; // Process each item for (let i = 0; i < items.length; i++) { const item = items[i]; const progress = `[${i + 1}/${items.length}]`; try { // Check if already processed (skip if chunks exist) const existingChunks = await getChunkCountForKnowledgeItem(item.id); if (existingChunks > 0) { console.log(`${progress} â­ī¸ Skipping ${item.id} (already has ${existingChunks} chunks)`); skipCount++; continue; } console.log(`${progress} 🔄 Processing ${item.id}...`); // Chunk and embed await writeKnowledgeChunksForItem(item); const newChunks = await getChunkCountForKnowledgeItem(item.id); console.log(`${progress} ✅ Success! Created ${newChunks} chunks`); successCount++; // Small delay to avoid rate limiting if (i < items.length - 1) { await new Promise((resolve) => setTimeout(resolve, 100)); } } catch (error) { console.error(`${progress} ❌ Failed to process ${item.id}:`, error); errorCount++; } } // Summary console.log(''); console.log('='.repeat(60)); console.log('📊 Migration Complete'); console.log('='.repeat(60)); console.log(`✅ Processed: ${successCount}`); console.log(`â­ī¸ Skipped (already exists): ${skipCount}`); console.log(`❌ Errors: ${errorCount}`); console.log(`đŸ“Ļ Total: ${items.length}`); console.log(''); if (errorCount > 0) { console.log('âš ī¸ Some items failed. Check logs above for details.'); process.exit(1); } else { console.log('🎉 All knowledge items successfully migrated to AlloyDB!'); process.exit(0); } } catch (error) { console.error(''); console.error('❌ Migration failed:', error); process.exit(1); } } // Parse command line arguments const projectId = process.argv[2]; if (projectId) { console.log(`â„šī¸ Processing single project: ${projectId}`); } else { console.log('â„šī¸ No projectId provided - processing ALL projects'); } // Run migration migrateKnowledgeItems(projectId);