VIBN Frontend for Coolify deployment

scripts/migrate-existing-knowledge-to-alloydb.ts

@@ -0,0 +1,161 @@
+#!/usr/bin/env tsx
+/**
+ * One-time migration script to process existing knowledge_items into AlloyDB
+ * 
+ * This script:
+ * 1. Fetches all knowledge_items from Firestore
+ * 2. For each item, chunks and embeds it into AlloyDB
+ * 3. Shows progress and handles errors gracefully
+ * 
+ * Usage:
+ *   npx tsx scripts/migrate-existing-knowledge-to-alloydb.ts [projectId]
+ *   
+ *   - If projectId is provided, processes only that project
+ *   - If omitted, processes ALL projects
+ */
+
+import { getAdminDb } from '../lib/firebase/admin';
+import { writeKnowledgeChunksForItem, getChunkCountForKnowledgeItem } from '../lib/server/vector-memory';
+
+interface KnowledgeItem {
+  id: string;
+  projectId: string;
+  content: string;
+  sourceMeta?: {
+    sourceType?: string;
+    importance?: 'primary' | 'supporting' | 'irrelevant';
+  };
+}
+
+async function getAllKnowledgeItems(projectId?: string): Promise<KnowledgeItem[]> {
+  const adminDb = getAdminDb();
+  const items: KnowledgeItem[] = [];
+
+  if (projectId) {
+    // Single project
+    console.log(`[Migration] Fetching knowledge items for project ${projectId}...`);
+    const snapshot = await adminDb
+      .collection('knowledge_items')
+      .where('projectId', '==', projectId)
+      .get();
+
+    snapshot.forEach((doc) => {
+      const data = doc.data();
+      items.push({
+        id: doc.id,
+        projectId: data.projectId,
+        content: data.content || '',
+        sourceMeta: data.sourceMeta,
+      });
+    });
+  } else {
+    // All projects
+    console.log(`[Migration] Fetching ALL knowledge items...`);
+    const snapshot = await adminDb.collection('knowledge_items').get();
+
+    snapshot.forEach((doc) => {
+      const data = doc.data();
+      items.push({
+        id: doc.id,
+        projectId: data.projectId,
+        content: data.content || '',
+        sourceMeta: data.sourceMeta,
+      });
+    });
+  }
+
+  return items;
+}
+
+async function migrateKnowledgeItems(projectId?: string) {
+  console.log('');
+  console.log('='.repeat(60));
+  console.log('🚀 AlloyDB Knowledge Migration');
+  console.log('='.repeat(60));
+  console.log('');
+
+  try {
+    // Fetch all items
+    const items = await getAllKnowledgeItems(projectId);
+    console.log(`✅ Found ${items.length} knowledge items to process`);
+    console.log('');
+
+    if (items.length === 0) {
+      console.log('ℹ️  No knowledge items found. Nothing to migrate.');
+      return;
+    }
+
+    let successCount = 0;
+    let skipCount = 0;
+    let errorCount = 0;
+
+    // Process each item
+    for (let i = 0; i < items.length; i++) {
+      const item = items[i];
+      const progress = `[${i + 1}/${items.length}]`;
+
+      try {
+        // Check if already processed (skip if chunks exist)
+        const existingChunks = await getChunkCountForKnowledgeItem(item.id);
+        if (existingChunks > 0) {
+          console.log(`${progress} ⏭️  Skipping ${item.id} (already has ${existingChunks} chunks)`);
+          skipCount++;
+          continue;
+        }
+
+        console.log(`${progress} 🔄 Processing ${item.id}...`);
+
+        // Chunk and embed
+        await writeKnowledgeChunksForItem(item);
+
+        const newChunks = await getChunkCountForKnowledgeItem(item.id);
+        console.log(`${progress} ✅ Success! Created ${newChunks} chunks`);
+        successCount++;
+
+        // Small delay to avoid rate limiting
+        if (i < items.length - 1) {
+          await new Promise((resolve) => setTimeout(resolve, 100));
+        }
+      } catch (error) {
+        console.error(`${progress} ❌ Failed to process ${item.id}:`, error);
+        errorCount++;
+      }
+    }
+
+    // Summary
+    console.log('');
+    console.log('='.repeat(60));
+    console.log('📊 Migration Complete');
+    console.log('='.repeat(60));
+    console.log(`✅ Processed: ${successCount}`);
+    console.log(`⏭️  Skipped (already exists): ${skipCount}`);
+    console.log(`❌ Errors: ${errorCount}`);
+    console.log(`📦 Total: ${items.length}`);
+    console.log('');
+
+    if (errorCount > 0) {
+      console.log('⚠️  Some items failed. Check logs above for details.');
+      process.exit(1);
+    } else {
+      console.log('🎉 All knowledge items successfully migrated to AlloyDB!');
+      process.exit(0);
+    }
+  } catch (error) {
+    console.error('');
+    console.error('❌ Migration failed:', error);
+    process.exit(1);
+  }
+}
+
+// Parse command line arguments
+const projectId = process.argv[2];
+
+if (projectId) {
+  console.log(`ℹ️  Processing single project: ${projectId}`);
+} else {
+  console.log('ℹ️  No projectId provided - processing ALL projects');
+}
+
+// Run migration
+migrateKnowledgeItems(projectId);
+