opusdev/vector-similarity-api
1
1const express = require("express");2const cors = require("cors");3const path = require("path");4const axios = require("axios");5const crypto = require("crypto");6require("dotenv").config();7 8const Post = require("../models/post");9const { QdrantClient } = require("@qdrant/js-client-rest");10 11// ── BullMQ Queue system (Redis-backed) ────────────────────────────────────12const { initQueues, enqueueInterestUpdate, enqueueVisibilityUpdate, getInterestQueue, getVisibilityQueue } = require("./queues/interestQueue");13const { startWorkers } = require("./queues/interestWorker");14const { isRedisAvailable } = require("./queues/redisConnection");15 16const app = express();17const BASE_DIR = __dirname;18const FRONTEND_DIST = path.join(BASE_DIR, "..", "frontend", "dist");19 20app.use(express.json());21app.use(22 cors({23 origin: "*",24 credentials: true,25 methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"],26 allowedHeaders: ["*"],27 }),28);29app.use((req, res, next) => {30 res.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate');31 res.set('Pragma', 'no-cache');32 res.set('Expires', '0');33 next();34});35app.use("/assets", express.static(path.join(FRONTEND_DIST, "assets")));36 37const REQUIRED_ENV = ["QDRANT_URL", "QDRANT_API_KEY", "MONGO_URI", "HF_API_TOKEN"];38const missingEnv = REQUIRED_ENV.filter((k) => !process.env[k]);39if (missingEnv.length > 0) {40 console.error("\nMissing env vars:", missingEnv.join(", "));41 process.exit(1);42}43 44const qdrantClient = new QdrantClient({45 url: process.env.QDRANT_URL,46 apiKey: process.env.QDRANT_API_KEY,47 checkCompatibility: false,48});49 50const COLLECTION_NAME = "social_posts";51const LIKES_COLLECTION = "user_likes";52const EVENTS_COLLECTION = "user_events";53const COMMENTS_COLLECTION = "post_comments";54const SAVES_COLLECTION = "user_saved_posts";55const SHARES_COLLECTION = "post_shares";56const INTEREST_VECTORS_COLLECTION = "user_interest_vectors";57 58// Event weights for engagement scoring algorithm59const EVENT_WEIGHTS = {60 like: 0.2,61 comment: 0.3,62 share: 0.4,63 save: 0.5,64 watch_time: 0.6,65};66 67// Event types68const EVENT_TYPES = {69 LIKE: "like",70 COMMENT: "comment",71 SHARE: "share",72 SAVE: "save",73 WATCH_TIME: "watch_time",74};75 76const CATEGORY_MAPPING = {77 'post_001': 'nature', 'post_002': 'tech', 'post_003': 'healthcare', 'post_004': 'food', 'post_005': 'tech',78 'post_006': 'art', 'post_007': 'nature', 'post_008': 'education', 'post_009': 'tech', 'post_010': 'travel',79 'post_011': 'healthcare', 'post_012': 'art', 'post_013': 'travel', 'post_014': 'food', 'post_015': 'music',80 'post_016': 'tech', 'post_017': 'tech', 'post_018': 'tech', 'post_019': 'tech', 'post_020': 'tech',81 'post_021': 'tech', 'post_022': 'tech', 'post_023': 'tech', 'post_024': 'tech', 'post_025': 'tech',82 'post_026': 'food', 'post_027': 'food', 'post_028': 'food', 'post_029': 'food', 'post_030': 'food',83 'post_031': 'food', 'post_032': 'food', 'post_033': 'food', 'post_034': 'food', 'post_035': 'food',84 'post_036': 'travel', 'post_037': 'travel', 'post_038': 'travel', 'post_039': 'travel', 'post_040': 'travel',85 'post_041': 'travel', 'post_042': 'travel', 'post_043': 'travel', 'post_044': 'travel', 'post_045': 'travel',86 'post_046': 'sports', 'post_047': 'sports', 'post_048': 'sports', 'post_049': 'sports', 'post_050': 'sports',87 'post_051': 'sports', 'post_052': 'healthcare', 'post_053': 'sports', 'post_054': 'sports', 'post_055': 'sports',88 'post_056': 'art', 'post_057': 'art', 'post_058': 'art', 'post_059': 'art', 'post_060': 'art',89 'post_061': 'art', 'post_062': 'art', 'post_063': 'art', 'post_064': 'art', 'post_065': 'art',90 'post_066': 'tech', 'post_067': 'tech', 'post_068': 'tech', 'post_069': 'tech', 'post_070': 'tech',91 'post_071': 'tech', 'post_072': 'tech', 'post_073': 'tech', 'post_074': 'tech', 'post_075': 'tech',92 'post_076': 'ai', 'post_077': 'ai', 'post_078': 'ai', 'post_079': 'ai', 'post_080': 'ai',93 'post_081': 'ai', 'post_082': 'ai', 'post_083': 'ai', 'post_084': 'ai', 'post_085': 'ai',94 'post_086': 'healthcare', 'post_087': 'healthcare', 'post_088': 'healthcare', 'post_089': 'healthcare', 'post_090': 'healthcare',95 'post_091': 'healthcare', 'post_092': 'healthcare', 'post_093': 'healthcare', 'post_094': 'healthcare', 'post_095': 'healthcare',96 'post_096': 'web3', 'post_097': 'web3', 'post_098': 'web3', 'post_099': 'web3', 'post_100': 'web3',97 'post_101': 'web3', 'post_102': 'web3', 'post_103': 'web3', 'post_104': 'web3', 'post_105': 'web3',98 'post_106': 'socialmedia', 'post_107': 'socialmedia', 'post_108': 'socialmedia', 'post_109': 'socialmedia', 'post_110': 'socialmedia',99 'post_111': 'socialmedia', 'post_112': 'socialmedia', 'post_113': 'socialmedia', 'post_114': 'socialmedia', 'post_115': 'socialmedia',100 'post_116': 'food', 'post_117': 'food', 'post_118': 'food', 'post_119': 'food', 'post_120': 'food',101 'post_121': 'food', 'post_122': 'food', 'post_123': 'food', 'post_124': 'food', 'post_125': 'food',102 'post_126': 'sports', 'post_127': 'sports', 'post_128': 'sports', 'post_129': 'sports', 'post_130': 'sports',103 'post_131': 'sports', 'post_132': 'sports', 'post_133': 'sports', 'post_134': 'sports', 'post_135': 'sports',104 'post_136': 'finance', 'post_137': 'finance', 'post_138': 'finance', 'post_139': 'finance', 'post_140': 'finance',105 'post_141': 'finance', 'post_142': 'finance', 'post_143': 'finance', 'post_144': 'finance', 'post_145': 'finance',106 'post_146': 'movies', 'post_147': 'movies', 'post_148': 'movies', 'post_149': 'movies', 'post_150': 'movies',107 'post_151': 'movies', 'post_152': 'movies', 'post_153': 'movies', 'post_154': 'movies', 'post_155': 'movies',108 'post_156': 'music', 'post_157': 'music', 'post_158': 'music', 'post_159': 'music', 'post_160': 'music',109 'post_161': 'music', 'post_162': 'music', 'post_163': 'music', 'post_164': 'music', 'post_165': 'music',110 'post_166': 'education', 'post_167': 'education', 'post_168': 'education', 'post_169': 'education', 'post_170': 'education',111 'post_171': 'education', 'post_172': 'education', 'post_173': 'education', 'post_174': 'education', 'post_175': 'education',112 'post_176': 'nature', 'post_177': 'nature', 'post_178': 'nature', 'post_179': 'nature', 'post_180': 'nature',113 'post_181': 'nature', 'post_182': 'nature', 'post_183': 'nature', 'post_184': 'nature', 'post_185': 'nature',114 'post_186': 'stocks', 'post_187': 'stocks', 'post_188': 'stocks', 'post_189': 'stocks', 'post_190': 'stocks',115 'post_191': 'stocks', 'post_192': 'stocks', 'post_193': 'stocks', 'post_194': 'stocks', 'post_195': 'stocks',116 'post_196': 'vehicles', 'post_197': 'vehicles', 'post_198': 'vehicles', 'post_199': 'vehicles', 'post_200': 'vehicles',117 'post_201': 'vehicles', 'post_202': 'vehicles', 'post_203': 'vehicles', 'post_204': 'vehicles', 'post_205': 'vehicles',118 'post_206': 'cafes', 'post_207': 'cafes', 'post_208': 'cafes', 'post_209': 'cafes', 'post_210': 'cafes',119 'post_211': 'cafes', 'post_212': 'cafes', 'post_213': 'cafes', 'post_214': 'cafes', 'post_215': 'cafes'120};121 122// Auto-sync missing categories on start123(async () => {124 try {125 await Post.syncMissingCategories(CATEGORY_MAPPING);126 } catch (e) {127 console.error("Failed to sync categories:", e.message);128 }129})();130 131let _embedder = null;132const _embedderReady = (async () => {133 try {134 const { pipeline, env } = await import("@xenova/transformers");135 env.cacheDir = process.env.XENOVA_CACHE_DIR || "/app/.cache/xenova";136 env.localFilesOnly = true;137 console.log("Loading embedding model (Xenova/all-MiniLM-L6-v2)...");138 _embedder = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");139 console.log("✓ Embedding model ready (384-dim, in-process)");140 } catch (e) {141 console.error("✗ Embedding model failed to load:", e.message);142 }143})();144 145async function getEmbedding(text) {146 await _embedderReady;147 if (!_embedder) throw new Error("Embedding model unavailable — check build logs");148 const output = await _embedder(text, { pooling: "mean", normalize: true });149 return Array.from(output.data);150}151 152 153// Qdrant: ensure user_likes collection exists + payload index on user_id154(async () => {155 try {156 await qdrantClient.createCollection(LIKES_COLLECTION, {157 vectors: { size: 384, distance: "Cosine" },158 });159 console.log(`Created ${LIKES_COLLECTION}`);160 } catch {161 console.log(` ${LIKES_COLLECTION} exists`);162 }163 try {164 await qdrantClient.createPayloadIndex(LIKES_COLLECTION, {165 field_name: "user_id",166 field_schema: "keyword",167 });168 console.log(` [Index] ${LIKES_COLLECTION}.user_id ready`);169 } catch (e) {170 console.log(` [Index] ${LIKES_COLLECTION}.user_id exists`);171 }172})();173 174// Qdrant: ensure user_events collection exists + payload indexes for filtered queries175(async () => {176 try {177 await qdrantClient.createCollection(EVENTS_COLLECTION, {178 vectors: { size: 384, distance: "Cosine" },179 });180 console.log(`Created ${EVENTS_COLLECTION}`);181 } catch {182 console.log(` ${EVENTS_COLLECTION} exists`);183 }184 // Keyword indexes are required by Qdrant before any filtered scroll/search.185 // createPayloadIndex is idempotent — safe to call on every startup.186 for (const field of ["user_id", "post_id", "event_type"]) {187 try {188 await qdrantClient.createPayloadIndex(EVENTS_COLLECTION, {189 field_name: field,190 field_schema: "keyword",191 });192 console.log(` [Index] ${EVENTS_COLLECTION}.${field} ready`);193 } catch (e) {194 console.log(` [Index] ${EVENTS_COLLECTION}.${field} exists`);195 }196 }197})();198 199// Qdrant: ensure user_interest_vectors collection exists + payload index on user_id200(async () => {201 try {202 await qdrantClient.createCollection(INTEREST_VECTORS_COLLECTION, {203 vectors: { size: 384, distance: "Cosine" },204 });205 console.log(`Created ${INTEREST_VECTORS_COLLECTION}`);206 } catch {207 console.log(` ${INTEREST_VECTORS_COLLECTION} exists`);208 }209 try {210 await qdrantClient.createPayloadIndex(INTEREST_VECTORS_COLLECTION, {211 field_name: "user_id",212 field_schema: "keyword",213 });214 console.log(` [Index] ${INTEREST_VECTORS_COLLECTION}.user_id ready`);215 } catch (e) {216 console.log(` [Index] ${INTEREST_VECTORS_COLLECTION}.user_id exists`);217 }218})();219 220// Qdrant: ensure main social_posts collection exists for RAG queries221(async () => {222 try {223 await qdrantClient.createCollection(COLLECTION_NAME, {224 vectors: { size: 384, distance: "Cosine" },225 });226 console.log(`Created ${COLLECTION_NAME}`);227 } catch {228 console.log(` ${COLLECTION_NAME} exists`);229 }230 try {231 await qdrantClient.createPayloadIndex(COLLECTION_NAME, {232 field_name: "post_id",233 field_schema: "keyword",234 });235 console.log(` [Index] ${COLLECTION_NAME}.post_id ready`);236 } catch (e) {237 console.log(` [Index] ${COLLECTION_NAME}.post_id exists`);238 }239})();240 241// ── Start BullMQ queues + workers ─────────────────────────────────────────242(async () => {243 try {244 await initQueues();245 await startWorkers(qdrantClient);246 } catch (e) {247 console.warn("[Queue] Failed to start queues/workers:", e.message);248 }249})();250 251async function generateLLMAnswer(prompt) {252 const response = await axios.post(253 "https://api-inference.huggingface.co/models/mistralai/Mistral-7B-Instruct-v0.1/v1/chat/completions",254 {255 messages: [256 { role: "system", content: "You analyze social media posts concisely." },257 { role: "user", content: prompt },258 ],259 temperature: 0.5,260 max_tokens: 500,261 },262 {263 headers: {264 Authorization: `Bearer ${process.env.HF_API_TOKEN}`,265 "Content-Type": "application/json",266 },267 timeout: 60000,268 },269 );270 return response.data.choices[0].message.content.trim();271}272 273function shapePost(payload, score, source, meta = {}) {274 return {275 post_id: payload.post_id,276 name: payload.name,277 caption: payload.caption,278 media_url: payload.media_url || "",279 media_type: payload.media_type || "image",280 category: payload.category || "unknown",281 similarity_score: Math.round(score * 10000) / 10000,282 similarity_percentage:283 source === "random" ? "Random" : `${Math.round(score * 10000) / 100}%`,284 source,285 ...meta,286 };287}288 289 290const RANK_LABELS = [291 "primary",292 "secondary",293 "tertiary",294 "quaternary",295 "quinary",296 "senary",297 "septenary",298 "octonary",299];300 301// INTEREST TIER THRESHOLDS302const INTEREST_TIER_CRITERIA = {303 PRIMARY: 6, 304 SECONDARY: 3, 305 TERTIARY: 1, 306};307 308const DECAY_LAMBDA = 0.05;309 310function likeWeight(timestamp, posInBucket, bucketSize) {311 const ageHours =312 (Date.now() - new Date(timestamp).getTime()) / (1000 * 60 * 60);313 const recency = Math.exp(-DECAY_LAMBDA * ageHours);314 const position = 1 - (posInBucket / Math.max(bucketSize, 1)) * 0.5;315 return recency * position;316}317 318// Calculate event weight based on event type319function getEventWeight(eventType) {320 return EVENT_WEIGHTS[eventType] || EVENT_WEIGHTS.like;321}322 323 324function categorizeInterestTiers(rankedInterests) {325 if (!rankedInterests || rankedInterests.length === 0) return [];326 327 328 const primaryBucket = []; 329 const secondaryBucket = []; 330 const tertiaryBucket = []; 331 332 rankedInterests.forEach((interest, originalIdx) => {333 const tierData = {334 ...interest,335 tier: null,336 tier_idx: 0,337 original_rank_idx: originalIdx,338 };339 340 if (interest.count >= INTEREST_TIER_CRITERIA.PRIMARY) {341 tierData.tier = "PRIMARY";342 primaryBucket.push(tierData);343 } else if (interest.count >= INTEREST_TIER_CRITERIA.SECONDARY) {344 tierData.tier = "SECONDARY";345 secondaryBucket.push(tierData);346 } else {347 tierData.tier = "TERTIARY";348 tertiaryBucket.push(tierData);349 }350 });351 352 // Assign tier_idx within each bucket (based on original ranking order preserved)353 const tieredInterests = [];354 355 primaryBucket.forEach((item, idx) => {356 item.tier_idx = idx;357 item.rank = "primary";358 tieredInterests.push(item);359 });360 361 secondaryBucket.forEach((item, idx) => {362 item.tier_idx = idx;363 item.rank = "secondary";364 tieredInterests.push(item);365 });366 367 tertiaryBucket.forEach((item, idx) => {368 item.tier_idx = idx;369 // Preserve original rank labels for tertiary and below370 item.rank = RANK_LABELS[idx] || `rank_${idx + 1}`;371 tieredInterests.push(item);372 });373 374 return tieredInterests;375}376 377// Build ranked interest buckets from session events with weighted scoring378function computeInterestRanking(sessionEvents) {379 if (!sessionEvents || sessionEvents.length === 0) return [];380 381 const buckets = {};382 sessionEvents.forEach((ev, idx) => {383 let key = (ev.category || "").toLowerCase().trim();384 if (!key || key === "unknown") key = `_anon_${ev.post_id}`;385 if (!buckets[key]) buckets[key] = { category: key, events: [] };386 buckets[key].events.push({ ...ev, seqIdx: idx });387 });388 389 const now = Date.now();390 const scored = Object.values(buckets).map((bucket) => {391 const n = bucket.events.length;392 let totalScore = 0;393 bucket.events.forEach((ev, posInBucket) => {394 // Get base weight from recency and position395 const baseWeight = likeWeight(ev.timestamp, posInBucket, n);396 // Apply event type multiplier (default to like if not specified)397 const eventTypeWeight = getEventWeight(ev.event_type || EVENT_TYPES.LIKE);398 // Total weight considers both recency/position AND event importance399 totalScore += baseWeight * eventTypeWeight;400 });401 return {402 category: bucket.category,403 events: bucket.events,404 post_ids: bucket.events.map((e) => e.post_id),405 count: n,406 score: Math.round(totalScore * 1000) / 1000,407 weighted_score: Math.round(totalScore * 1000) / 1000,408 lastSeen: Math.max(409 ...bucket.events.map((e) => new Date(e.timestamp).getTime()),410 ),411 };412 });413 414 scored.sort((a, b) =>415 b.score !== a.score ? b.score - a.score : b.lastSeen - a.lastSeen,416 );417 418 const rankedInterests = scored.map((bucket, idx) => ({419 category: bucket.category.startsWith("_anon_") ? "liked" : bucket.category,420 events: bucket.events,421 post_ids: bucket.post_ids,422 count: bucket.count,423 score: bucket.score,424 weighted_score: bucket.weighted_score,425 rank: RANK_LABELS[idx] || `rank_${idx + 1}`,426 rank_idx: idx,427 }));428 429 // APPLY TIER-BASED CATEGORIZATION (PRIMARY >= 6, SECONDARY >= 4)430 return categorizeInterestTiers(rankedInterests);431} 432function computeBudget(rankedInterests) {433 const RANDOM = 2;434 const MAX_INT = 3;435 const iSlots = Math.min(rankedInterests.length, MAX_INT);436 return { query: 10 - RANDOM - iSlots, interest: iSlots, random: RANDOM };437}438 439async function scrollAllPosts() {440 let all = [],441 offset = null;442 do {443 const r = await qdrantClient.scroll(COLLECTION_NAME, {444 limit: 250,445 offset,446 with_vector: false,447 with_payload: true,448 });449 all.push(...(r.points || []));450 offset = r.next_page_offset;451 } while (offset != null);452 return all;453}454 455const ALL_POSTS_CACHE_TTL_MS = parseInt(456 process.env.ALL_POSTS_CACHE_TTL_MS || "30000",457 10,458);459let allPostsCache = {460 data: null,461 expiresAt: 0,462};463 464function invalidateAllPostsCache() {465 allPostsCache = { data: null, expiresAt: 0 };466}467 468async function getAllPostsCached() {469 const now = Date.now();470 if (allPostsCache.data && allPostsCache.expiresAt > now) {471 return allPostsCache.data;472 }473 474 const all = await scrollAllPosts();475 allPostsCache = {476 data: all,477 expiresAt: now + ALL_POSTS_CACHE_TTL_MS,478 };479 return all;480}481 482function pickRandomPosts(allPosts, usedPostIds, count) {483 if (count <= 0) return [];484 const pool = allPosts.filter(485 (p) => p.payload?.post_id && !usedPostIds.has(p.payload.post_id),486 );487 // shuffler488 for (let i = pool.length - 1; i > 0; i--) {489 const j = Math.floor(Math.random() * (i + 1));490 [pool[i], pool[j]] = [pool[j], pool[i]];491 }492 return pool.slice(0, count).map((p) => {493 usedPostIds.add(p.payload.post_id);494 return shapePost(p.payload, 0, "random");495 });496}497 498async function buildInterestPosts(rankedInterests, usedPostIds, slotCount) {499 if (slotCount <= 0 || !rankedInterests.length) return [];500 const results = [];501 502 for (const interest of rankedInterests.slice(0, slotCount)) {503 console.log(504 ` [${interest.rank}] cat="${interest.category}" n=${interest.count} score=${interest.score}`,505 );506 507 const nums = interest.post_ids508 .map((pid) => parseInt(pid.replace("post_", "")))509 .filter((n) => !isNaN(n));510 511 //vectrRetieveal512 let retrieved = [];513 try {514 retrieved = await qdrantClient.retrieve(COLLECTION_NAME, {515 ids: nums,516 with_vector: true,517 with_payload: false,518 });519 } catch (e) {520 console.log(` retrieve failed: ${e.message}`);521 }522 523 const items = retrieved524 .map((r) => {525 if (!Array.isArray(r.vector) || !r.vector.length) return null;526 const postId = `post_${r.id}`;527 const evIdx = interest.events.findIndex((e) => e.post_id === postId);528 const weight =529 evIdx >= 0530 ? likeWeight(531 interest.events[evIdx].timestamp,532 evIdx,533 interest.events.length,534 )535 : 0.5;536 return { vector: r.vector, weight };537 })538 .filter(Boolean);539 540 if (!items.length) {541 console.log(` no vectors — collapse`);542 results.push(null);543 continue;544 }545 546 547 548 const totalW = items.reduce((s, v) => s + v.weight, 0);549 const dim = items[0].vector.length;550 const centroid = new Array(dim).fill(0);551 items.forEach(({ vector, weight }) => {552 vector.forEach((val, i) => {553 centroid[i] += val * (weight / totalW);554 });555 });556 557 // relvance search — skipe used n liked post558 let hits = [];559 try {560 hits = await qdrantClient.search(COLLECTION_NAME, {561 vector: centroid,562 limit: 100,563 score_threshold: 0.0,564 });565 } catch (e) {566 console.log(` centroid search failed: ${e.message}`);567 results.push(null);568 continue;569 }570 571 let filled = false;572 for (const hit of hits) {573 const pid = hit.payload?.post_id;574 if (!pid || usedPostIds.has(pid) || interest.post_ids.includes(pid))575 continue;576 usedPostIds.add(pid);577 results.push(578 shapePost(hit.payload, hit.score, "interest", {579 interest_rank: interest.rank,580 interest_rank_idx: interest.rank_idx,581 interest_category: interest.category,582 interest_count: interest.count,583 interest_score: interest.score,584 }),585 );586 console.log(`picked ${pid} score=${hit.score.toFixed(4)}`);587 filled = true;588 break;589 }590 if (!filled) {591 console.log(`no post — collapse to query`);592 results.push(null);593 }594 }595 return results;596}597 598async function buildQueryPosts(599 queryEmbedding,600 usedPostIds,601 needed,602 minScore = 0.0,603) {604 if (needed <= 0) return [];605 let hits = [];606 try {607 hits = await qdrantClient.search(COLLECTION_NAME, {608 vector: queryEmbedding,609 limit: needed + 100,610 score_threshold: minScore,611 });612 } catch (e) {613 console.error("Query search failed:", e.message);614 return [];615 }616 const posts = [];617 for (const r of hits) {618 if (posts.length >= needed) break;619 const pid = r.payload?.post_id;620 if (!pid || usedPostIds.has(pid)) continue;621 posts.push(shapePost(r.payload, r.score, "query"));622 usedPostIds.add(pid);623 }624 return posts;625}626 627// postjumbleness 628function assembleFeed(queryPosts, interestPosts, randomPosts) {629 const base = [...queryPosts];630 const anchors = [2, 4, 6];631 interestPosts.forEach((ip, idx) => {632 const pos = anchors[idx] !== undefined ? anchors[idx] : idx * 2 + 2;633 base.splice(Math.min(pos, base.length), 0, ip);634 });635 const result = [...base];636 const total = result.length + randomPosts.length;637 randomPosts.forEach((rp, i) => {638 const pos = Math.floor(((i + 1) / (randomPosts.length + 1)) * total);639 result.splice(Math.min(pos, result.length), 0, rp);640 });641 return result;642}643 644// prompt and template for ai 645const RAG_SYSTEM_PROMPT = `You are an intelligent AI assistant specialized in analyzing social media posts and providing insightful answers. 646 647BEHAVIOR GUIDELINES:6481. For KEY INSIGHTS: Ground your analysis in the provided context (social media posts)649 - Extract key insights directly from source posts650 - Reference post authors and content651 - Cite specific data points from sources652 6532. For AI PERSPECTIVE: Provide INDEPENDENT analytical viewpoint654 - Do NOT repeat or summarize source data655 - Do NOT cite the provided posts656 - Focus solely on the query topic itself657 - Offer your own reasoning about the query658 - Discuss implications, risks, opportunities based on the query topic659 - Challenge or expand on the topic from fresh angles660 6613. Maintain professional yet conversational tone6624. Be honest about limitations6635. Clearly separate source-based insights from independent perspective`;664 665const RAG_ANALYSIS_PROMPT = (question, contextBlocks, sourceData) => `666TASK: Analyze the following social media posts and provide TWO different types of answers to the user's query.667 668USER QUERY: "${question}"669 670CONTEXT POSTS (for KEY INSIGHTS ONLY):671${contextBlocks.join("\n---\n")}672 673RESPONSE REQUIREMENTS:674You must provide your response in JSON format with the following structure:675{676 "key_insights": [677 {"point": "insight statement", "post_reference": "post_id or author name", "explanation": "explanation grounded in source data"},678 ... (exactly 5 points)679 ],680 "ai_perspective": "Write a comprehensive 6-line paragraph expressing YOUR INDEPENDENT perspective and analysis on the query topic itself. Do NOT reference the provided posts. Do NOT summarize source data. Instead, analyze the query topic from your own reasoning and provide insights that stand alone.",681 "summary": "2-3 sentence executive summary of findings"682}683 684IMPORTANT DISTINCTION:685 686KEY INSIGHTS (Based on provided posts):687- Must be grounded in the source posts provided688- Frame using: "According to the posts...", "The data suggests...", "Based on analysis..."689- Always reference the source post or author690- Extract specific information from the posts691- Make connections between multiple sources when relevant692 693AI PERSPECTIVE (Independent Analysis):694- Do NOT look at or reference the provided posts695- Analyze the QUERY TOPIC itself696- Provide your own reasoning about the topic697- Discuss implications, risks, benefits of the topic698- Challenge assumptions in the query699- Offer forward-thinking analysis700- Example: If query is "Is pizza and coke a deadly combo?"701 → Your perspective discusses nutrition science, body chemistry, health impacts702 → NOT what social media posts said about pizza and coke703 → Your own independent analysis of why/why not it's dangerous704 705GUIDELINES FOR KEY INSIGHTS:706- Each insight must be distinct and add unique value707- Extract from source posts only708- Always reference the source post or author709- Make connections between multiple sources when relevant710- Use data-grounded framing711 712GUIDELINES FOR AI PERSPECTIVE:713- Write in first person714- Discuss the query topic deeply from your analytical viewpoint715- Do NOT cite or reference the provided posts at all716- Offer independent reasoning and expert analysis717- Include 2-3 substantive points about the topic718- Express appropriate uncertainty where relevant719- Make this completely independent from key_insights720 721Format your response ONLY as valid JSON, no additional text.`;722 723async function generateStructuredRAGAnswer(question, contextBlocks, sourceData) {724 try {725 const prompt = RAG_ANALYSIS_PROMPT(question, contextBlocks, sourceData);726 727 const response = await axios.post(728 "https://api-inference.huggingface.co/models/mistralai/Mistral-7B-Instruct-v0.1/v1/chat/completions",729 {730 messages: [731 {732 role: "system",733 content: RAG_SYSTEM_PROMPT734 },735 {736 role: "user",737 content: prompt738 },739 ],740 temperature: 0.7,741 max_tokens: 800,742 },743 {744 headers: {745 Authorization: `Bearer ${process.env.HF_API_TOKEN}`,746 "Content-Type": "application/json",747 },748 timeout: 60000,749 },750 );751 752 const content = response.data.choices[0].message.content.trim();753 754 // Parse JSON response755 const jsonMatch = content.match(/\{[\s\S]*\}/);756 if (!jsonMatch) {757 throw new Error("Invalid JSON response from LLM");758 }759 760 return JSON.parse(jsonMatch[0]);761 } catch (error) {762 console.error("Error generating structured RAG answer:", error.message);763 throw error;764 }765}766 767app.post("/rag", async (req, res) => {768 try {769 const { question, limit = 5, min_score = 0.1, user_id = "default_user" } = req.body;770 771 if (!question?.trim()) {772 return res.status(400).json({ detail: "Question required" });773 }774 775 console.log(`\n${"═".repeat(60)}`);776 console.log(`RAG REQUEST: "${question}" | user=${user_id} | limit=${limit}`);777 778 // Step 1: Get embeddings for the question779 const questionEmbedding = await getEmbedding(question);780 console.log(`✓ Got embedding, dim=${questionEmbedding.length}`);781 782 // Step 2: Check collection status783 try {784 const collectionInfo = await qdrantClient.getCollection(COLLECTION_NAME);785 console.log(`✓ Collection exists: ${collectionInfo.points_count} points`);786 if (collectionInfo.points_count === 0) {787 return res.status(500).json({788 detail: "No data",789 hint: "Collection is empty. Please seed posts first."790 });791 }792 } catch (collErr) {793 console.error(`✗ Collection check failed:`, collErr.message);794 return res.status(500).json({795 detail: "Collection error",796 hint: `Collection not found or inaccessible: ${collErr.message}`797 });798 }799 800 // Step 3: Search for relevant posts801 const hits = await qdrantClient.search(COLLECTION_NAME, {802 vector: questionEmbedding,803 limit: Math.max(limit, 5),804 score_threshold: min_score,805 });806 console.log(`✓ Search returned ${hits?.length || 0} hits`);807 808 if (!hits?.length) {809 return res.json({810 query: question,811 status: "no_relevant_content",812 answer: "I apologize, but I couldn't find relevant posts to answer your question. Please try a different search term.",813 key_insights: [],814 ai_perspective: "The current database doesn't contain sufficient information to provide a meaningful analysis.",815 featured_image: null,816 source_posts: [],817 });818 }819 820 // Step 4: Prepare context blocks and source data821 const contextBlocks = [];822 const sourceData = [];823 const allSourcePosts = [];824 825 hits.forEach((hit, index) => {826 const {827 caption = "",828 name = "",829 post_id = "",830 media_url = "",831 media_type = "image",832 category = "unknown"833 } = hit.payload;834 835 // Build context for LLM836 const contextBlock = `[Post ${index + 1}] Author: ${name} | Category: ${category}\nContent: ${caption}`;837 contextBlocks.push(contextBlock);838 839 // Store source data for reference840 sourceData.push({841 post_id,842 name,843 caption,844 media_url,845 media_type,846 category,847 similarity_score: Math.round(hit.score * 10000) / 10000,848 similarity_percentage: `${Math.round(hit.score * 10000) / 100}%`,849 });850 851 // Add to all posts (we'll select first as featured, rest for bottom)852 allSourcePosts.push({853 post_id,854 name,855 caption,856 media_url,857 media_type,858 category,859 similarity_score: Math.round(hit.score * 10000) / 10000,860 });861 });862 863 864 let analysisResult;865 try {866 console.log(`→ Calling GROQ with ${contextBlocks.length} context blocks...`);867 analysisResult = await generateStructuredRAGAnswer(question, contextBlocks, sourceData);868 console.log(`✓ RAG Analysis generated successfully`);869 } catch (error) {870 console.error("Failed to generate structured analysis:", error.message);871 // Fallback response872 analysisResult = {873 key_insights: contextBlocks.slice(0, 5).map((block, i) => ({874 point: `Post insight ${i + 1}`,875 post_reference: sourceData[i]?.name || "Unknown",876 explanation: sourceData[i]?.caption || block,877 })),878 ai_perspective: "The provided posts contain relevant information, but I encountered a limitation in generating a detailed perspective. Please review the source posts directly for comprehensive analysis.",879 summary: "Analysis based on relevant social media posts.",880 };881 }882 const featuredImage = allSourcePosts[0]?.media_url || null;883 const sourcePosts = allSourcePosts.slice(0, 5).map((post) => ({884 post_id: post.post_id,885 name: post.name,886 caption: post.caption,887 media_url: post.media_url,888 media_type: post.media_type,889 category: post.category,890 similarity_score: post.similarity_score,891 }));892 893 const response = {894 query: question,895 user_id,896 status: "success",897 timestamp: new Date().toISOString(),898 899 summary: analysisResult.summary || "Analysis complete",900 901 featured_image: {902 url: featuredImage,903 source: allSourcePosts[0]?.name || "Top result",904 post_id: allSourcePosts[0]?.post_id || "",905 },906 key_insights: analysisResult.key_insights?.slice(0, 5).map((insight, idx) => ({907 rank: idx + 1,908 point: insight.point || insight.explanation,909 post_reference: insight.post_reference,910 explanation: insight.explanation,911 source_post: sourceData[idx] || null,912 })) || [],913 914 ai_perspective: analysisResult.ai_perspective || "Unable to generate perspective",915 source_posts: sourcePosts,916 metadata: {917 total_relevant_posts: hits.length,918 posts_analyzed: contextBlocks.length,919 min_similarity_score: min_score,920 top_similarity: sourceData[0]?.similarity_score || 0,921 },922 };923 924 console.log(`RAG RESPONSE: ${response.key_insights.length} insights | ${sourcePosts.length} posts`);925 console.log(`${"═".repeat(60)}\n`);926 927 res.json(response);928 } catch (error) {929 console.error("\n✗ RAG ERROR:", error.message);930 if (error.response?.data) {931 console.error("Response data:", JSON.stringify(error.response.data, null, 2));932 }933 if (error.code) {934 console.error("Error code:", error.code);935 }936 const errorDetail = error.response?.data?.detail || error.response?.data || error.message;937 const hint = error.response?.data?.hint || "Check logs for details. Verify GROQ_API_KEY is set and not expired.";938 console.error(`${"═".repeat(60)}\n`);939 res.status(500).json({940 detail: typeof errorDetail === "string" ? errorDetail : JSON.stringify(errorDetail),941 hint942 });943 }944});945 946// Seed endpoint: populates Qdrant with sample posts947app.post("/seed", async (req, res) => {948 try {949 console.log("\n" + "═".repeat(60));950 console.log("SEED REQUEST: Seeding posts to Qdrant...");951 952 const SAMPLE_POSTS = [953 { post_id: "post_001", name: "Alice", caption: "Beautiful sunset at the mountain", media_type: "image" },954 { post_id: "post_002", name: "Bob", caption: "Latest AI technology breakthrough", media_type: "video" },955 { post_id: "post_003", name: "Carol", caption: "Tips for healthy living and fitness", media_type: "image" },956 { post_id: "post_004", name: "David", caption: "Recipe for delicious chocolate cake", media_type: "image" },957 { post_id: "post_005", name: "Eve", caption: "Machine learning in production systems", media_type: "text" },958 { post_id: "post_006", name: "Frank", caption: "Abstract art gallery exhibition", media_type: "image" },959 { post_id: "post_007", name: "Grace", caption: "Travel guide to Paris and France", media_type: "image" },960 { post_id: "post_008", name: "Henry", caption: "Educational content on blockchain", media_type: "video" },961 { post_id: "post_009", name: "Iris", caption: "Deep learning neural networks explained", media_type: "text" },962 { post_id: "post_010", name: "Jack", caption: "World travel photography collection", media_type: "image" },963 ];964 965 let seededCount = 0;966 for (const post of SAMPLE_POSTS) {967 try {968 const embedding = await getEmbedding(`${post.name}: ${post.caption}`);969 970 const point = {971 id: seededCount + 1,972 vector: embedding,973 payload: {974 post_id: post.post_id,975 name: post.name,976 caption: post.caption,977 media_url: `https://via.placeholder.com/400?text=${encodeURIComponent(post.post_id)}`,978 media_type: post.media_type,979 category: "general"980 }981 };982 983 await qdrantClient.upsert(COLLECTION_NAME, {984 points: [point]985 });986 987 seededCount++;988 console.log(` ✓ Seeded ${post.post_id}`);989 } catch (e) {990 console.error(` ✗ Failed to seed ${post.post_id}:`, e.message);991 }992 }993 994 console.log(`✓ Seeded ${seededCount}/${SAMPLE_POSTS.length} posts`);995 console.log("═".repeat(60) + "\n");996 997 res.json({998 status: "success",999 message: `Seeded ${seededCount} sample posts to collection '${COLLECTION_NAME}'`,1000 count: seededCount1001 });1002 } catch (error) {1003 console.error("SEED ERROR:", error.message);1004 res.status(500).json({1005 detail: error.message,1006 hint: "Failed to seed posts. Check Qdrant connection."1007 });1008 }1009});1010 1011// Health check endpoint1012app.get("/health", async (req, res) => {1013 try {1014 const health = {};1015 1016 // Check embedder1017 try {1018 const testEmbedding = await getEmbedding("test");1019 health.embedder = { status: "healthy", dimension: testEmbedding.length };1020 } catch (e) {1021 health.embedder = { status: "error", message: e.message };1022 }1023 1024 // Check Qdrant1025 try {1026 const collInfo = await qdrantClient.getCollection(COLLECTION_NAME);1027 health.qdrant = { status: "healthy", collection: COLLECTION_NAME, points: collInfo.points_count };1028 } catch (e) {1029 health.qdrant = { status: "error", message: e.message };1030 }1031 1032 // Check HF Inference API1033 try {1034 const hfCheck = await axios.post(1035 "https://api-inference.huggingface.co/models/mistralai/Mistral-7B-Instruct-v0.1/v1/chat/completions",1036 {1037 messages: [{ role: "user", content: "ping" }],1038 max_tokens: 10,1039 },1040 {1041 headers: {1042 Authorization: `Bearer ${process.env.HF_API_TOKEN}`,1043 "Content-Type": "application/json",1044 },1045 timeout: 10000,1046 }1047 );1048 health.llm = { status: "healthy", provider: "Hugging Face Inference API" };1049 } catch (e) {1050 health.llm = { status: "error", message: e.response?.status === 429 ? "Rate limited" : e.message };1051 }1052 1053 // Overall status1054 const allHealthy = Object.values(health).every(h => h.status === "healthy");1055 1056 res.json({1057 overall: allHealthy ? "healthy" : "degraded",1058 components: health,1059 });1060 } catch (e) {1061 res.status(500).json({ error: e.message });1062 }1063});1064 1065// routes handler1066app.get("/api", (req, res) =>1067 res.json({ message: "Smart Ranking API", status: "running" }),1068);1069app.get("/", (req, res) =>1070 res.sendFile(path.join(FRONTEND_DIST, "index.html")),1071);1072 1073app.get("/random", async (req, res) => {1074 try {1075 const count = Math.min(parseInt(req.query.count) || 12, 50);1076 const all = await getAllPostsCached();1077 for (let i = all.length - 1; i > 0; i--) {1078 const j = Math.floor(Math.random() * (i + 1));1079 [all[i], all[j]] = [all[j], all[i]];1080 }1081 res.json({1082 total: count,1083 posts: all.slice(0, count).map((p) => ({1084 post_id: p.payload.post_id,1085 name: p.payload.name,1086 caption: p.payload.caption,1087 media_url: p.payload.media_url || "",1088 media_type: p.payload.media_type || "image",1089 category: p.payload.category || CATEGORY_MAPPING[p.payload.post_id] || "unknown",1090 })),1091 });1092 } catch (e) {1093 res.status(500).json({ detail: e.message });1094 }1095});1096 1097app.get("/posts", async (req, res) => {1098 try {1099 const posts = await Post.getAllPosts();1100 posts.forEach((p) => {1101 p._id = p._id.toString();1102 p.created_at = p.created_at.toISOString();1103 // Ensure category is always present from master mapping1104 if (!p.category || p.category === "unknown") {1105 p.category = CATEGORY_MAPPING[p.post_id] || "unknown";1106 }1107 });1108 res.json({ total: posts.length, posts });1109 } catch (e) {1110 res.status(500).json({ detail: e.message });1111 }1112});1113app.get("/posts/:post_id", async (req, res) => {1114 try {1115 const post = await Post.getPost(req.params.post_id);1116 if (!post) return res.status(404).json({ detail: "Post not found" });1117 post._id = post._id.toString();1118 post.created_at = post.created_at.toISOString();1119 res.json(post);1120 } catch (e) {1121 res.status(500).json({ detail: e.message });1122 }1123});1124app.post("/posts", async (req, res) => {1125 try {1126 const {1127 post_id,1128 name,1129 caption,1130 media_url,1131 media_type = "image",1132 category = "unknown",1133 } = req.body;1134 if (1135 !post_id.startsWith("post_") ||1136 !post_id.replace("post_", "").match(/^\d+$/)1137 )1138 return res1139 .status(400)1140 .json({ detail: "post_id must be in format 'post_001'" });1141 if (await Post.getPost(post_id))1142 return res.status(400).json({ detail: "Post already exists" });1143 await Post.createPost({ post_id, name, caption, media_url, media_type, category });1144 const embedding = await getEmbedding(`${name}: ${caption}`);1145 const postIdNum = parseInt(post_id.replace("post_", ""));1146 await qdrantClient.upsert(COLLECTION_NAME, {1147 wait: true,1148 points: [1149 {1150 id: postIdNum,1151 vector: embedding,1152 payload: {1153 post_id,1154 name,1155 caption,1156 media_url,1157 media_type,1158 category,1159 created_at: new Date().toISOString(),1160 },1161 },1162 ],1163 });1164 invalidateAllPostsCache();1165 res.status(201).json({ message: "Post created", post_id });1166 } catch (e) {1167 res.status(500).json({ detail: e.message });1168 }1169});1170 1171// Create post with auto-generated post_id (NEW FEATURE)1172app.post("/posts/create", async (req, res) => {1173 try {1174 const {1175 name,1176 caption,1177 media_url,1178 media_type,1179 category,1180 } = req.body;1181 1182 // Validate all required fields1183 if (!name || !name.trim()) {1184 return res.status(400).json({ detail: "Name is required" });1185 }1186 if (!caption || !caption.trim()) {1187 return res.status(400).json({ detail: "Caption is required" });1188 }1189 if (!media_url || !media_url.trim()) {1190 return res.status(400).json({ detail: "Media URL is required" });1191 }1192 if (!media_type || !media_type.trim()) {1193 return res.status(400).json({ detail: "Media type is required" });1194 }1195 if (!category || !category.trim()) {1196 return res.status(400).json({ detail: "Category is required" });1197 }1198 1199 // Get all posts to find the max post_id number1200 const allPosts = await Post.getAllPosts();