Added semantic layer, QDrant collection initialization implemented

packages/memory-service/src/index.js

@@ -3,6 +3,7 @@ const express = require('express');
 const {getEnv} = require('@nexusai/shared');
 const { getDB } = require('./db');
 const episodic = require('./episodic');
+const semantic = require('./semantic');
 
 const app = express();
 app.use(express.json());
@@ -12,6 +13,10 @@ const  PORT = getEnv('PORT', '3002');  // Default to 3002 if PORT is not set
 //initialize database on startup
 const db = getDB();
 
+semantic.initCollections()
+    .then(() => console.log(`QDrant collections ready`))
+    .catch(err => console.error(`QDrant initialization error:`, err.message));
+
 // Health check endpoint
 app.get('/health', (req, res) => {
     res.json({ service: 'Memory Service', status: 'healthy' });

packages/memory-service/src/semantic/index.js

@@ -0,0 +1,118 @@
+const {QdrantClient} = require('@qdrant/js-client-rest');
+const {getEnv} = require('@nexusai/shared');
+
+let client;
+
+// ********** QDrant Client **********
+function getClient(){
+    if(!client){
+        const url = getEnv('QDrant_URL', 'http://localhost:6333');
+        client = new QdrantClient({url});
+    }
+    return client;
+}
+
+// ********** Collections **********
+const COLLECTIONS = {
+    EPISODES: 'episodes',
+    ENTITIES: 'entities',
+    SUMMARIES: 'summaries'
+};
+
+// Vectoir size much match the embedding model output dimension
+// nomic-embed-text outputs 768 dimensions
+const VECTOR_SIZE = 768;
+
+async function initCollections() {
+  const client = getClient();
+  for (const name of Object.values(COLLECTIONS)) {
+    const exists = await collectionExists(name);
+    if (!exists) {
+      await client.createCollection(name, {
+        vectors: {
+          size: VECTOR_SIZE,
+          distance: 'Cosine'  // Cosine similarity — best for text embeddings
+        }
+      });
+      console.log(`Created Qdrant collection: ${name}`);
+    } else {
+      console.log(`Qdrant collection already exists: ${name}`);
+    }
+  }
+}
+
+async function collectionExists(name) {
+    try {
+        const client = getClient();
+        await client.getCollection(name);
+        return true;
+    } catch (err) {
+        return false;
+    }
+}
+
+// ********* Upsert Vectors *********
+
+//payload is metadata stored alongside the vector in QDrant.
+//SQLite ID stored here to be able to link back to original data
+async function upsertVector(collection, id, vector, payload = {}) {
+    const client = getClient();
+    await client.upsertVector(collection, {
+        wait: true,  // Wait for the operation to complete before returning
+        points: [{id, vector, payload}]
+    });
+}
+
+// Upsert an episode vector into the EPISODES collection, with the episode ID as the point ID
+async function upsertEpisode(id, vector, payload={}) {
+    return upsertVector(COLLECTIONS.EPISODES, id, vector, payload);
+}
+
+async function upsertEntity(id, vector, payload={}) {
+    return upsertVector(COLLECTIONS.ENTITIES, id, vector, payload);
+}
+
+async function upsertSummary(id, vector, payload={}) {
+    return upsertVector(COLLECTIONS.SUMMARIES, id, vector, payload);
+}
+
+//********** Search Vectors ********** */
+async function searchCollection(collection, vector, limit = 10, filter = null){
+    const client = getClient();
+    const params = {vector,limit, with_payload: true};
+
+    if (filter) params.filter = filter;
+
+    return client.search(collection, params);
+}
+async function searchEpisodes(vector, limit = 10, filter = null) {
+    return searchCollection(COLLECTIONS.EPISODES, vector, limit, filter);
+}
+async function searchEntities(vector, limit = 10, filter = null) {
+    return searchCollection(COLLECTIONS.ENTITIES, vector, limit, filter);
+}
+async function searchSummaries(vector, limit = 10, filter = null) {
+    return searchCollection(COLLECTIONS.SUMMARIES, vector, limit, filter);
+}
+
+//********** Delete Vectors ********** */
+
+async function deleteVector(collection, id) {
+    const client = getClient();
+    await client.delete(collection, {
+        wait: true,
+        points: [id]
+    });
+}
+
+module.exports = {
+    COLLECTIONS,
+    initCollections,
+    upsertEpisode,
+    upsertEntity,
+    upsertSummary,
+    searchEpisodes,
+    searchEntities,
+    searchSummaries,
+    deleteVector
+};