diff --git a/packages/memory-service/src/index.js b/packages/memory-service/src/index.js index 6d2ae42..0e25488 100644 --- a/packages/memory-service/src/index.js +++ b/packages/memory-service/src/index.js @@ -3,6 +3,7 @@ const express = require('express'); const {getEnv} = require('@nexusai/shared'); const { getDB } = require('./db'); const episodic = require('./episodic'); +const semantic = require('./semantic'); const app = express(); app.use(express.json()); @@ -12,6 +13,10 @@ const PORT = getEnv('PORT', '3002'); // Default to 3002 if PORT is not set //initialize database on startup const db = getDB(); +semantic.initCollections() + .then(() => console.log(`QDrant collections ready`)) + .catch(err => console.error(`QDrant initialization error:`, err.message)); + // Health check endpoint app.get('/health', (req, res) => { res.json({ service: 'Memory Service', status: 'healthy' }); diff --git a/packages/memory-service/src/semantic/index.js b/packages/memory-service/src/semantic/index.js new file mode 100644 index 0000000..de5ea60 --- /dev/null +++ b/packages/memory-service/src/semantic/index.js @@ -0,0 +1,118 @@ +const {QdrantClient} = require('@qdrant/js-client-rest'); +const {getEnv} = require('@nexusai/shared'); + +let client; + +// ********** QDrant Client ********** +function getClient(){ + if(!client){ + const url = getEnv('QDrant_URL', 'http://localhost:6333'); + client = new QdrantClient({url}); + } + return client; +} + +// ********** Collections ********** +const COLLECTIONS = { + EPISODES: 'episodes', + ENTITIES: 'entities', + SUMMARIES: 'summaries' +}; + +// Vectoir size much match the embedding model output dimension +// nomic-embed-text outputs 768 dimensions +const VECTOR_SIZE = 768; + +async function initCollections() { + const client = getClient(); + for (const name of Object.values(COLLECTIONS)) { + const exists = await collectionExists(name); + if (!exists) { + await client.createCollection(name, { + vectors: { + size: VECTOR_SIZE, + distance: 'Cosine' // Cosine similarity — best for text embeddings + } + }); + console.log(`Created Qdrant collection: ${name}`); + } else { + console.log(`Qdrant collection already exists: ${name}`); + } + } +} + +async function collectionExists(name) { + try { + const client = getClient(); + await client.getCollection(name); + return true; + } catch (err) { + return false; + } +} + +// ********* Upsert Vectors ********* + +//payload is metadata stored alongside the vector in QDrant. +//SQLite ID stored here to be able to link back to original data +async function upsertVector(collection, id, vector, payload = {}) { + const client = getClient(); + await client.upsertVector(collection, { + wait: true, // Wait for the operation to complete before returning + points: [{id, vector, payload}] + }); +} + +// Upsert an episode vector into the EPISODES collection, with the episode ID as the point ID +async function upsertEpisode(id, vector, payload={}) { + return upsertVector(COLLECTIONS.EPISODES, id, vector, payload); +} + +async function upsertEntity(id, vector, payload={}) { + return upsertVector(COLLECTIONS.ENTITIES, id, vector, payload); +} + +async function upsertSummary(id, vector, payload={}) { + return upsertVector(COLLECTIONS.SUMMARIES, id, vector, payload); +} + +//********** Search Vectors ********** */ +async function searchCollection(collection, vector, limit = 10, filter = null){ + const client = getClient(); + const params = {vector,limit, with_payload: true}; + + if (filter) params.filter = filter; + + return client.search(collection, params); +} +async function searchEpisodes(vector, limit = 10, filter = null) { + return searchCollection(COLLECTIONS.EPISODES, vector, limit, filter); +} +async function searchEntities(vector, limit = 10, filter = null) { + return searchCollection(COLLECTIONS.ENTITIES, vector, limit, filter); +} +async function searchSummaries(vector, limit = 10, filter = null) { + return searchCollection(COLLECTIONS.SUMMARIES, vector, limit, filter); +} + +//********** Delete Vectors ********** */ + +async function deleteVector(collection, id) { + const client = getClient(); + await client.delete(collection, { + wait: true, + points: [id] + }); +} + +module.exports = { + COLLECTIONS, + initCollections, + upsertEpisode, + upsertEntity, + upsertSummary, + searchEpisodes, + searchEntities, + searchSummaries, + deleteVector +}; \ No newline at end of file