src/pages/api/docHandle.ts

import type { NextApiRequest, NextApiResponse } from 'next';

import {
  readHNSWLibModelFromLocal,
  storesDir,
  vectorStoreToHNSWLibModel,
} from '@/utils/file-handler';
import fs from 'fs-extra';
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
import { HNSWLib } from 'langchain/vectorstores/hnswlib';
import { XenovaTransformersEmbeddings } from '../../embed/hf'

async function handleDocs(text: string) {
  const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
  const docs = await textSplitter.createDocuments([text]);
  console.log(docs);

  const vectorStore = await HNSWLib.fromDocuments(docs, new XenovaTransformersEmbeddings());
  console.log(vectorStore);

  return vectorStore;
}

export default async function handler(
  req: NextApiRequest,
  res: NextApiResponse,
) {
  const { text } = JSON.parse(req.body);
  // console.log(text);

  if (!text) {
    return res.status(400).json({ message: 'No question in the request' });
  }

  const exists = await fs.exists(storesDir);
  console.log(exists);

  if (exists) {
    console.log('read from ' + storesDir);
    const model = await readHNSWLibModelFromLocal();
    return res.status(200).send({
      ...model,
    });
  }

  const vectorStore = await handleDocs(text);
  const model = await vectorStoreToHNSWLibModel(vectorStore);
  res.status(200).send({
    ...model,
  });
}

export const config = {
  api: {
    bodyParser: true, // Disallow body parsing, consume as stream
  },
};