packages/omni-server/src/utils/llm.js

/**
 * Copyright (c) 2023 MERCENARIES.AI PTE. LTD.
 * All rights reserved.
 */

//@ts-check

import path from 'path';
// @ts-ignore
import {
  walkDirForExtension,
  validateDirectoryExists,
  validateFileExists,
  readJsonFromDisk
} from './files.js';
import { is_valid, console_log, pauseForSeconds } from './utils.js';

const DEFAULT_UNKNOWN_CONTEXT_SIZE = 2048;
const MODELS_DIR_JSON_PATH = ['..', '..', 'user_files', 'local_llms_directories.json']; // from process.cwd(), which is ./packages/server/

// @ts-ignore
function generateModelId(model_name, model_provider) {
  return `${model_name}|${model_provider}`;
}

// @ts-ignore
function getModelNameAndProviderFromId(model_id) {
  if (!model_id) throw new Error(`getModelNameAndProviderFromId: model_id is not valid: ${model_id}`);
  const splits = model_id.split('|');
  if (splits.length !== 2) throw new Error(`splitModelNameFromType: model_id is not valid: ${model_id}`);
  return { model_name: splits[0], model_provider: splits[1] };
}

// @ts-ignore
async function isProviderAvailable(model_provider) {
  const models_dir_json = await getModelsDirJson();
  if (!models_dir_json) return false;

  const provider_model_dir = models_dir_json[model_provider];
  if (!provider_model_dir) return false;

  const dir_exists = await validateDirectoryExists(provider_model_dir);
  if (!dir_exists) return false;

  return true;
}

// @ts-ignore
async function addLocalLlmChoices(choices, llm_model_types, llm_context_sizes, model_type, model_provider) {
  const models_dir_json = await getModelsDirJson();
  if (!models_dir_json) return;

  const provider_model_dir = models_dir_json[model_provider];
  if (!provider_model_dir) return;

  const dir_exists = await validateDirectoryExists(provider_model_dir);
  if (!dir_exists) return;

  // @ts-ignore
  let filePaths = [];
  // @ts-ignore
  filePaths = await walkDirForExtension(filePaths, provider_model_dir, '.bin');

  for (const filepath of filePaths) {
    const name = path.basename(filepath);
    const id = generateModelId(name, model_provider);
    const title = deduceLlmTitle(name, model_provider);
    const description = deduceLlmDescription(name);
    const choice = { value: id, title, description };

    llm_model_types[name] = model_type;
    llm_context_sizes[name] = DEFAULT_UNKNOWN_CONTEXT_SIZE;
    choices.push(choice);
  }
}

// @ts-ignore
function deduceLlmTitle(model_name, model_provider, provider_icon = '?') {
  const title =
    provider_icon +
    // @ts-ignore
    model_name.replace(/-/g, ' ').replace(/\b\w/g, (l) => l.toUpperCase()) +
    ' (' +
    model_provider +
    ')';
  return title;
}

// @ts-ignore
function deduceLlmDescription(model_name, context_size = 0) {
  let description = model_name.substring(0, model_name.length - 4); // remove ".bin"
  if (context_size > 0) description += ` (${Math.floor(context_size / 1024)}k)`;
  return description;
}

async function getModelsDirJson() {
  const json_path = path.resolve(process.cwd(), ...MODELS_DIR_JSON_PATH);
  const file_exist = await validateFileExists(json_path);
  if (!file_exist) return null;

  const models_dir_json = await readJsonFromDisk(json_path);

  return models_dir_json;
}

// @ts-ignore
async function fixJsonWithLlm(llm, json_string_to_fix) {
  const ctx = llm.ctx;
  let response = null;
  const args = {};
  args.user = ctx.userId;
  args.prompt = json_string_to_fix;
  args.instruction = 'Fix the JSON string below. Do not output anything else but the carefully fixed JSON string.';
  args.temperature = 0;

  try {
    response = await llm.runLlmBlock(ctx, args);
  } catch (err) {
    console.error(`[FIXING] fixJsonWithLlm: Error fixing json: ${err}`);
    return null;
  }

  const text = response?.answer_text || '';
  console_log(`[FIXING] fixJsonWithLlm: text: ${text}`);

  if (!is_valid(text)) return null;

  return text;
}

// @ts-ignore
async function fixJsonString(llm, passed_string) {
  if (!is_valid(passed_string)) {
    throw new Error(`[FIXING] fixJsonString: passed string is not valid: ${passed_string}`);
  }
  if (typeof passed_string !== 'string') {
    throw new Error(
      `[FIXING] fixJsonString: passed string is not a string: ${passed_string}, type = ${typeof passed_string}`
    );
  }

  // Replace \n with actual line breaks
  const cleanedString = passed_string.replace(/\\n/g, '\n');
  let jsonObject = null;
  let fixed = false;
  let attempt_count = 0;
  let attempt_at_cleaned_string = cleanedString;
  while (!fixed && attempt_count < 10) {
    attempt_count++;
    console_log(`[FIXING] Attempting to fix JSON string after ${attempt_count} attempts.\n`);

    try {
      jsonObject = JSON.parse(attempt_at_cleaned_string);
    } catch (err) {
      console.error(
        `[FIXING] [${attempt_count}] Error fixing JSON string: ${err}, attempt_at_cleaned_string: ${attempt_at_cleaned_string}`
      );
    }

    if (jsonObject !== null && jsonObject !== undefined) {
      fixed = true;
      console_log(`[FIXING] Successfully fixed JSON string after ${attempt_count} attempts.\n`);
      return jsonObject;
    }

    const response = await fixJsonWithLlm(llm, passed_string);
    if (response !== null && response !== undefined) {
      attempt_at_cleaned_string = response;
    }
    await pauseForSeconds(0.5);
  }

  if (!fixed) {
    throw new Error(`Error fixing JSON string after ${attempt_count} attempts.\ncleanedString: ${cleanedString})`);
  }

  return '{}';
}

class Llm {
  // @ts-ignore
  constructor(tokenizer, params = null) {
    this.tokenizer = tokenizer;
    this.context_sizes = {};
  }

  // @ts-ignore
  countTextTokens(text) {
    return this.tokenizer.countTextTokens(text);
  }

  // @ts-ignore
  getModelContextSizeFromModelInfo(model_name) {
    // @ts-ignore
    return this.context_sizes[model_name];
  }

  // -----------------------------------------------------------------------
  /**
   * @param {any} ctx
   * @param {string} prompt
   * @param {string} instruction
   * @param {string} model_name
   * @param {number} [temperature=0]
   * @param {any} args
   * @returns {Promise<{ answer_text: string; answer_json: any; }>}
   */
  // @ts-ignore
  async query(ctx, prompt, instruction, model_name, temperature = 0, args = null) {
    throw new Error('You have to implement this method');
  }

  /**
   * @param {any} ctx
   * @param {any} args
   * @returns {Promise<{ answer_text: string; answer_json: any; }>}
   */
  // @ts-ignore
  async runLlmBlock(ctx, args) {
    throw new Error('You have to implement this method');
  }

  getProvider() {
    throw new Error('You have to implement this method');
  }

  getModelType() {
    throw new Error('You have to implement this method');
  }

  // @ts-ignore
  async getModelChoices(choices, llm_model_types, llm_context_sizes) {
    throw new Error('You have to implement this method');
  }
}

export {
  Llm,
  generateModelId,
  getModelNameAndProviderFromId,
  isProviderAvailable,
  addLocalLlmChoices,
  deduceLlmTitle,
  deduceLlmDescription,
  getModelsDirJson,
  fixJsonString
};
export { DEFAULT_UNKNOWN_CONTEXT_SIZE };