packages/omni-server/src/utils/cdn.js

/**
 * Copyright (c) 2023 MERCENARIES.AI PTE. LTD.
 * All rights reserved.
 */

//@ts-check

import { is_valid, console_log, clean_string } from './utils.js';
import { user_db_put, user_db_get, user_db_delete } from './database.js';

// @ts-ignore
async function save_text_to_cdn(ctx, text) {
  //@ts-ignore
  const buffer = Buffer.from(text);
  const cdn_response = await ctx.app.cdn.putTemp(buffer, { mimeType: 'text/plain; charset=utf-8', userId: ctx.userId, jobId: ctx.jobId });
  console_log(`cdn_response = ${JSON.stringify(cdn_response)}`);

  return cdn_response;
}

// @ts-ignore
async function save_json_to_cdn(ctx, json) {
  const responses_string = JSON.stringify(json, null, 2).trim();
  //@ts-ignore
  const buffer = Buffer.from(responses_string);
  const cdn_response = await ctx.app.cdn.putTemp(buffer, { mimeType: 'text/plain; charset=utf-8', userId: ctx.userId, jobId: ctx.jobId });
  console_log(`cdn_response = ${JSON.stringify(cdn_response)}`);

  return cdn_response;
}

// @ts-ignore
async function get_json_from_cdn(ctx, cdn_response) {
  if (!('ticket' in cdn_response))
    throw new Error(`get_json_from_cdn: cdn_response = ${JSON.stringify(cdn_response)} is invalid`);

  const response_from_cdn = await ctx.app.cdn.get(cdn_response.ticket, null, 'asBase64');
  if (response_from_cdn == null)
    throw new Error(`get_json_from_cdn: document = ${JSON.stringify(response_from_cdn)} is invalid`);

  let json = null;
  try {
    const str = response_from_cdn.data.toString();
    //@ts-ignore
    const buffer = Buffer.from(str, 'base64');
    const json_string = buffer.toString('utf8');

    json = JSON.parse(json_string);
  } catch (e) {
    throw new Error(`get_json_from_cdn: error converting response_from_cdn.data to utf-8, error = ${e}`);
  }

  return json;
}

// @ts-ignore
async function save_json_to_cdn_as_buffer(ctx, json) {
  const responses_string = JSON.stringify(json, null, 2).trim();
  //@ts-ignore
  const buffer = Buffer.from(responses_string);
  const cdn_response = await ctx.app.cdn.putTemp(buffer, { userId: ctx.userId, jobId: ctx.jobId });
  console_log(`cdn_response = ${JSON.stringify(cdn_response)}`);
  return cdn_response;
}

// @ts-ignore
async function get_chunks_from_cdn(ctx, chunks_cdn) {
  const chunks_json = await get_json_from_cdn(ctx, chunks_cdn);
  const chunks = chunks_json.chunks;
  if (!is_valid(chunks))
    throw new Error(`[get_chunks_from_cdn] Error getting chunks from database with cdn= ${JSON.stringify(chunks_cdn)}`);

  return chunks;
}

// @ts-ignore
async function get_cached_cdn(ctx, object_id, overwrite = false) {
  let cdn = null;
  if (overwrite) {
    await user_db_delete(ctx, object_id);
  } else {
    cdn = await user_db_get(ctx, object_id);
  }
  console_log(`[get_cached_cdn] cdn = ${JSON.stringify(cdn)}, typeof cdn = ${typeof cdn}`);

  return cdn;
}

// @ts-ignore
async function save_chunks_cdn_to_db(ctx, chunks_cdn, chunks_id) {
  const success = await user_db_put(ctx, chunks_cdn, chunks_id);
  if (!success) throw new Error('ERROR: could not save chunks_cdn to db');
  return success;
}

// return an array of texts gathered from all the documents (1 per document)
// @ts-ignore
async function downloadTextsFromCdn(ctx, documents_cdns) {
  if (!is_valid(documents_cdns))
    throw new Error(`ERROR: documents is invalid. documents = ${JSON.stringify(documents_cdns)}`);

  const texts = [];
  for (let i = 0; i < documents_cdns.length; i++) {
    const document_cdn = documents_cdns[i];
    //TBD: convert docs files to text when necessary
    try {
      const document = await ctx.app.cdn.get(document_cdn.ticket);
      //const mimeType = document_cdn.mimeType || 'text/plain; charset=utf-8';
      const text = document.data.toString() || '';
      if (!is_valid(text)) {
        console_log(`WARNING: text is null or undefined or empty for document = ${JSON.stringify(document)}`);
        continue;
      }

      const clearn_text = clean_string(text);
      texts.push(clearn_text);
    } catch (error) {
      console_log(`WARNING: document ${JSON.stringify(document_cdn)} cannot be retrieved from cdn`);
    }
  }

  if (!is_valid(texts)) throw new Error('ERROR: texts is invalid');

  return texts;
}

export {
  save_text_to_cdn,
  save_json_to_cdn,
  get_json_from_cdn,
  save_json_to_cdn_as_buffer,
  get_chunks_from_cdn,
  get_cached_cdn,
  save_chunks_cdn_to_db,
  downloadTextsFromCdn
};