src/pages/api/get_file_list.ts

import process from "node:process";
import { NextApiRequest, NextApiResponse } from "next";

import cheerio from 'cheerio';
import axios from 'axios';
import { SocksProxyAgent } from 'socks-proxy-agent';
import { HFRes } from "@/types";

const socksProxyUrl = 'socks5h://localhost:9909';
const agent = new SocksProxyAgent(socksProxyUrl);

async function get_file_list(base: string, dir: string) {
  const html = (
    await axios.get(base + "/" + dir)
  ).data;
  const $ = cheerio.load(html);
  const res = $("[data-target=ViewerIndexTreeList]").attr(
    "data-props"
  ) as string;
  return JSON.parse(res) as HFRes;
}

export default async function handler(
  request: NextApiRequest,
  response: NextApiResponse
) {
  return response.status(200).json(await get_file_list(
    'https://huggingface.co/datasets/banned-historical-archives/wenhuibao_disk/tree/main',
    '1'
  ));
}