Add Django InteractiveSurvey project

.env

@@ -0,0 +1,3 @@
+OPENAI_API_KEY=sk-d474bcdf6cac4cceb472233d66d637bd
+OPENAI_API_BASE=https://dashscope.aliyuncs.com/compatible-mode/v1
+MODEL=qwen-plus-latest

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+src/static/img/paper_with_arrow.png filter=lfs diff=lfs merge=lfs -text
+src/static/img/papers.png filter=lfs diff=lfs merge=lfs -text
+src/db.sqlite3 filter=lfs diff=lfs merge=lfs -text

.gitattributes copy

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,19 @@
+# 忽略 Python 缓存文件
+__pycache__/
+*.pyc
+
+src/static/data
+chromadb
+resources
+logs
+examples
+
+src/demo/survey_generation_pipeline/info
+src/demo/survey_generation_pipeline/txt
+src/demo/survey_generation_pipeline/logs
+src/demo/survey_generation_pipeline/md
+src/demo/survey_generation_pipeline/pdfs
+src/demo/survey_generation_pipeline/result
+src/demo/survey_generation_pipeline/tsv
+src/demo/survey_generation_pipeline/txt
+InteractiveSurvey-default-report.pdf

Dockerfile

@@ -0,0 +1,34 @@
+# 使用官方的轻量镜像
+FROM python:3.10-slim
+
+# 设置工作目录
+WORKDIR /app
+
+# 更新系统并安装依赖
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git wget curl build-essential \
+    graphviz texlive-xetex texlive-fonts-recommended texlive-latex-recommended libgl1 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# RUN apt-get update && apt-get install ffmpeg libsm6 libxext6  -y
+# 复制项目依赖文件并安装环境
+COPY scripts/ /app/
+RUN python setup_env.py
+
+# 下载模型文件并调整存储路径
+RUN wget https://github.com/opendatalab/MinerU/raw/master/scripts/download_models_hf.py -O download_models_hf.py && \
+    python download_models_hf.py && \
+    rm -rf /root/.cache/pip && \
+    python additional_scripts.py
+
+# 复制项目代码
+COPY . /app/
+
+# 暴露服务端口
+EXPOSE 7860
+
+# 健康检查
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s CMD curl -f http://localhost:7860/ || exit 1
+
+# 启动命令
+CMD ["python", "src/manage.py", "runserver", "0.0.0.0:7860"]

README copy.md

@@ -0,0 +1,12 @@
+---
+title: InteractiveSurvey
+emoji: 🦀
+colorFrom: yellow
+colorTo: purple
+sdk: docker
+pinned: false
+license: apache-2.0
+short_description: 'InteractiveSurvey: An LLM-based Personalized and Interactive'
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

flowchart_classifier.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a69d2097e1bc09e3e2b69272c5afade1087acebead83c30f1c6dc6561804badd
+size 16348318

scripts/additional_scripts.py

@@ -0,0 +1,40 @@
+import nltk
+nltk.download('averaged_perceptron_tagger', download_dir='/usr/local/nltk_data')
+
+import json
+import os
+
+file_path = "/root/magic-pdf.json"
+
+new_config = {
+    "device-mode": "cuda",
+    "layout-config": {
+        "model": "layoutlmv3"
+    },
+    "formula-config": {
+        "mfd_model": "yolo_v8_mfd",
+        "mfr_model": "unimernet_small",
+        "enable": False
+    },
+    "table-config": {
+        "model": "tablemaster",
+        "enable": False,
+        "max_time": 400
+    }
+}
+
+if os.path.exists(file_path):
+    with open(file_path, "r", encoding="utf-8") as file:
+        try:
+            data = json.load(file) 
+        except json.JSONDecodeError:
+            data = {}
+else:
+    data = {}
+
+data.update(new_config)
+
+with open(file_path, "w", encoding="utf-8") as file:
+    json.dump(data, file, indent=4)
+
+print(f"File '{file_path}' has been updated.")

scripts/setup_env.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+import shutil
+import os
+
+def clear_pip_cache():
+    print("🧹 Cleaning...")
+    try:
+        # 获取 pip 缓存目录
+        result = subprocess.run([sys.executable, "-m", "pip", "cache", "dir"], stdout=subprocess.PIPE, check=True, text=True)
+        cache_dir = result.stdout.strip()
+        if os.path.exists(cache_dir):
+            shutil.rmtree(cache_dir)
+            print(f"✅ : {cache_dir}")
+        else:
+            print("No cache dir found.")
+    except Exception as e:
+        print(f"❌ {e}")
+
+# 按顺序构造 pip 安装命令列表（全部加上 --no-cache-dir）
+commands = [
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "unstructured==0.16.10"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "requests==2.32.3"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "chromadb==0.5.4"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "langchain-huggingface==0.1.2"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "markdown_pdf==1.3"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "bertopic==0.16.3"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "-U", "langchain-community"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--force-reinstall", "torch==2.3.1", "torchvision==0.18.1", "numpy<2.0.0", "--index-url", "https://download.pytorch.org/whl/cu118"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "-U", "magic-pdf[full]", "--extra-index-url", "https://wheels.myhloli.com"],
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "Django==2.2.5"]
+    [sys.executable, "-m", "pip", "install", "--no-cache-dir", "graphviz"]
+]
+
+def run_commands(cmds):
+    for cmd in cmds:
+        cmd_str = " ".join(cmd)
+        print(f"🚀 {cmd_str}")
+        try:
+            subprocess.run(cmd, check=True)
+        except subprocess.CalledProcessError:
+            print(f"❌ {cmd_str}")
+            sys.exit(1)
+
+if __name__ == "__main__":
+    clear_pip_cache()
+    run_commands(commands)

src/.idea/asg.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.tmp" />
+      <excludeFolder url="file://$MODULE_DIR$/temp" />
+      <excludeFolder url="file://$MODULE_DIR$/tmp" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

src/.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+</project>

src/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/asg.iml" filepath="$PROJECT_DIR$/.idea/asg.iml" />
+    </modules>
+  </component>
+</project>

src/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+  </component>
+</project>

src/.idea/workspace.xml

@@ -0,0 +1,271 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="cf214f68-be48-41a9-bfbc-51edc15fe9c5" name="Default Changelist" comment="">
+      <change afterPath="$PROJECT_DIR$/demo/templates/demo/index_beta.html" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/../.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/demo/category_and_tsne.py" beforeDir="false" afterPath="$PROJECT_DIR$/demo/category_and_tsne.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/demo/taskDes.py" beforeDir="false" afterPath="$PROJECT_DIR$/demo/taskDes.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/demo/templates/demo/index.html" beforeDir="false" afterPath="$PROJECT_DIR$/demo/templates/demo/index.html" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/demo/views.py" beforeDir="false" afterPath="$PROJECT_DIR$/demo/views.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/model_dm" beforeDir="false" afterPath="$PROJECT_DIR$/model_dm" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/static/data/001.tsv" beforeDir="false" afterPath="$PROJECT_DIR$/static/data/001.tsv" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/static/img/tsne_2907070.png" beforeDir="false" afterPath="$PROJECT_DIR$/static/img/tsne_2907070.png" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/static/img/tsne_3274658.png" beforeDir="false" afterPath="$PROJECT_DIR$/static/img/tsne_3274658.png" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/test.tsv" beforeDir="false" afterPath="$PROJECT_DIR$/test.tsv" afterDir="false" />
+    </list>
+    <ignored path="$PROJECT_DIR$/.tmp/" />
+    <ignored path="$PROJECT_DIR$/temp/" />
+    <ignored path="$PROJECT_DIR$/tmp/" />
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FUSProjectUsageTrigger">
+    <session id="-1703485925">
+      <usages-collector id="statistics.lifecycle.project">
+        <counts>
+          <entry key="project.closed" value="6" />
+          <entry key="project.open.time.1" value="1" />
+          <entry key="project.open.time.3" value="4" />
+          <entry key="project.open.time.4" value="1" />
+          <entry key="project.opened" value="6" />
+        </counts>
+      </usages-collector>
+      <usages-collector id="statistics.file.extensions.open">
+        <counts>
+          <entry key="html" value="2" />
+        </counts>
+      </usages-collector>
+      <usages-collector id="statistics.file.types.open">
+        <counts>
+          <entry key="HTML" value="2" />
+        </counts>
+      </usages-collector>
+      <usages-collector id="statistics.file.extensions.edit">
+        <counts>
+          <entry key="html" value="1590" />
+        </counts>
+      </usages-collector>
+      <usages-collector id="statistics.file.types.edit">
+        <counts>
+          <entry key="HTML" value="1590" />
+        </counts>
+      </usages-collector>
+    </session>
+  </component>
+  <component name="FileEditorManager">
+    <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
+      <file pinned="false" current-in-tab="true">
+        <entry file="file://$PROJECT_DIR$/demo/templates/demo/index_beta.html">
+          <provider selected="true" editor-type-id="text-editor">
+            <state relative-caret-position="858">
+              <caret line="370" column="5" lean-forward="true" selection-start-line="322" selection-start-column="4" selection-end-line="370" selection-end-column="5" />
+              <folding>
+                <element signature="n#div#1;n#div#0;n#div#0;n#body#0;n#html#0;n#!!top" />
+                <element signature="n#div#2;n#div#0;n#div#0;n#body#0;n#html#0;n#!!top" />
+                <element signature="n#div#0;n#div#2;n#div#0;n#div#0;n#body#0;n#html#0;n#!!top" />
+                <element signature="n#style#0;n#li#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h3#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#p#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+                <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+              </folding>
+            </state>
+          </provider>
+        </entry>
+      </file>
+    </leaf>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$/.." />
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/demo/templates/demo/index.html" />
+        <option value="$PROJECT_DIR$/demo/templates/demo/index_beta.html" />
+      </list>
+    </option>
+  </component>
+  <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
+  <component name="JsGulpfileManager">
+    <detection-done>true</detection-done>
+    <sorting>DEFINITION_ORDER</sorting>
+  </component>
+  <component name="ProjectFrameBounds" fullScreen="true">
+    <option name="x" value="937" />
+    <option name="y" value="1440" />
+    <option name="width" value="1680" />
+    <option name="height" value="1050" />
+  </component>
+  <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
+  <component name="ProjectView">
+    <navigator proportions="" version="1">
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="asg" type="b2602c69:ProjectViewProjectNode" />
+              <item name="asg" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="asg" type="b2602c69:ProjectViewProjectNode" />
+              <item name="asg" type="462c0819:PsiDirectoryNode" />
+              <item name="demo" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="asg" type="b2602c69:ProjectViewProjectNode" />
+              <item name="asg" type="462c0819:PsiDirectoryNode" />
+              <item name="demo" type="462c0819:PsiDirectoryNode" />
+              <item name="templates" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="asg" type="b2602c69:ProjectViewProjectNode" />
+              <item name="asg" type="462c0819:PsiDirectoryNode" />
+              <item name="demo" type="462c0819:PsiDirectoryNode" />
+              <item name="templates" type="462c0819:PsiDirectoryNode" />
+              <item name="demo" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="asg" type="b2602c69:ProjectViewProjectNode" />
+              <item name="asg" type="462c0819:PsiDirectoryNode" />
+              <item name="static" type="462c0819:PsiDirectoryNode" />
+            </path>
+            <path>
+              <item name="asg" type="b2602c69:ProjectViewProjectNode" />
+              <item name="asg" type="462c0819:PsiDirectoryNode" />
+              <item name="static" type="462c0819:PsiDirectoryNode" />
+              <item name="img" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+      <pane id="Scope" />
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
+    <property name="nodejs_npm_path_reset_for_default_project" value="true" />
+  </component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$/demo/templates/demo" />
+      <recent name="$PROJECT_DIR$/static/img" />
+    </key>
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="SvnConfiguration">
+    <configuration />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="cf214f68-be48-41a9-bfbc-51edc15fe9c5" name="Default Changelist" comment="" />
+      <created>1655724718572</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1655724718572</updated>
+      <workItem from="1655724720238" duration="1478000" />
+      <workItem from="1657850351467" duration="27000" />
+      <workItem from="1657851735564" duration="5134000" />
+      <workItem from="1657960829388" duration="4835000" />
+      <workItem from="1658119972542" duration="6469000" />
+      <workItem from="1658150456661" duration="1476000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TimeTrackingManager">
+    <option name="totallyTimeSpent" value="19419000" />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="937" y="1440" width="1680" height="1050" extended-state="0" />
+    <editor active="true" />
+    <layout>
+      <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.16696805" />
+      <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
+      <window_info id="Favorites" order="2" side_tool="true" />
+      <window_info anchor="bottom" id="Message" order="0" />
+      <window_info anchor="bottom" id="Find" order="1" />
+      <window_info anchor="bottom" id="Run" order="2" />
+      <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
+      <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
+      <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
+      <window_info anchor="bottom" id="TODO" order="6" />
+      <window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
+      <window_info anchor="bottom" id="Version Control" order="8" show_stripe_button="false" />
+      <window_info anchor="bottom" id="Terminal" order="9" />
+      <window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
+      <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
+      <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
+      <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
+    </layout>
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="1" />
+  </component>
+  <component name="VcsContentAnnotationSettings">
+    <option name="myLimit" value="2678400000" />
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/demo/templates/demo/index.html">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="983">
+          <caret line="121" lean-forward="true" selection-start-line="121" selection-end-line="121" />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/demo/templates/demo/index_beta.html">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="858">
+          <caret line="370" column="5" lean-forward="true" selection-start-line="322" selection-start-column="4" selection-end-line="370" selection-end-column="5" />
+          <folding>
+            <element signature="n#div#1;n#div#0;n#div#0;n#body#0;n#html#0;n#!!top" />
+            <element signature="n#div#2;n#div#0;n#div#0;n#body#0;n#html#0;n#!!top" />
+            <element signature="n#div#0;n#div#2;n#div#0;n#div#0;n#body#0;n#html#0;n#!!top" />
+            <element signature="n#style#0;n#li#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h3#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#p#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+            <element signature="n#style#0;n#h2#0;n#!!top" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>

src/asg/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for asg project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'asg.settings')
+
+application = get_asgi_application()

src/asg/settings.py

@@ -0,0 +1,126 @@
+"""
+Django settings for asg project.
+
+Generated by 'django-admin startproject' using Django 3.0.7.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/3.0/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'g#7@+76ha9hwc1hl8!tnvr6rh1k_z8_@sleb=*8i@nh=h3_oi#'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+
+ALLOWED_HOSTS = ['*',]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'demo',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'asg.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': ['demo/templates'],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'asg.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/3.0/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.0/howto/static-files/
+
+STATIC_URL = '/static/'
+
+STATICFILES_DIRS = (
+    os.path.join(BASE_DIR, "static"),
+)

src/asg/urls.py

@@ -0,0 +1,22 @@
+"""asg URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/3.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path, include
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+    path('', include('demo.urls')),
+]

src/asg/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for asg project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'asg.settings')
+
+application = get_wsgi_application()

src/db.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1424881b6d73f9e786d8ed6e666f3ac3e9e3159cab7bfe62ed52b1b7259b3eae
+size 131072

src/demo/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

src/demo/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class DemoConfig(AppConfig):
+    name = 'demo'

src/demo/asg_abstract.py

@@ -0,0 +1,247 @@
+import os
+
+
+class AbstractGenerator:
+    def __init__(self, pipeline):
+        self.pipeline = pipeline
+        
+    def generate(self, title, intro, mode='lora'):
+        if mode == 'lora' or mode == 'test':
+            if mode == 'lora':
+                self.pipeline.model.set_adapter("abstract")
+
+            system_prompt = f'''You are a helpful assistant that help to generate the abstract of the survey paper given the survey title and survey introduction.'''
+            # user_prompt = {"survey_title":survey_title, "claims":cluster_with_claims}
+            user_prompt = f'''Help me to generate the abstract of a survey paper given the title: *{title}*, and and the introduction:{intro}'''
+
+            messages = [
+                {"role": "system", "content": system_prompt}, 
+                {"role": "user", "content": user_prompt},
+                {"role": "assistant", "content":"Abstract: This survey "}
+            ]
+
+            outputs = self.pipeline(
+                messages,
+                max_new_tokens=4096,
+            )
+            result = outputs[0]["generated_text"][-1]['content']
+            return result
+        else:
+            raise ValueError('mode not supported')
+
+if __name__ == '__main__':
+    from transformers import pipeline
+    import torch
+    import transformers
+
+    model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+    Global_pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_id,
+        model_kwargs={"torch_dtype": torch.bfloat16},
+        token = os.getenv('HF_API_KEY'),
+        device_map="auto",
+    )
+    Global_pipeline.model.load_adapter(peft_model_id = "technicolor/llama3.1_8b_outline_generation", adapter_name="outline")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_abstract_generation", adapter_name="abstract")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_conclusion_generation", adapter_name="conclusion")
+    title = "A Survey of Large Language Models"
+    intro = '''L
+ANGUAGE is a prominent ability in human beings to
+express and communicate, which develops in early
+childhood and evolves over a lifetime [3, 4]. Machines,
+however, cannot naturally grasp the abilities of understanding and communicating in the form of human language,
+unless equipped with powerful artificial intelligence (AI)
+algorithms. It has been a longstanding research challenge
+to achieve this goal, to enable machines to read, write, and
+communicate like humans [5].
+Technically, language modeling (LM) is one of the major
+approaches to advancing language intelligence of machines.
+In general, LM aims to model the generative likelihood
+of word sequences, so as to predict the probabilities of
+future (or missing) tokens. The research of LM has received
+extensive attention in the literature, which can be divided
+into four major development stages:
+• Statistical language models (SLM). SLMs [6–9] are developed based on statistical learning methods that rose in
+the 1990s. The basic idea is to build the word prediction
+model based on the Markov assumption, e.g., predicting the
+next word based on the most recent context. The SLMs with
+a fixed context length n are also called n-gram language
+models, e.g., bigram and trigram language models. SLMs
+have been widely applied to enhance task performance
+in information retrieval (IR) [10, 11] and natural language
+processing (NLP) [12–14]. However, they often suffer from
+the curse of dimensionality: it is difficult to accurately
+estimate high-order language models since an exponential
+number of transition probabilities need to be estimated.
+Thus, specially designed smoothing strategies such as backoff estimation [15] and Good–Turing estimation [16] have
+been introduced to alleviate the data sparsity problem.
+• Neural language models (NLM). NLMs [1, 17, 18] characterize the probability of word sequences by neural networks,
+e.g., multi-layer perceptron (MLP) and recurrent neural networks (RNNs). As a remarkable contribution, the work in
+[1] introduced the concept of distributed representation of
+words and built the word prediction function conditioned
+on the aggregated context features (i.e., the distributed
+word vectors). By extending the idea of learning effective
+features for text data, a general neural network approach
+was developed to build a unified, end-to-end solution for
+various NLP tasks [2]. Furthermore, word2vec [19, 20] was
+proposed to build a simplified shallow neural network
+for learning distributed word representations, which were
+demonstrated to be very effective across a variety of NLP
+tasks. These studies have initiated the use of language
+models for representation learning (beyond word sequence
+modeling), having an important impact on the field of NLP.
+• Pre-trained language models (PLM). As an early attempt, ELMo [21] was proposed to capture context-aware
+word representations by first pre-training a bidirectional
+LSTM (biLSTM) network (instead of learning fixed word
+representations) and then fine-tuning the biLSTM network
+according to specific downstream tasks. Furthermore, based
+on the highly parallelizable Transformer architecture [22]
+with self-attention mechanisms, BERT [23] was proposed by
+pre-training bidirectional language models with specially
+designed pre-training tasks on large-scale unlabeled corpora. These pre-trained context-aware word representations
+are very effective as general-purpose semantic features,
+which have largely raised the performance bar of NLP
+tasks. This study has inspired a large number of follow-up
+work, which sets the “pre-training and fine-tuning” learning
+paradigm. Following this paradigm, a great number of studies on PLMs have been developed, introducing either different architectures [24, 25] (e.g., GPT-2 [26] and BART [24]) or
+improved pre-training strategies [27–29]. In this paradigm, it
+often requires fine-tuning the PLM for adapting to different
+downstream tasks.
+• Large language models (LLM). Researchers find that
+scaling PLM (e.g., scaling model size or data size) often
+leads to an improved model capacity on downstream tasks
+(i.e., following the scaling law [30]). A number of studies
+have explored the performance limit by training an ever
+larger PLM (e.g., the 175B-parameter GPT-3 and the 540Bparameter PaLM). Although scaling is mainly conducted
+in model size (with similar architectures and pre-training
+tasks), these large-sized PLMs display different behaviors
+from smaller PLMs (e.g., 330M-parameter BERT and 1.5Bparameter GPT-2) and show surprising abilities (called emergent abilities [31]) in solving a series of complex tasks. For
+example, GPT-3 can solve few-shot tasks through in-context
+learning, whereas GPT-2 cannot do well. Thus, the research
+community coins the term “large language models (LLM)”
+1
+for these large-sized PLMs [32–35], which attract increasing
+research attention (See Figure 1). A remarkable application
+of LLMs is ChatGPT2
+that adapts the LLMs from the GPT
+series for dialogue, which presents an amazing conversation
+ability with humans. We can observe a sharp increase of the
+arXiv papers that are related to LLMs after the release of
+ChatGPT in Figure 1.
+As discussed before, language model is not a new technical concept specially for LLMs, but has evolved with the
+advance of artificial intelligence over the decades. Early language models mainly aim to model and generate text data,
+while latest language models (e.g., GPT-4) focus on complex
+task solving. From language modeling to task solving, it is an
+important leap in scientific thinking, which is the key to
+understand the development of language models in the research history. From the perspective of task solving, the four
+generations of language models have exhibited different levels of model capacities. In Figure 2, we describe the evolution process of language models in terms of the task solving
+capacity. At first, statistical language models mainly assisted
+in some specific tasks (e.g., retrieval or speech tasks), in
+which the predicted or estimated probabilities can enhance
+the performance of task-specific approaches. Subsequently,
+neural language models focused on learning task-agnostic
+representations (e.g., features), aiming to reduce the efforts
+for human feature engineering. Furthermore, pre-trained
+language models learned context-aware representations that
+can be optimized according to downstream tasks. For the
+latest generation of language model, LLMs are enhanced by
+exploring the scaling effect on model capacity, which can be
+considered as general-purpose task solvers. To summarize,
+in the evolution process, the task scope that can be solved
+by language models have been greatly extended, and the
+task performance attained by language models have been
+significantly enhanced.
+In the existing literature, PLMs have been widely discussed and surveyed [36–39], while LLMs are seldom reviewed in a systematic way. To motivate our survey, we first
+highlight three major differences between LLMs and PLMs.
+First, LLMs display some surprising emergent abilities that
+may not be observed in previous smaller PLMs. These abilities are key to the performance of language models on complex tasks, making AI algorithms unprecedently powerful
+and effective. Second, LLMs would revolutionize the way
+that humans develop and use AI algorithms. Unlike small
+PLMs, the major approach to accessing LLMs is through
+the prompting interface (e.g., GPT-4 API). Humans have to
+understand how LLMs work and format their tasks in a way
+that LLMs can follow. Third, the development of LLMs no
+longer draws a clear distinction between research and engineering. The training of LLMs requires extensive practical
+experiences in large-scale data processing and distributed
+parallel training. To develop capable LLMs, researchers
+have to solve complicated engineering issues, working with
+engineers or being engineers.
+Nowadays, LLMs are posing a significant impact on
+the AI community, and the advent of ChatGPT and GPT-4
+leads to the rethinking of the possibilities of artificial general
+intelligence (AGI). OpenAI has published a technical article
+entitled “Planning for AGI and beyond”, which discusses
+the short-term and long-term plans to approach AGI [40],
+and a more recent paper has argued that GPT-4 might be
+considered as an early version of an AGI system [41]. The
+research areas of AI are being revolutionized by the rapid
+progress of LLMs. In the field of NLP, LLMs can serve as a
+general-purpose language task solver (to some extent), and
+the research paradigm has been shifting towards the use
+of LLMs. In the field of IR, traditional search engines are
+challenged by the new information seeking way through AI
+chatbots (i.e., ChatGPT), and New Bing3 presents an initial
+attempt that enhances the search results based on LLMs. In
+the field of CV, the researchers try to develop ChatGPT-like
+vision-language models that can better serve multimodal
+dialogues [42–45], and GPT-4 [46] has supported multimodal input by integrating the visual information. This new
+wave of technology would potentially lead to a prosperous
+ecosystem of real-world applications based on LLMs. For
+instance, Microsoft 365 is being empowered by LLMs (i.e.,
+Copilot) to automate the office work, and OpenAI supports
+the use of plugins in ChatGPT for implementing special
+functions.
+Despite the progress and impact, the underlying principles of LLMs are still not well explored. Firstly, it is
+mysterious why emergent abilities occur in LLMs, instead of
+smaller PLMs. As a more general issue, there lacks a deep,
+detailed investigation of the key factors that contribute to
+the superior abilities of LLMs. It is important to study when
+and how LLMs obtain such abilities [47]. Although there are
+some meaningful discussions about this problem [31, 47],
+more principled investigations are needed to uncover the
+“secrets“ of LLMs. Secondly, it is difficult for the research
+community to train capable LLMs. Due to the huge demand of computation resources, it is very costly to carry
+out repetitive, ablating studies for investigating the effect
+of various strategies for training LLMs. Indeed, LLMs are
+mainly trained by industry, where many important training
+details (e.g., data collection and cleaning) are not revealed
+to the public. Thirdly, it is challenging to align LLMs with
+human values or preferences. Despite the capacities, LLMs
+are also likely to produce toxic, fictitious, or harmful contents. It requires effective and efficient control approaches
+to eliminating the potential risk of the use of LLMs [46].
+Faced with both opportunities and challenges, it needs
+more attention on the research and development of LLMs. In
+order to provide a basic understanding of LLMs, this survey
+conducts a literature review of the recent advances in LLMs
+from four major aspects, including pre-training (how to pretrain a capable LLM), adaptation (how to effectively adapt
+pre-trained LLMs for better use), utilization (how to use
+LLMs for solving various downstream tasks) and capability
+evaluation (how to evaluate the abilities of LLMs and existing
+empirical findings). We thoroughly comb the literature and
+summarize the key findings, techniques, and methods of
+LLMs. For this survey, we also create a GitHub project
+website by collecting the supporting resources for LLMs, at
+the link https://github.com/RUCAIBox/LLMSurvey. We
+are also aware of several related review articles on PLMs
+or LLMs [32, 36, 38, 39, 43, 48–54]. These papers either
+discuss PLMs or some specific (or general) aspects of LLMs.
+Compared with them, we focus on the techniques and
+methods to develop and use LLMs and provide a relatively
+comprehensive reference to important aspects of LLMs.
+The remainder of this survey is organized as follows:
+Section 2 introduces the background for LLMs and the evolution of GPT-series models, followed by the summarization
+of available resources for developing LLMs in Section 3.
+Sections 4, 5, 6, and 7 review and summarize the recent
+progress from the four aspects of pre-training, adaptation,
+utilization, and capacity evaluation, respectively. Then, Section 8 discusses the practical guide for prompt design,
+and Section 9 reviews the applications of LLMs in several
+representative domains. Finally, we conclude the survey in
+Section 10 by summarizing the major findings and discuss
+the remaining issues for future work.
+'''
+
+
+    abstract_generator = AbstractGenerator(Global_pipeline)
+    with_lora = abstract_generator.generate(title, intro, mode='lora')
+    with_test = abstract_generator.generate(title, intro, mode='test')

src/demo/asg_add_flowchart.py

@@ -0,0 +1,313 @@
+import json
+import os
+import re
+from urllib.parse import quote
+
+import os
+import json
+import torch
+import torchvision.transforms as transforms
+from torchvision import models
+from PIL import Image
+
+# 常量定义
+BASE_DIR = os.path.normpath("src/static/data/md")  # 根目录
+INFO_DIR = os.path.normpath("src/static/data/info")  # 存放 JSON 结果的目录
+
+# 加载 PyTorch EfficientNet 训练好的 3 类分类模型
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model = models.efficientnet_b0(pretrained=False)
+
+# 修改最后一层，适应 3 类（flowchart, non-flowchart, other）
+num_features = model.classifier[1].in_features
+model.classifier[1] = torch.nn.Linear(num_features, 3)  # 3 类
+model.load_state_dict(torch.load("flowchart_classifier.pth", map_location=device))
+model.to(device)  # 确保模型移动到正确的设备
+model.eval()
+
+# 预处理图片
+transform = transforms.Compose([
+    transforms.Resize((224, 224)),
+    transforms.ToTensor(),
+])
+
+def detect_flowcharts(survey_id):
+    """ 在指定 survey_id 目录下查找 flowchart，并保存 JSON 结果 """
+    survey_path = os.path.join(BASE_DIR, survey_id)  # 该 survey_id 的目录
+    if not os.path.exists(survey_path):
+        print(f"❌ 目录 {survey_path} 不存在！")
+        return
+
+    flowchart_dict = {}  # 存储 flowchart 结果
+
+    # 遍历该 survey 目录下的所有 PDF 文件夹
+    for pdf_folder in os.listdir(survey_path):
+        pdf_folder_path = os.path.join(survey_path, pdf_folder)
+
+        if not os.path.isdir(pdf_folder_path):
+            continue  # 只处理文件夹
+
+        print(f"🔍 处理 PDF 文件夹: {pdf_folder}")
+
+        # 遍历所有 `xxx/auto/images` 目录
+        for root, dirs, files in os.walk(pdf_folder_path):
+            if "auto/images" in root.replace("\\", "/"):  # 兼容 Windows 和 Linux
+                for filename in sorted(files):  # 按文件名排序，保证第一个找到的 Flowchart 被选用
+                    if not filename.lower().endswith(".jpg"):  # 只处理 JPG
+                        continue
+
+                    image_path = os.path.join(root, filename)
+                    img = Image.open(image_path).convert("RGB")  # 打开图片并转换为 RGB
+
+                    # 预处理图片并转换为张量
+                    img_tensor = transform(img).unsqueeze(0).to(device)
+
+                    # 运行分类模型
+                    with torch.no_grad():
+                        output = model(img_tensor)
+                        predicted_class = torch.argmax(output).item()
+
+                    # **确保 predicted_class == 0 表示 flowchart**
+                    if predicted_class == 2:  # `0` 代表 Flowchart 类别
+                        print(f"✅ Flowchart detected: {image_path}")
+                        flowchart_dict[pdf_folder] = image_path
+                        break  # **只存当前 PDF 文件夹的第一张 flowchart**
+
+    # 只有检测到 Flowchart 时才保存 JSON
+    if flowchart_dict:
+        os.makedirs(os.path.join(INFO_DIR, survey_id), exist_ok=True)  # 确保目录存在
+        json_path = os.path.join(INFO_DIR, survey_id, "flowchart_results.json")
+        with open(json_path, "w", encoding="utf-8") as f:
+            json.dump(flowchart_dict, f, indent=4, ensure_ascii=False)
+
+        print(f"📁 Flowchart 结果已保存: {json_path}")
+    else:
+        print(f"⚠️ 没有检测到 Flowchart，未生成 JSON")
+
+# 示例调用
+# survey_id = "test"  # 例如 "test"
+# detect_flowcharts(survey_id)
+
+def insert_ref_images(json_path, ref_names, text):
+    """
+    参数:
+      json_path: JSON 文件路径，其内容格式例如：
+                 {
+                   "Accelerating federated learning with data and model parallelism in edge computing":
+                     "src/static/data/md/test/Accelerating federated learning with data and model parallelism in edge computing/auto/images/xxx.jpg",
+                   ... 
+                 }
+      ref_names: 引用名称列表，其中第 1 个元素对应 [1]，第 2 个对应 [2]，以此类推。
+      text: 包含类似 [1]、[2] 等引用的 Markdown 文本。
+
+    返回:
+      修改后的文本字符串。在每个引用标记首次出现行的下方插入对应的 HTML 代码块，
+      格式如下：
+      
+      <div style="text-align:center">
+          <img src="image_path" alt="the flow chart of [ref_name]" style="width:50%;"/>
+      </div>
+      <div style="text-align:center">
+          Fig [ref_num]: The flow chart of [ref_name]
+      </div>
+      
+      其中 [ref_num] 为引用编号（ref_names 中的 1-based index），[ref_name] 为引用名称。
+
+    说明：
+      1. JSON 中存储的路径已是目标路径，但可能混合了正斜杠和反斜杠。
+      2. 代码将先拆分路径字符串，再利用 os.path.join 拼接生成当前系统的标准路径，
+         最后转换为统一的正斜杠格式并进行 URL ��码，以适配所有系统。
+    """
+    # 加载 JSON 文件内容
+    try:
+        with open(json_path, 'r', encoding='utf-8') as f:
+            img_mapping = json.load(f)
+    except Exception as e:
+        raise Exception(f"加载 JSON 文件出错: {e}")
+
+    inserted_refs = {}  # 记录每个引用标记是否已插入图片
+    lines = text.splitlines()
+    new_lines = []
+    # 匹配类似 [1]、[2] 的引用标记
+    ref_pattern = re.compile(r'\[(\d+)\]')
+    img_index = 2
+    for line in lines:
+        new_lines.append(line)
+        matches = ref_pattern.findall(line)
+        for ref_num_str in matches:
+            try:
+                ref_num = int(ref_num_str)
+            except ValueError:
+                continue
+
+            # 仅在引用标记首次出现时插入 HTML 块
+            if ref_num not in inserted_refs:
+                inserted_refs[ref_num] = True
+
+                if 1 <= ref_num <= len(ref_names):
+                    ref_name = ref_names[ref_num - 1]
+                    jpg_path = img_mapping.get(ref_name, "")
+                else:
+                    ref_name = f"ref_{ref_num}"
+                    jpg_path = ""
+                
+                if jpg_path:
+                    # 将路径中可能混合的正斜杠和反斜杠拆分为多个部分
+                    parts = re.split(r'[\\/]+', jpg_path)
+                    # 使用 os.path.join 拼接成当前系统的规范路径
+                    normalized_jpg_path = os.path.join(*parts)
+                    # 转换为适用于 HTML 的路径格式（全部替换为正斜杠）
+                    normalized_jpg_path = normalized_jpg_path.replace(os.sep, '/')
+                    # 对路径进行 URL 编码（保留斜杠）
+                    normalized_jpg_path_url = quote(normalized_jpg_path, safe="/")
+
+                    html_block = (
+                        f"<div style=\"text-align:center\">\n"
+                        f"    <img src=\"{normalized_jpg_path_url}\" alt=\"the chart of {ref_name}\" style=\"width:60%;\"/>\n"
+                        f"</div>\n"
+                        f"<div style=\"text-align:center;font-size:smaller;\">\n"
+                        f"    Fig {img_index}: Chart from \'{ref_name}\'\n"
+                        f"</div>"
+                    )
+                    new_lines.append(html_block)
+                    new_lines.append("")  # 增加一个空行分隔
+                    img_index += 1
+
+    return "\n".join(new_lines)
+
+def insert_tex_images(json_path, ref_names, text):
+    """
+    将 Markdown 文本中出现的数字引用（例如 [1], \[1], \[1\]）替换为 LaTeX figure 环境。
+    仅在每个引用编号第一次出现时插入对应图片，后续出现同编号不再重复插入。
+
+    参数:
+        json_path: JSON 文件路径，其内容格式例如：
+            {
+              "Accelerating federated learning with data and model parallelism in edge computing":
+                "src/static/data/md/test/Accelerating federated learning with data and model parallelism in edge computing/auto/images/xxx.jpg",
+              ...
+            }
+        ref_names: 引用名称列表。其中第 1 个元素对应 [1]，第 2 个对应 [2]，以此类推。
+        text: 包含类似 [1]、\[1]、\[1\] 等形式的 Markdown 文本。
+
+    返回:
+        修改后的文本字符串。在每个引用标记首次出现行的下方插入对应的 LaTeX figure 环境：
+
+        \begin{figure}[htbp]
+          \centering
+          \includegraphics[width=0.6\textwidth]{image_path}
+          \caption{Fig 2: Chart from 'ref_name'}
+        \end{figure}
+
+    说明：
+      1. JSON 中存储的路径可能含正反斜杠。
+      2. 我们按系统拼接路径，再统一转为正斜杠并进行 URL 编码。
+      3. figure 的计数从 1 开始（可根据需求调整）。
+      4. 若某引用编号未在 JSON 中匹配到图片，则不插入 figure。
+    """
+
+    # 读取 JSON
+    try:
+        with open(json_path, 'r', encoding='utf-8') as f:
+            img_mapping = json.load(f)
+    except Exception as e:
+        raise Exception(f"加载 JSON 文件出错: {e}")
+
+    # 用于记录某个编号是否已插入过
+    inserted_refs = {}
+
+    # 按行处理文本
+    lines = text.splitlines()
+    new_lines = []
+
+    # --------------------------
+    # 匹配 [1], \[1], \[1\] 等数字引用
+    # --------------------------
+    # 含义:
+    #   (?:\\)?    -> 可选的反斜杠 0或1次
+    #   \[         -> 文字 '[' (在正则中需转义)
+    #   (\d+)      -> 捕获一个或多个数字
+    #   (?:\\)?    -> 可选的反斜杠 0或1次
+    #   \]         -> 文字 ']' (需转义)
+    # 整体匹配可能出现以下形式:
+    #   [1], \[1], \[1\], [12], \[12] 等
+    ref_pattern = re.compile(r'(?:\\)?\[(\d+)(?:\\)?\]')
+
+    # figure 计数
+    figure_index = 1
+
+    for line in lines:
+        new_lines.append(line)  # 先把此行内容写入新文本
+
+        # 查找本��中所有符合模式的引用
+        matches = ref_pattern.findall(line)
+        for ref_num_str in matches:
+            try:
+                ref_num = int(ref_num_str)
+            except ValueError:
+                continue
+
+            # 若该引用编号尚未插入过图片，则执行插入
+            if ref_num not in inserted_refs:
+                inserted_refs[ref_num] = True
+
+                # 判断这个编号是否在 ref_names 范围内
+                if 1 <= ref_num <= len(ref_names):
+                    ref_name = ref_names[ref_num - 1]
+                    jpg_path = img_mapping.get(ref_name, "")
+                else:
+                    ref_name = f"ref_{ref_num}"
+                    jpg_path = ""
+
+                if jpg_path:
+                    # 规范化路径
+                    parts = re.split(r'[\\/]+', jpg_path)
+                    normalized_jpg_path = os.path.join(*parts)
+                    normalized_jpg_path = normalized_jpg_path.replace(os.sep, '/')
+                    # URL 编码（保留 '/')
+                    # normalized_jpg_path_url = quote(normalized_jpg_path, safe="/")
+                    normalized_jpg_path_url = normalized_jpg_path
+
+                    # 构建 LaTeX figure 块
+                    tex_block = (
+                        r"\begin{figure}[htbp]" "\n"
+                        r"  \centering" "\n"
+                        f"  \\includegraphics[width=0.5\\textwidth]{{{normalized_jpg_path_url}}}\n"
+                        f"  \\caption{{Chart from \\textit{ref_name}}}\n"
+                        r"\end{figure}"
+                    )
+
+                    # 插到新文本中，再加个空行分隔
+                    new_lines.append(tex_block)
+                    new_lines.append("")
+                    figure_index += 1
+
+    return "\n".join(new_lines)
+
+
+# 示例用法
+if __name__ == "__main__":
+    # Markdown 文件路径
+    md_file_path = "src/static/data/info/test/survey_test_processed.md"
+    # JSON 文件路径
+    json_file_path = "src/static/data/info/test/flowchart_results.json"
+
+    try:
+        with open(md_file_path, "r", encoding="utf-8") as f:
+            text = f.read()
+    except FileNotFoundError:
+        print(f"错误: Markdown 文件 {md_file_path} 未找到！")
+        text = ""
+
+    ref_names = [
+        "An explainable federated learning and blockchain based secure credit modeling method",
+        "Bafl a blockchain based asynchronous",
+        "Biscotti a blockchain system for private and secure federated learning",
+        "Blockdfl a blockchain based fully decentralized peer to peer",
+        "Accelerating blockchain enabled federated learning with clustered clients",
+        "A fast blockchain based federated learning framework with compressed communications"
+    ]
+
+    result = insert_ref_images(json_file_path, ref_names, text)
+    print("修改后的文本为：\n")
+    print(result)

src/demo/asg_clustername.py

@@ -0,0 +1,228 @@
+import os
+import pandas as pd
+import re  # Import the regular expressions module
+from openai import OpenAI
+import ast
+
+def generate_cluster_name_qwen_sep(tsv_path, survey_title):
+    data = pd.read_csv(tsv_path, sep='\t')
+    
+    # Define the system prompt once, outside the loop
+    system_prompt = f'''You are a research assistant working on a survey paper. The survey paper is about "{survey_title}". \
+    '''
+    
+    result = []  # Initialize the result list
+
+    for i in range(3):  # Assuming labels are 0, 1, 2
+        sentence_list = []  # Reset sentence_list for each label
+        for j in range(len(data)):
+            if data['label'][j] == i:
+                sentence_list.append(data['retrieval_result'][j])
+        
+        # Convert the sentence list to a string representation
+        user_prompt = f'''
+        Given a list of descriptions of sentences about an aspect of the survey, you need to use one phrase (within 8 words) to summarize it and treat it as a section title of your survey paper. \
+Your response should be a list with only one element and without any other information, for example, ["Post-training of LLMs"]  \
+Your response must contain one keyword of the survey title, unspecified or irrelevant results are not allowed. \
+The description list is:{sentence_list}'''
+        
+        messages = [
+            {"role": "system", "content": system_prompt}, 
+            {"role": "user", "content": user_prompt},
+        ]
+        
+        openai_api_key = os.getenv("OPENAI_API_KEY")
+        openai_api_base = os.getenv("OPENAI_API_BASE")
+        client = OpenAI(
+            api_key=openai_api_key,
+            base_url=openai_api_base,
+        )
+        
+        chat_response = client.chat.completions.create(
+            model=os.environ.get("MODEL"),
+            max_tokens=768,
+            temperature=0.5,
+            stop="<|im_end|>",
+            stream=True,
+            messages=messages
+        )
+        
+        # Stream the response to a single text string
+        text = ""
+        for chunk in chat_response:
+            if chunk.choices[0].delta.content:
+                text += chunk.choices[0].delta.content
+        
+        # Use regex to extract the first content within []
+        match = re.search(r'\[(.*?)\]', text)
+        if match:
+            cluster_name = match.group(1).strip()  # Extract and clean the cluster name
+            # 去除集群名称两侧的引号（如果存在）
+            cluster_name = cluster_name.strip('"').strip("'")
+            result.append(cluster_name)
+        else:
+            result.append("No Cluster Name Found")  # Handle cases where pattern isn't found
+    # print("The generated cluster names are:")
+    # print(result)
+    return result  # This will be a list with three elements
+    
+# Example usage:
+# result = generate_cluster_name_qwen_sep('path_to_your_file.tsv', 'Your Survey Title')
+# print(result)  # Output might look like ["Cluster One", "Cluster Two", "Cluster Three"]
+
+def refine_cluster_name(cluster_names, survey_title):
+    cluster_names = str(cluster_names)  # Convert to string to handle list input
+    # Define the system prompt to set the context
+    system_prompt = f'''You are a research assistant tasked with optimizing and refining a set of section titles for a survey paper. The survey paper is about "{survey_title}". 
+'''
+    
+    # Construct the user prompt, including all cluster names
+    user_prompt = f'''
+Here is a set of section titles generated for the survey topic "{survey_title}":
+{cluster_names}
+Please ensure that all cluster names are coherent and consistent with each other, and that each name is clear, concise, and accurately reflects the corresponding section.
+Notice to remove the overlapping information between the cluster names.
+Each cluster name should be within 8 words and include a keyword from the survey title.
+Response with a list of section titles in the following format without any other irrelevant information,
+For example, ["Refined Title 1", "Refined Title 2", "Refined Title 3"]
+'''
+    
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    # Initialize OpenAI client
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    try:
+        chat_response = client.chat.completions.create(
+            model=os.environ.get("MODEL"),
+            max_tokens=256,
+            temperature=0.5,
+            stop="<|im_end|>",
+            stream=True,
+            messages=messages
+        )
+    
+        # Stream the response and concatenate into a complete text
+        text = ""
+        for chunk in chat_response:
+            if chunk.choices[0].delta.content:
+                text += chunk.choices[0].delta.content
+
+        # print("The raw response text is:")
+        # print(text)
+    
+        # Use regex to extract content within square brackets
+        match = re.search(r'\[(.*?)\]', text)
+        if match:
+            refined_cluster_names = match.group(1).strip()  # Extract and clean the cluster name
+        else:
+            refined_cluster_names = [
+                survey_title + ": Definition",
+                survey_title + ": Methods",
+                survey_title + ": Evaluation"
+            ]  # Handle cases where pattern isn't found
+    
+    except Exception as e:
+        print(f"An error occurred while refining cluster names: {e}")
+        refined_cluster_names = ["Refinement Error"] * len(cluster_names)
+    
+    refined_cluster_names = ast.literal_eval(refined_cluster_names)  # Convert string to list
+    
+    # print("The refined cluster names are:")
+    # print(refined_cluster_names)
+    return refined_cluster_names  # Returns a list with the refined cluster names、
+
+
+
+
+def generate_cluster_name_new(tsv_path, survey_title, cluster_num = 3):
+    data = pd.read_csv(tsv_path, sep='\t')
+    desp=[]
+
+
+    for i in range(cluster_num):  # Assuming labels are 0, 1, 2
+        sentence_list = []  # Initialize the sentence list
+        for j in range(len(data)):
+            if data['label'][j] == i:
+                sentence_list.append(data['retrieval_result'][j])
+        desp.append(sentence_list)
+
+    system_prompt = f'''
+    You are a research assistant working on a survey paper. The survey paper is about "{survey_title}". '''
+    
+    cluster_info = "\n".join([f'Cluster {i+1}: "{desp[i]}"' for i in range(cluster_num)])
+
+    user_prompt = f'''
+    Your task is to generate {cluster_num} distinctive cluster names (e.g., "Pre-training of LLMs") of the given clusters of reference papers, each reference paper is described by a sentence.
+
+    The clusters of reference papers are: 
+    {cluster_info}
+
+    Your output should be a single list of {cluster_num} cluster names, e.g., ["Pre-training of LLMs", "Fine-tuning of LLMs", "Evaluation of LLMs"]
+    Do not output any other text or information.
+    '''
+
+    messages = [
+        {"role": "system", "content": system_prompt}, 
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    chat_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=768,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages
+    )
+    
+    # Stream the response to a single text string
+    text = ""
+    for chunk in chat_response:
+        if chunk.choices[0].delta.content:
+            text += chunk.choices[0].delta.content
+    # print("The raw response text is:")
+    # print(text)
+
+    # Use regex to extract content within square brackets
+    match = re.search(r'\[(.*?)\]', text)
+    if match:
+        refined_cluster_names = match.group(1).strip()  # Extract and clean the cluster name
+    else:
+        predefined_sections = [
+            "Definition", "Methods", "Evaluation", "Applications",
+            "Challenges", "Future Directions", "Comparisons", "Case Studies"
+        ]
+        
+        # 根据 cluster_num 选择前 cluster_num 个预定义类别
+        refined_cluster_names = [
+            f"{survey_title}: {predefined_sections[i]}" for i in range(cluster_num)
+        ]
+    
+    refined_cluster_names = ast.literal_eval(refined_cluster_names)  # Convert string to list
+    
+    # print("The refined cluster names are:")
+    # print(refined_cluster_names)
+    return refined_cluster_names  # Returns a list with the refined cluster names、
+
+
+if __name__ == "__main__":
+    refined_result = refine_cluster_name(["Pre-training of LLMs", "Fine-tuning of LLMs", "Evaluation of LLMs"], 'Survey of LLMs')
+    # print(refined_result)
+        
+    

src/demo/asg_conclusion.py

@@ -0,0 +1,253 @@
+import os
+
+
+class ConclusionGenerator:
+    def __init__(self, pipeline):
+        self.pipeline = pipeline
+        
+    def generate(self, title, intro, mode='lora'):
+        if mode == 'lora' or mode == 'test':
+            if mode == 'lora':
+                self.pipeline.model.set_adapter("conclusion")
+
+            system_prompt = f'''You are a helpful assistant that help to generate the conclusion of the survey paper given the survey title and survey introduction.'''
+            # user_prompt = {"survey_title":survey_title, "claims":cluster_with_claims}
+            user_prompt = f'''Help me to generate the conclusion of a survey paper given the title: *{title}*, and and the introduction:{intro}'''
+
+            messages = [
+                {"role": "system", "content": system_prompt}, 
+                {"role": "user", "content": user_prompt},
+                {"role": "assistant", "content":"Conclusion: This survey "}
+                ] 
+
+            outputs = self.pipeline(
+                messages,
+                max_new_tokens=4096,
+            )
+            result = outputs[0]["generated_text"][-1]['content']
+            return result
+        else:
+            raise ValueError('mode not supported')
+
+if __name__ == '__main__':
+    from transformers import pipeline
+    import torch
+    import transformers
+
+    model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+    Global_pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_id,
+        model_kwargs={"torch_dtype": torch.bfloat16},
+        token = os.getenv('HF_API_KEY'),
+        device_map="auto",
+    )
+    Global_pipeline.model.load_adapter(peft_model_id = "technicolor/llama3.1_8b_outline_generation", adapter_name="outline")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_conclusion_generation", adapter_name="conclusion")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_abstract_generation", adapter_name="abstract")
+
+
+    title = "A Survey of Large Language Models"
+    intro = '''L
+ANGUAGE is a prominent ability in human beings to
+express and communicate, which develops in early
+childhood and evolves over a lifetime [3, 4]. Machines,
+however, cannot naturally grasp the abilities of understanding and communicating in the form of human language,
+unless equipped with powerful artificial intelligence (AI)
+algorithms. It has been a longstanding research challenge
+to achieve this goal, to enable machines to read, write, and
+communicate like humans [5].
+Technically, language modeling (LM) is one of the major
+approaches to advancing language intelligence of machines.
+In general, LM aims to model the generative likelihood
+of word sequences, so as to predict the probabilities of
+future (or missing) tokens. The research of LM has received
+extensive attention in the literature, which can be divided
+into four major development stages:
+• Statistical language models (SLM). SLMs [6–9] are developed based on statistical learning methods that rose in
+the 1990s. The basic idea is to build the word prediction
+model based on the Markov assumption, e.g., predicting the
+next word based on the most recent context. The SLMs with
+a fixed context length n are also called n-gram language
+models, e.g., bigram and trigram language models. SLMs
+have been widely applied to enhance task performance
+in information retrieval (IR) [10, 11] and natural language
+processing (NLP) [12–14]. However, they often suffer from
+the curse of dimensionality: it is difficult to accurately
+estimate high-order language models since an exponential
+number of transition probabilities need to be estimated.
+Thus, specially designed smoothing strategies such as backoff estimation [15] and Good–Turing estimation [16] have
+been introduced to alleviate the data sparsity problem.
+• Neural language models (NLM). NLMs [1, 17, 18] characterize the probability of word sequences by neural networks,
+e.g., multi-layer perceptron (MLP) and recurrent neural networks (RNNs). As a remarkable contribution, the work in
+[1] introduced the concept of distributed representation of
+words and built the word prediction function conditioned
+on the aggregated context features (i.e., the distributed
+word vectors). By extending the idea of learning effective
+features for text data, a general neural network approach
+was developed to build a unified, end-to-end solution for
+various NLP tasks [2]. Furthermore, word2vec [19, 20] was
+proposed to build a simplified shallow neural network
+for learning distributed word representations, which were
+demonstrated to be very effective across a variety of NLP
+tasks. These studies have initiated the use of language
+models for representation learning (beyond word sequence
+modeling), having an important impact on the field of NLP.
+• Pre-trained language models (PLM). As an early attempt, ELMo [21] was proposed to capture context-aware
+word representations by first pre-training a bidirectional
+LSTM (biLSTM) network (instead of learning fixed word
+representations) and then fine-tuning the biLSTM network
+according to specific downstream tasks. Furthermore, based
+on the highly parallelizable Transformer architecture [22]
+with self-attention mechanisms, BERT [23] was proposed by
+pre-training bidirectional language models with specially
+designed pre-training tasks on large-scale unlabeled corpora. These pre-trained context-aware word representations
+are very effective as general-purpose semantic features,
+which have largely raised the performance bar of NLP
+tasks. This study has inspired a large number of follow-up
+work, which sets the “pre-training and fine-tuning” learning
+paradigm. Following this paradigm, a great number of studies on PLMs have been developed, introducing either different architectures [24, 25] (e.g., GPT-2 [26] and BART [24]) or
+improved pre-training strategies [27–29]. In this paradigm, it
+often requires fine-tuning the PLM for adapting to different
+downstream tasks.
+• Large language models (LLM). Researchers find that
+scaling PLM (e.g., scaling model size or data size) often
+leads to an improved model capacity on downstream tasks
+(i.e., following the scaling law [30]). A number of studies
+have explored the performance limit by training an ever
+larger PLM (e.g., the 175B-parameter GPT-3 and the 540Bparameter PaLM). Although scaling is mainly conducted
+in model size (with similar architectures and pre-training
+tasks), these large-sized PLMs display different behaviors
+from smaller PLMs (e.g., 330M-parameter BERT and 1.5Bparameter GPT-2) and show surprising abilities (called emergent abilities [31]) in solving a series of complex tasks. For
+example, GPT-3 can solve few-shot tasks through in-context
+learning, whereas GPT-2 cannot do well. Thus, the research
+community coins the term “large language models (LLM)”
+1
+for these large-sized PLMs [32–35], which attract increasing
+research attention (See Figure 1). A remarkable application
+of LLMs is ChatGPT2
+that adapts the LLMs from the GPT
+series for dialogue, which presents an amazing conversation
+ability with humans. We can observe a sharp increase of the
+arXiv papers that are related to LLMs after the release of
+ChatGPT in Figure 1.
+As discussed before, language model is not a new technical concept specially for LLMs, but has evolved with the
+advance of artificial intelligence over the decades. Early language models mainly aim to model and generate text data,
+while latest language models (e.g., GPT-4) focus on complex
+task solving. From language modeling to task solving, it is an
+important leap in scientific thinking, which is the key to
+understand the development of language models in the research history. From the perspective of task solving, the four
+generations of language models have exhibited different levels of model capacities. In Figure 2, we describe the evolution process of language models in terms of the task solving
+capacity. At first, statistical language models mainly assisted
+in some specific tasks (e.g., retrieval or speech tasks), in
+which the predicted or estimated probabilities can enhance
+the performance of task-specific approaches. Subsequently,
+neural language models focused on learning task-agnostic
+representations (e.g., features), aiming to reduce the efforts
+for human feature engineering. Furthermore, pre-trained
+language models learned context-aware representations that
+can be optimized according to downstream tasks. For the
+latest generation of language model, LLMs are enhanced by
+exploring the scaling effect on model capacity, which can be
+considered as general-purpose task solvers. To summarize,
+in the evolution process, the task scope that can be solved
+by language models have been greatly extended, and the
+task performance attained by language models have been
+significantly enhanced.
+In the existing literature, PLMs have been widely discussed and surveyed [36–39], while LLMs are seldom reviewed in a systematic way. To motivate our survey, we first
+highlight three major differences between LLMs and PLMs.
+First, LLMs display some surprising emergent abilities that
+may not be observed in previous smaller PLMs. These abilities are key to the performance of language models on complex tasks, making AI algorithms unprecedently powerful
+and effective. Second, LLMs would revolutionize the way
+that humans develop and use AI algorithms. Unlike small
+PLMs, the major approach to accessing LLMs is through
+the prompting interface (e.g., GPT-4 API). Humans have to
+understand how LLMs work and format their tasks in a way
+that LLMs can follow. Third, the development of LLMs no
+longer draws a clear distinction between research and engineering. The training of LLMs requires extensive practical
+experiences in large-scale data processing and distributed
+parallel training. To develop capable LLMs, researchers
+have to solve complicated engineering issues, working with
+engineers or being engineers.
+Nowadays, LLMs are posing a significant impact on
+the AI community, and the advent of ChatGPT and GPT-4
+leads to the rethinking of the possibilities of artificial general
+intelligence (AGI). OpenAI has published a technical article
+entitled “Planning for AGI and beyond”, which discusses
+the short-term and long-term plans to approach AGI [40],
+and a more recent paper has argued that GPT-4 might be
+considered as an early version of an AGI system [41]. The
+research areas of AI are being revolutionized by the rapid
+progress of LLMs. In the field of NLP, LLMs can serve as a
+general-purpose language task solver (to some extent), and
+the research paradigm has been shifting towards the use
+of LLMs. In the field of IR, traditional search engines are
+challenged by the new information seeking way through AI
+chatbots (i.e., ChatGPT), and New Bing3 presents an initial
+attempt that enhances the search results based on LLMs. In
+the field of CV, the researchers try to develop ChatGPT-like
+vision-language models that can better serve multimodal
+dialogues [42–45], and GPT-4 [46] has supported multimodal input by integrating the visual information. This new
+wave of technology would potentially lead to a prosperous
+ecosystem of real-world applications based on LLMs. For
+instance, Microsoft 365 is being empowered by LLMs (i.e.,
+Copilot) to automate the office work, and OpenAI supports
+the use of plugins in ChatGPT for implementing special
+functions.
+Despite the progress and impact, the underlying principles of LLMs are still not well explored. Firstly, it is
+mysterious why emergent abilities occur in LLMs, instead of
+smaller PLMs. As a more general issue, there lacks a deep,
+detailed investigation of the key factors that contribute to
+the superior abilities of LLMs. It is important to study when
+and how LLMs obtain such abilities [47]. Although there are
+some meaningful discussions about this problem [31, 47],
+more principled investigations are needed to uncover the
+“secrets“ of LLMs. Secondly, it is difficult for the research
+community to train capable LLMs. Due to the huge demand of computation resources, it is very costly to carry
+out repetitive, ablating studies for investigating the effect
+of various strategies for training LLMs. Indeed, LLMs are
+mainly trained by industry, where many important training
+details (e.g., data collection and cleaning) are not revealed
+to the public. Thirdly, it is challenging to align LLMs with
+human values or preferences. Despite the capacities, LLMs
+are also likely to produce toxic, fictitious, or harmful contents. It requires effective and efficient control approaches
+to eliminating the potential risk of the use of LLMs [46].
+Faced with both opportunities and challenges, it needs
+more attention on the research and development of LLMs. In
+order to provide a basic understanding of LLMs, this survey
+conducts a literature review of the recent advances in LLMs
+from four major aspects, including pre-training (how to pretrain a capable LLM), adaptation (how to effectively adapt
+pre-trained LLMs for better use), utilization (how to use
+LLMs for solving various downstream tasks) and capability
+evaluation (how to evaluate the abilities of LLMs and existing
+empirical findings). We thoroughly comb the literature and
+summarize the key findings, techniques, and methods of
+LLMs. For this survey, we also create a GitHub project
+website by collecting the supporting resources for LLMs, at
+the link https://github.com/RUCAIBox/LLMSurvey. We
+are also aware of several related review articles on PLMs
+or LLMs [32, 36, 38, 39, 43, 48–54]. These papers either
+discuss PLMs or some specific (or general) aspects of LLMs.
+Compared with them, we focus on the techniques and
+methods to develop and use LLMs and provide a relatively
+comprehensive reference to important aspects of LLMs.
+The remainder of this survey is organized as follows:
+Section 2 introduces the background for LLMs and the evolution of GPT-series models, followed by the summarization
+of available resources for developing LLMs in Section 3.
+Sections 4, 5, 6, and 7 review and summarize the recent
+progress from the four aspects of pre-training, adaptation,
+utilization, and capacity evaluation, respectively. Then, Section 8 discusses the practical guide for prompt design,
+and Section 9 reviews the applications of LLMs in several
+representative domains. Finally, we conclude the survey in
+Section 10 by summarizing the major findings and discuss
+the remaining issues for future work.
+'''
+
+
+    conclusion_generator = ConclusionGenerator(Global_pipeline)
+    with_lora = conclusion_generator.generate(title, intro, mode='lora')
+    # print("The conclusion generated with LORA is: \n", with_lora)
+    # print("=============================================================")
+    with_test = conclusion_generator.generate(title, intro, mode='test')
+    # print("The conclusion generated with test is: \n", with_test)
+    # print("=============================================================")

src/demo/asg_generator.py

@@ -0,0 +1,90 @@
+from openai import OpenAI
+import transformers
+import os
+import re
+import ast
+import json
+import base64
+
+def getQwenClient(): 
+    openai_api_key = os.environ.get("OPENAI_API_KEY")
+    openai_api_base = os.environ.get("OPENAI_API_BASE")
+
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    return client
+
+def generateResponse(client, prompt):
+    chat_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=768,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=[{"role": "user", "content": prompt}]
+    )
+
+    text = ""
+    for chunk in chat_response:
+        if chunk.choices[0].delta.content:
+            text += chunk.choices[0].delta.content
+    return text
+
+def generate_sentence_patterns(keyword, num_patterns=5, temp=0.7):
+    template = f"""
+You are a helpful assistant that provides only the output requested, without any additional text.
+
+Please generate {num_patterns} commonly used sentence templates in academic papers to describe the '{keyword}'.
+- Do not include any explanations, sign-offs, or additional text.
+- The list should be in the following format:
+[
+    "First template should be here",
+    "Second template should be here",
+    ...
+]
+
+Begin your response immediately with the list, and do not include any other text.
+"""
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    response = generateResponse(client, template)
+    return response
+
+def generate(context, keyword, paper_title, temp=0.7):
+    template = f"""
+Context:
+{context}
+------------------------------------------------------------
+Based on the above context, answer the question: What {keyword} are mentioned in the paper {paper_title}?
+Please provide a direct answer in one paragraph, no longer than 100 words. 
+
+If the context provides enough information, answer strictly based on it. 
+If the context provided does not contain any specified {keyword}, deduce and integrate your own opinion as if the {keyword} were described in the context. 
+Ensure that your answer remains consistent with the style and format of the provided context, as if the information you provide is naturally part of it.
+------------------------------------------------------------
+Answer:
+The {keyword} mentioned in this paper discuss [Your oberservation or opinion]...
+"""
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    response = generateResponse(client, template)
+    return response
+
+def extract_query_list(text):
+    pattern = re.compile(
+        r'\[\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*,\s*"[^"]+"\s*\]'
+    )
+    match = pattern.search(text)
+    if match:
+        return match.group(0)
+    return None

src/demo/asg_latex.py

@@ -0,0 +1,816 @@
+import re
+import subprocess
+import os
+
+from openai import OpenAI
+import dotenv
+from .asg_add_flowchart import insert_tex_images
+from .asg_mindmap import insert_outline_figure
+
+
+def _remove_div_blocks(lines):
+    """
+    从给定的行列表中，移除所有形如:
+      <div style="...">
+          ... (若干行)
+      </div>
+    的 HTML 块（含首尾 <div> ... </div>）整段跳过。
+    返回处理后的新行列表。
+    """
+    new_lines = []
+    i = 0
+    n = len(lines)
+
+    while i < n:
+        line = lines[i]
+        # 如果该行以 <div style= 开头，则进入跳过模式
+        if line.strip().startswith("<div style="):
+            # 跳过本行
+            i += 1
+            # 一直向后找，直到遇到 '</div>' 行
+            while i < n and not lines[i].strip().startswith("</div>"):
+                i += 1
+            # 这里再跳过 '</div>' 那一行
+            i += 1
+        else:
+            new_lines.append(line)
+            i += 1
+
+    return new_lines
+
+def _convert_setext_to_atx(lines):
+    """
+    将形如:
+
+        标题文字
+        ===
+    
+    转换为:
+    
+        # 标题文字
+
+    将形如:
+
+        标题文字
+        ---
+
+    转换为:
+
+        ## 标题文字
+    """
+    setext_equal_pattern = re.compile(r'^\s*=+\s*$')  # 匹配全 `===`
+    setext_dash_pattern  = re.compile(r'^\s*-+\s*$')  # 匹配全 `---`
+
+    new_lines = []
+    i = 0
+    n = len(lines)
+
+    while i < n:
+        line = lines[i]
+        if i < n - 1:
+            next_line = lines[i + 1].strip()
+            # 若下一行是 ===
+            if setext_equal_pattern.match(next_line):
+                heading_text = line.strip()
+                new_lines.append(f"# {heading_text}")
+                i += 2  # 跳过下一行
+                continue
+            # 若下一行是 ---
+            if setext_dash_pattern.match(next_line):
+                heading_text = line.strip()
+                new_lines.append(f"## {heading_text}")
+                i += 2
+                continue
+        # 否则不改动
+        new_lines.append(line)
+        i += 1
+
+    return new_lines
+
+def preprocess_md(md_input_path: str, md_output_path: str = None) -> str:
+    """
+    预处理一个 Markdown 文件：
+      1. 移除所有 <div style="..."> ... </div> 这类 HTML 块
+      2. 将 setext 标题 (===, ---) 转为 ATX 标题 (#, ##)
+      3. 覆盖写回或输出到新文件
+    
+    参数:
+      md_input_path: 原始 Markdown 文件路径
+      md_output_path: 处理后要写出的 Markdown 文件路径; 若为 None 则覆盖原始文件.
+
+    返回:
+      str: 返回处理后 Markdown 文件的实际写出路径 (md_output_path).
+    """
+    if md_output_path is None:
+        md_output_path = md_input_path
+    
+    # 1) 读入行
+    with open(md_input_path, 'r', encoding='utf-8') as f:
+        lines = f.read().splitlines()
+
+    # 2) 移除 <div style="..."> ... </div> 片段
+    lines_no_div = _remove_div_blocks(lines)
+
+    # 3) 将 setext 标题转换为 ATX
+    lines_atx = _convert_setext_to_atx(lines_no_div)
+
+    # 4) 写出
+    with open(md_output_path, 'w', encoding='utf-8') as f:
+        for ln in lines_atx:
+            f.write(ln + "\n")
+
+    return md_output_path
+
+def search_sections(md_path: str):
+    """
+    解析仅含 ATX 风格标题的 Markdown 文件，返回一个列表，
+    每个元素是一个三元组: (level, heading_text, content_string)
+    
+    说明:
+      - 标题行形如 "# 标题"、"## 标题"、"### 标题" 等（在井号后有一个空格）。
+      - level = (井号个数 - 1)，即 "# -> level=0"、"## -> level=1"、"### -> level=2" ...
+      - 移除类似 "3.1.3 "、"2.10.1 " 这类数字点前缀（含其后空格）。
+      - content_string 为该标题之后、直到下一个标题行或文件结束为止的所有文本（换行拼接）。
+    """
+
+    # 用于匹配 ATX 标题（如 "# 标题", "## 3.1.3 标题" 等）
+    atx_pattern = re.compile(r'^(#+)\s+(.*)$')
+    
+    # 用于去除标题前缀的数字.数字.数字... (可能有空格)
+    # 示例匹配: "3.1.3 "、"2.10.1 " 等
+    leading_numbers_pattern = re.compile(r'^\d+(\.\d+)*\s*')
+
+    # 读入行
+    with open(md_path, "r", encoding="utf-8") as f:
+        lines = f.read().splitlines()
+
+    sections = []
+    i = 0
+    n = len(lines)
+
+    def gather_content(start_idx: int):
+        """
+        从 start_idx 开始，收集正文，直到遇到下一个 ATX 标题或文档末尾。
+        返回 (content_string, end_idx).
+        """
+        content_lines = []
+        idx = start_idx
+        while idx < n:
+            line = lines[idx].rstrip()
+            # 如果此行匹配到 ATX 标题模式，则停止收集正文
+            if atx_pattern.match(line):
+                break
+            content_lines.append(lines[idx])
+            idx += 1
+        return "\n".join(content_lines), idx
+
+    while i < n:
+        line = lines[i].rstrip()
+
+        # 判断是否为 ATX 标题
+        match_atx = atx_pattern.match(line)
+        if match_atx:
+            # group(1) 例如 "##"
+            # group(2) 例如 "3.1 Introduction"
+            hashes = match_atx.group(1)
+            heading_text_raw = match_atx.group(2).strip()
+            
+            # 计算标题层级: "# -> level=0, ## -> level=1, ### -> level=2"
+            heading_level = len(hashes) - 1
+
+            # 移除类似 "3.1.3 " 的前缀
+            heading_text = leading_numbers_pattern.sub('', heading_text_raw).strip()
+
+            i += 1  # 跳过标题行，准备收集正文
+            content_string, new_idx = gather_content(i)
+
+            sections.append((heading_level, heading_text, content_string))
+            i = new_idx
+            print(heading_level, heading_text)
+        else:
+            # 否则跳到下一行
+            i += 1
+        
+        # [可选调试输出] 打印当前标题层级及其文本
+        
+
+    return sections[1:]
+    
+def abstract_to_tex(section):
+    """
+    将 Markdown 中的 abstract 段落转化为 LaTeX 片段。
+
+    参数:
+        section: (level, heading_text, content_string)
+            level: 0 表示一级标题, 1 表示二级标题, etc.
+            heading_text: 当前标题文字
+            content_string: 该标题下的 Markdown 文本
+    
+    返回:
+        一个字符串，包含对应的 LaTeX abstract 环境。
+    """
+    level, heading_text, content_string = section
+
+    # 如果标题不是 "Abstract"，则直接返回空字符串
+    if heading_text.lower() != "abstract":
+        return ""
+
+    # 生成 LaTeX abstract 环境
+    latex_abstract = (
+        "\\begin{abstract}\n"
+        f"{content_string}\n"
+        "\\end{abstract}"
+    )
+    return latex_abstract
+
+def references_to_tex(section):
+    """
+    将 Markdown 中的 references 段落转化为 LaTeX 片段。
+
+    参数:
+        section: (level, heading_text, content_string)
+            level: 0 表示一级标题, 1 表示二级标题, etc.
+            heading_text: 当前标题文字
+            content_string: 该标题下的 Markdown 文本
+    
+    返回:
+        一个字符串，包含对应的 LaTeX references 环境。
+    """
+    level, heading_text, content_string = section
+
+    # 如果标题不是 "References"，则直接返回空字符串
+    if heading_text.lower() != "references":
+        return ""
+
+    # 在每一行的末尾添加 \\ 以实现换行
+    lines = content_string.splitlines()
+    latex_content = " \\\\{}\n".join(line.strip() for line in lines if line.strip())
+
+    # 生成 LaTeX 片段，使用 \section* 创建不带编号的标题
+    latex_references = (
+        "\\section*{References}\n"  # 不带编号的 section
+        f"{latex_content}"
+    )
+    return latex_references
+
+def md_to_tex_section(section):
+
+    """
+    将单个 Markdown 分段 (level, heading, content) 转化为 LaTeX 片段。
+    会根据标题的深度生成 \\section, \\subsection, 或 \\subsubsection 等。
+    同时对 markdown 中的图片 div 进行正则替换，转化为 LaTeX figure 环境。
+    
+    参数:
+        section: (level, heading_text, content_string)
+            level: 0 表示一级标题, 1 表示二级标题, etc.
+            heading_text: 当前标题文字
+            content_string: 该标题下的 Markdown 文本
+    
+    返回:
+        一个字符串，包含对应的 LaTeX 标题以及内容。
+        内容由 OpenAI 模型将 Markdown 转为 LaTeX，并将图片 div 转为 LaTeX figure。
+    """
+    level, heading_text, content_string = section
+    
+    # 根据 heading level 生成对应的 LaTeX 命令
+    if level == 0:
+        latex_heading = f"\\section{{{heading_text}}}"
+    elif level == 1:
+        latex_heading = f"\\subsection{{{heading_text}}}"
+    elif level == 2:
+        latex_heading = f"\\subsubsection{{{heading_text}}}"
+    else:
+        # 更深入的层级可自行添加
+        latex_heading = f"\\paragraph{{{heading_text}}}"
+    
+    # 先粗略替换图片 div 为占位符，后续交由 OpenAI 模型或自身再做处理
+    # 这里我们先把 <div style="text-align:center">...<img ...>...</div><div ...>Fig x: ...</div> 转换为一个自定义标记 [IMG_BLOCK] ... [END_IMG_BLOCK]
+    # 这样后面可以更好控制让 OpenAI 转成正确的 LaTeX 也行，或在本地处理也行。
+    # 这里我们本地进行处理，将它直接转换为 LaTeX figure。
+    
+    def replace_img_div(match):
+        """
+        将 <div style="text-align:center"> <img src="..." alt="..." style="width:60%;"/> </div>
+        <div style="text-align:center;font-size:smaller;">Fig x: ...</div>
+        这种模式转换为标准 LaTeX figure 环境
+        """
+        whole_block = match.group(0)
+        
+        # 提取 src
+        src_match = re.search(r'<img.*?src="(.*?)".*?>', whole_block, re.DOTALL)
+        src_path = src_match.group(1) if src_match else "image_not_found"
+        
+        # 提取 alt
+        alt_match = re.search(r'<img.*?alt="(.*?)".*?>', whole_block, re.DOTALL)
+        alt_text = alt_match.group(1) if alt_match else ""
+        
+        # 提取 caption (Fig x: ...)
+        fig_match = re.search(r'Fig\s*\d+:\s*(.*?)<\/div>', whole_block, re.DOTALL)
+        fig_caption = fig_match.group(1).strip() if fig_match else ""
+        
+        # 生成 LaTeX figure
+        latex_figure = (
+            "\\begin{figure}[htbp]\n"
+            "  \\centering\n"
+            f"  \\includegraphics[width=0.6\\textwidth]{{{src_path}}}\n"
+            f"  \\caption{{{alt_text if alt_text else fig_caption}}}\n"
+            # 也可以根据需求决定是否加 label
+            "\\end{figure}\n"
+        )
+        
+        return latex_figure
+    
+    # 用正则定位该模式并转换为 latex figure
+    # 该模式大概是:
+    # <div style="text-align:center">.*?<img src="...".*?>.*?</div>\s*<div style="text-align:center;font-size:smaller;">.*?</div>
+    # 这里用非贪婪模式, DOTALL 允许匹配换行
+    pattern_img_div = re.compile(
+        r'<div\s+style="text-align:center".*?>.*?<img.*?>.*?</div>\s*<div\s+style="text-align:center;font-size:smaller;">.*?<\/div>',
+        re.DOTALL
+    )
+    
+    content_converted_images = re.sub(pattern_img_div, replace_img_div, content_string)
+    
+    # ------------------------------------------------
+    # 调用 OpenAI 接口，将 (转换好图片 div 的) Markdown 转为 LaTeX
+    # ------------------------------------------------
+    system_prompt = (
+        "You are a helpful assistant that converts Markdown text to rigorous LaTeX. "
+        "Maintain inline formatting like bold, italics, and code blocks when possible. "
+        "Simply format horizontally aligned text, lists, tables, etc. into valid LaTeX."
+        "Use [LaTeX] ... [/LaTeX] to wrap the final content without the \\section\{\}."
+        "If the content is mathematically descriptive, please insert exactly one LaTeX math equation with explaination ($...$)to describe it."
+        "Do not include any other irrelevant information."
+        "Remember to clean the refs such as \[1], \[2], \[3] inside the text to strip the backslashes to [1], [2], [3]. No any extra backslashes."
+    )
+    
+    user_prompt = (
+        "Convert the following Markdown content to LaTeX. The text may already contain "
+        "some partial LaTeX for figures:\n\n"
+        f"{content_converted_images}"
+    )
+    
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    # 从环境变量中获取 openai key 和 base url
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    
+    # 初始化 Client
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    chat_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=2048,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages
+    )
+    
+    # Stream the response
+    tex_body = ""
+    for chunk in chat_response:
+        if chunk.choices[0].delta.content:
+            tex_body += chunk.choices[0].delta.content
+    
+    # 假设我们想在聊天回复中，使用 [LaTeX] ... [/LaTeX] 包裹最终内容，类似:
+    # [LaTeX]
+    # 你要的 tex body ...
+    # [/LaTeX]
+    
+    # 可以用正则截取中间内容:
+    pattern = r'\[LaTeX\](.*?)\[/LaTeX\]'
+    match = re.search(pattern, tex_body, re.DOTALL)
+    if match:
+        # 如果拿到中间的内容 就用它, 否则就用全部
+        tex_body = match.group(1).strip()
+    
+    # 去掉多余的空白
+    tex_body = re.sub(r'\s+', ' ', tex_body).strip()
+    
+    # 整合 LaTeX 标题和转好的正文
+    final_tex_snippet = latex_heading + "\n\n" + tex_body + "\n"
+    print("Tex snippet:")
+    print(final_tex_snippet)
+    return final_tex_snippet
+
+def md_to_tex_section_without_jpg(section):
+    """
+    将单个 Markdown 分段 (level, heading_text, content_string) 转化为 LaTeX 片段，
+    不处理任何 HTML 或图片 div，仅调用 OpenAI 模型将普通 Markdown 转为 LaTeX。
+
+    参数:
+        section: (level, heading_text, content_string)
+            - level: 0 表示一级标题, 1 表示二级标题, 2 表示三级标题等
+            - heading_text: 当前标题文字
+            - content_string: 该标题下的 Markdown 文本
+
+    返回:
+        一个字符串，包含对应的 LaTeX 标题以及转换后的正文。
+    """
+
+    level, heading_text, content_string = section
+    
+    # 1) 根据 level 生成对应的 LaTeX 命令
+    #    你也可以改成更灵活的逻辑，比如多级。
+    if level == 0:
+        latex_heading = f"\\section{{{heading_text}}}"
+    elif level == 1:
+        latex_heading = f"\\subsection{{{heading_text}}}"
+    elif level == 2:
+        latex_heading = f"\\subsubsection{{{heading_text}}}"
+    else:
+        latex_heading = f"\\paragraph{{{heading_text}}}"
+
+    # 2) 判断是否要跳过 LLM 转换
+    #    这里给出几种常见原因：
+    #    - 内容字符串为空或全是空白 (content_string.strip() == "")
+    #    - 标题看起来只是一个段落号, 形如"3"、"3.1"、"3.1.1" 等 (可根据需要调宽或调窄判断规则)
+
+    # 例：用一个正则匹配 `数字(.数字)*`，可带可不带后缀空格
+    #   如果 heading_text 完全匹配这个模式，就认为它是个“纯编号标题”，不必调用 LLM
+    pure_number_pattern = re.compile(r'^\d+(\.\d+)*$')
+
+    # 先去一下两端空格
+    ht_stripped = heading_text.strip()
+    # 若正文为空，或标题是纯数字/编号，就跳过 LLM
+    skip_llm = (not content_string.strip()) or bool(pure_number_pattern.match(ht_stripped))
+
+    if skip_llm:
+        # 直接返回标题 + 原始正文 (若有也可保留)
+        # 如果你只想输出标题，就让正文为空
+        tex_body = content_string
+        # 也可以选择把正文丢弃，比如:
+        # tex_body = ""
+    else:
+        # 3) 需要调用 LLM 的情况
+        # system_prompt = (
+        #     "You are a helpful assistant that converts Markdown text to rigorous LaTeX. "
+        #     "Maintain inline formatting like bold, italics, and code blocks when possible. "
+        #     "Simply format horizontally aligned text, lists, tables, etc. into valid LaTeX."
+        #     "Use [LaTeX] ... [/LaTeX] to wrap the final content without the \\section\{\}."
+        #     "If the content is mathematically descriptive, please insert exactly one LaTeX math equation with explaination (\\[...\\])to describe it."
+        #     "You are forced to use \\begin{dmath} and \\end{dmath} to replace the origin square brackets and wrap the equation"
+        #     "Do not include any other irrelevant information."
+        #     "Remember to clean the refs such as \[1], \[2], \[3] inside the text to strip the backslashes to [1], [2], [3]. No any extra backslashes."
+        # )
+        system_prompt = (
+            "You are a helpful assistant that converts Markdown text to rigorous LaTeX. "
+            "Maintain inline formatting like bold, italics, and code blocks when possible. "
+            "Format horizontally aligned text, lists, and tables into valid LaTeX.\n\n"
+
+            "Use [LaTeX] ... [/LaTeX] to wrap the final content without the \\section{}.\n\n"
+            "If the content is mathematically descriptive, please insert exactly one LaTeX math equation to describe it."
+            "For mathematical content, strictly follow the **standard equation format** below:\n\n"
+
+            "1. **Wrap equations inside `equation`**:\n"
+            "   ```latex\n"
+            "   \\begin{equation}\n"
+            "       \\resizebox{0.95\\columnwidth}{!}{$\n"
+            "       ...  % (Insert the equation here)\n"
+            "       $}\n"
+            "   \\end{equation}\n"
+            "   ```\n"
+            "   - **All equations must be enclosed in `\\resizebox{0.95\\columnwidth}{!}{...}`**.\n"
+            "   - **Ensure the equation fits within `\\columnwidth`** in two-column layouts.\n\n"
+
+            "2. **For descriptions, simply use plain text with double backslashes, for example:\n"
+            "$f_i(x)$ is the local objective function of node $i$.\\"
+            "$\mathcal{N}_i$ is the set of in-neighbors of node $i$.\\"
+
+            "3. **Ensure proper formatting**:\n"
+            "   - **DO NOT use `align`, `multline`, or `split`**—only `equation` with `resizebox`.\n"
+            "   - **DO NOT allow formulas to exceed column width**.\n"
+            "   - **DO NOT allow any other latex syntax such as" 
+            "    \\documentclass{article} \\usepackage{amsmath} \\usepackage{graphicx} \\begin{document}** use the plain content with formula.\n"
+
+            "   - **Maintain the original refs and ensure that references like [1], [2], [3], do not contain unnecessary backslashes**.\n\n"
+
+            "All generated LaTeX content **must strictly adhere to this structure**."
+        )
+
+        user_prompt = (
+            "Convert the following Markdown content to LaTeX. "
+            f"{content_string}"
+        )
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+
+        # 从环境变量中获取 openai key 和 base url
+        openai_api_key = os.getenv("OPENAI_API_KEY")
+        openai_api_base = os.getenv("OPENAI_API_BASE")
+
+        # 初始化 Client
+        client = OpenAI(
+            api_key=openai_api_key,
+            base_url=openai_api_base,
+        )
+
+        chat_response = client.chat.completions.create(
+            model=os.environ.get("MODEL"),
+            max_tokens=2048,
+            temperature=0.5,
+            stop="<|im_end|>",
+            stream=True,
+            messages=messages
+        )
+
+        # 流式读取返���
+        tex_body = ""
+        for chunk in chat_response:
+            if chunk.choices[0].delta.content:
+                tex_body += chunk.choices[0].delta.content
+
+        # 提取 [LaTeX] ... [/LaTeX] 中间的内容
+        pattern = r'\[LaTeX\](.*?)\[/LaTeX\]'
+        match = re.search(pattern, tex_body, re.DOTALL)
+        if match:
+            tex_body = match.group(1).strip()
+
+        # 去掉多余的空白
+        tex_body = re.sub(r'\s+', ' ', tex_body).strip()
+
+    # 4) 最终拼接
+    final_tex_snippet = latex_heading + "\n\n" + tex_body + "\n"
+    print("Tex snippet:")
+    print(final_tex_snippet)
+    return final_tex_snippet
+
+def insert_section(tex_path: str, section_content: str):
+    """
+    将 section_content 追加到 .tex 文件“最后一个 section(或子节)的正文末尾”。
+    具体逻辑如下：
+      1. 如果文件内找不到任何 \section{...}、\subsection{...}、\subsubsection{...}，
+         那么就将 section_content 插入到 \end{abstract} 之后。
+      2. 如果在全文中能找到若干标题 (\section、\subsection、\subsubsection)，
+         则将 section_content 插入到最后出现的那个标题对应正文的末尾（即它和下一个标题/文件结束之间）。
+      3. 如果既没有 abstract 环境，也没有任何标题，则在 \end{document} 前插入。
+    
+    参数:
+        tex_path: str
+            .tex 文件的路径。
+        section_content: str
+            需要插入的段落字符串（LaTeX 格式）。
+    
+    注意：
+        - 这段逻辑会将新的内容**追加**到最后一个标题所对应正文的末尾，
+          这样可以避免把之前的内容“分割”或“顶开”。
+    """
+
+    if not os.path.exists(tex_path):
+        print(f"TeX 文件不存在: {tex_path}")
+        return
+
+    with open(tex_path, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+
+    # 正则匹配标题、abstract、document
+    # 注意 \section、\subsection、\subsubsection 都做单独分组，这样获取行号时好区分
+    title_pattern = re.compile(r'^(\\section|\\subsection|\\subsubsection)\{[^}]*\}')
+    end_abstract_pattern = re.compile(r'^\\end\{abstract\}')
+    end_document_pattern = re.compile(r'^\\end\{document\}')
+
+    # 找到所有标题行号，保存到列表
+    title_lines = []
+    end_abstract_line = None
+    end_document_line = None
+
+    for i, line in enumerate(lines):
+        if title_pattern.match(line.strip()):
+            title_lines.append(i)
+        elif end_abstract_pattern.match(line.strip()):
+            end_abstract_line = i
+        elif end_document_pattern.match(line.strip()):
+            end_document_line = i
+
+    # 将要插入的内容行列表
+    insert_content_lines = section_content.strip().split('\n')
+
+    # 如果找不到任何标题
+    if not title_lines:
+        # 如果有 \end{abstract}，就插在 \end{abstract} 后
+        if end_abstract_line is not None:
+            insert_idx = end_abstract_line + 1
+        else:
+            # 没有 \end{abstract}，就尝试在 \end{document} 之前插入
+            if end_document_line is not None:
+                insert_idx = end_document_line
+            else:
+                # 如果也没有 \end{document}，就插到文件末尾
+                insert_idx = len(lines)
+
+        new_lines = (
+            lines[:insert_idx]
+            + [l + "\n" for l in insert_content_lines]
+            + lines[insert_idx:]
+        )
+
+    else:
+        # 有标题时，将内容追加到“最后一个标题对应正文”的末尾
+        last_title_line = title_lines[-1]
+
+        # 找到下一个标题的行号（如果有），或 \end{document} 行号，以确定正文区间结束
+        # “最后标题正文”从 last_title_line+1 一直到 next_title_line-1（或结束）
+        next_boundaries = [end_document_line if end_document_line is not None else len(lines)]
+        for t_line in title_lines:
+            if t_line > last_title_line:
+                next_boundaries.append(t_line)
+        # next_boundary 是最后标题之后遇到的第一个 boundary（若没有, 就是文件末尾）
+        next_boundary = min(next_boundaries) if next_boundaries else len(lines)
+
+        # 我们希望将新的内容插在“最后标题正文的最末尾”之后，也就是说在 next_boundary 前。
+        # 不过若“最后标题”本身就处于全文件最终，next_boundary 可能表示文件末尾/文档结束。
+        # 这里为了避免把最后一行顶下去，可以先把其中的正文行都保留，再在最后插入 section_content。
+        new_lines = []
+        new_lines.extend(lines[:next_boundary])  # 保留从头到最后正文结束
+        new_lines.extend([l + "\n" for l in insert_content_lines])
+        new_lines.extend(lines[next_boundary:])
+
+    with open(tex_path, 'w', encoding='utf-8') as f:
+        f.writelines(new_lines)
+
+    print("成功插入 section 内容:", tex_path)
+
+def md_to_tex(md_path, tex_path, title):
+    """
+    将 Markdown 文件转换为 LaTeX 文件。
+
+    参数:
+        md_path (str): 输入的 Markdown 文件路径。
+        tex_path (str): 输出的 LaTeX 文件路径。
+    """
+    sections = search_sections(md_path)
+    section_index = 0
+    while section_index < len(sections):
+        print(f"Converting section {section_index+1}/{len(sections)}")
+        if section_index == 0:
+            tex = abstract_to_tex(sections[section_index])
+            print(tex)
+        elif section_index == len(sections) - 1:
+            postprocess(tex_path, title)
+            tex = references_to_tex(sections[section_index])
+            print(tex)
+        else:
+            tex = md_to_tex_section_without_jpg(sections[section_index])
+            print(tex)
+        insert_section(tex_path, tex)
+        section_index += 1
+    # tex_to_pdf(tex_path, output_dir=os.path.dirname(tex_path), compiler="pdflatex")
+
+def tex_to_pdf(tex_path, output_dir=None, compiler="xelatex"):
+    """
+    将 LaTeX 文件编译为 PDF 文件。
+
+    参数:
+        tex_path (str): 输入的 LaTeX 文件路径。
+        output_dir (str): 输出的 PDF 文件目录。
+        compiler (str): 编译器，默认为 "xelatex"。
+    """
+    if output_dir is None:
+        output_dir = os.path.dirname(tex_path)
+    tex_name = os.path.basename(tex_path)
+    tex_name_no_ext = os.path.splitext(tex_name)[0]
+    pdf_path = os.path.join(output_dir, f"{tex_name_no_ext}.pdf")
+    
+    subprocess.run([
+        compiler,
+        "-interaction=nonstopmode",
+        "-output-directory",
+        output_dir,
+        tex_path
+    ])
+    
+    print(f"PDF 文件已生成: {pdf_path}")
+
+def insert_figures(png_path, tex_path, json_path, ref_names, survey_title, new_tex_path):
+    """
+    读取给定的 TeX 文件 (tex_path)，先调用 insert_outline_figure 在其中插入概览图片；
+    然后再调用 insert_tex_images 在文中发现的引用标记位置插入 figure 环境。
+    最后把处理完的文本写入 new_tex_path。
+    
+    参数：
+      png_path:       大纲图片的路径（会传给 insert_outline_figure）。
+      tex_path:       原始 TeX 文件路径。
+      json_path:      图片对应的 JSON（会传给 insert_tex_images，内含 引用名称 -> 图片路径 的映射）。
+      ref_names:      引用名称列表 (index 从 0 开始)。
+      survey_title:   用于大纲图片 figure 中的说明文字。
+      new_tex_path:   处理后新的 TeX 文件输出路径。
+    """
+    # 1. 读取原始 tex 文件内容
+    with open(tex_path, 'r', encoding='utf-8') as f:
+        tex_content = f.read()
+
+    # 2. 在 '2 Introduction' 前插入一张占满整页的描述性图片（概览图）
+    updated_tex = insert_outline_figure(
+        png_path=png_path,
+        tex_content=tex_content,
+        survey_title=survey_title
+    )
+
+    # 3. 在文中其他引用 [n], \[n], \[n\] 等位置插入 figure
+    updated_tex = insert_tex_images(
+        json_path=json_path,
+        ref_names=ref_names,
+        text=updated_tex
+    )
+
+    # 4. 将处理结果写入新路径
+    with open(new_tex_path, 'w', encoding='utf-8') as f:
+        f.write(updated_tex)
+
+    print(f"已生成新的 TeX 文件: {new_tex_path}")
+    return new_tex_path
+
+def postprocess(tex_path, new_title):
+    """
+    读取给定的 TeX 文件 (tex_path):
+      1) 在第一处 \author 行的上一行插入 \title{new_title}。
+      2) 将所有形如 "\[1\]"、"\[1]"、以及 "\[12\]" 等引用标记，
+         以及 "[1\]" 之类的混合形式，全都去掉反斜杠，统一替换为 [1]、[12]。
+      3) 将所有由 \[ \] 包裹的数学公式都替换为 \begin{dmath} \end{dmath}。
+    最后将结果覆盖写回原始文件，并返回 tex_path。
+    """
+    new_title = 'A Survey of ' + new_title
+    # 1) 读取文件行
+    with open(tex_path, 'r', encoding='utf-8') as f:
+        lines = f.readlines()
+
+    # 2) 找到包含 "\author" 的行，在其上一行插入 \title{...}
+    inserted = False
+    for i, line in enumerate(lines):
+        # 如果这一行以 \author 开头，或包含 \author{...} 等
+        if line.strip().startswith(r'\author'):
+            # 在它前面插入一行 \title{new_title}
+            lines.insert(i, f'\\title{{{new_title}}}\n')
+            inserted = True
+            break
+
+    if not inserted:
+        # 若整份文件都没有 \author{...}，可以选择直接在文档末尾插入，或在开头插入，视需求而定
+        print(f"[警告] 未找到 '\\author' 行，未能插入 '\\title{{{new_title}}}'。")
+        # 下面演示直接在文档末尾插入（可按自己需求改放开头等）
+        lines.append(f'\\title{{{new_title}}}\n')
+
+    # 将行列表合并为一个整体字符串，便于进行正则替换
+    text_joined = ''.join(lines)
+
+    # 3) 将形如 "\[1\]"、"\[12]"、"[12\]" 等都换成 "[1]"、"[12]" 等
+    #    核心正则：'(?:\\)?\[(\d+)(?:\\)?\]'
+    #      (?:\\)? ---- 可选的一个反斜杠
+    #      \[      ---- 匹配方括号的开头 '['
+    #      (\d+)   ---- 匹配并捕获1--多位数字
+    #      (?:\\)? ---- 可选的一个反斜杠
+    #      \]      ---- 匹配方括号的结尾 ']'
+    ref_pattern = re.compile(r'(?:\\)?\[(\d+)(?:\\)?\]')
+    text_processed = ref_pattern.sub(r'[\1]', text_joined)
+
+    # 4) 将所有由 \[ \] 包裹的数学公式替换为 \begin{dmath} \end{dmath}
+    #    正则示例: 匹配 \[ ... \] 中间任意内容 (非贪婪)
+    #    使用 DOTALL 选项让 '.' 匹配换行
+    # eq_pattern = re.compile(r'\\\[(.*?)\\\]', re.DOTALL)
+    # text_processed = eq_pattern.sub(r'\\begin{dmath}\1\\end{dmath}', text_processed)
+
+    # 5) 写回原文件
+    with open(tex_path, 'w', encoding='utf-8') as f:
+        f.write(text_processed)
+
+    print(f"[完成] 已在 '{tex_path}' 中插入/追加 \\title{{{new_title}}}，替换引用标记并将公式转为 dmath 格式。")
+    return tex_path
+
+def md_to_tex_to_pdf(md_path, tex_path, pdf_path, png_path, json_path, ref_names, survey_title):
+    """
+    将 Markdown 文件转换为 LaTeX 文件，然后再编译为 PDF 文件。
+
+    参数:
+        md_path (str): 输入的 Markdown 文件路径。
+        tex_path (str): 输出的 LaTeX 文件路径。
+        pdf_path (str): 输出的 PDF 文件路径。
+    """
+    md_to_tex(md_path, tex_path)
+    new_tex_path = insert_figures(png_path, tex_path, json_path, ref_names, survey_title, tex_path)
+    # tex_to_pdf(new_tex_path, output_dir=os.path.dirname(tex_path), compiler="pdflatex")
+
+if __name__ == "__main__":
+    # 读取环境变量 
+    dotenv.load_dotenv()
+    # md_path = preprocess_md("src/demo/latex_template/test copy.md", "src/demo/latex_template/test_preprocessed.md")
+    md_path = 'src/static/data/info/undefined/survey_undefined_preprocessed.md'
+    tex_path = "src/static/data/info/undefined/template.tex"
+    md_to_tex(md_path, tex_path, title="A Comprehensive Review of ADMM On Consensus Distributed Optimization")
+    # insert_figures('src/static/data/info/undefined/outline.png', 
+    #                'src/demo/latex_template/template.tex', 
+    #                'src/static/data/info/undefined/flowchart_results.json', 
+    #                ['A comprehensive review of recommender systems transitioning from theory to practice', 'A large language model enhanced conversational recommender system'],
+    #                'Survey Title',
+    #                'src/demo/latex_template/template_with_figures.tex')
+    tex_to_pdf(tex_path, output_dir=os.path.dirname(tex_path), compiler="xelatex")

src/demo/asg_loader.py

@@ -0,0 +1,256 @@
+import os
+import re
+import json
+import subprocess
+from langchain_community.document_loaders import UnstructuredMarkdownLoader
+from langchain_core.documents import Document
+import shutil
+
+class DocumentLoading:
+    def convert_pdf_to_md(self, pdf_file, output_dir="output", method="auto"):
+        base_name = os.path.splitext(os.path.basename(pdf_file))[0]
+        target_dir = os.path.join(output_dir, base_name)
+        md_file_path = os.path.join(target_dir, method, f"{base_name}.md")
+        print("The md file path is: ", md_file_path)
+
+        if os.path.exists(md_file_path):
+            print(f"Markdown file for {pdf_file} already exists at {md_file_path}. Skipping conversion.", flush=True)
+            return
+            
+        command = ["magic-pdf", "-p", pdf_file, "-o", output_dir, "-m", method]
+        try:
+            subprocess.run(command, check=True)
+            # 检查是否生成了 Markdown 文件
+            if not os.path.exists(md_file_path):
+                print(f"Conversion failed: Markdown file not found at {md_file_path}. Cleaning up folder...")
+                shutil.rmtree(target_dir)  # 删除生成的文件夹
+            else:
+                print(f"Successfully converted {pdf_file} to markdown format in {target_dir}.")
+        except subprocess.CalledProcessError as e:
+            print(f"An error occurred during conversion: {e}")
+            # 如果发生错误且文件夹已生成，则删除文件夹
+            if os.path.exists(target_dir):
+                print(f"Cleaning up incomplete folder: {target_dir}")
+                shutil.rmtree(target_dir)
+    # new
+    def convert_pdf_to_md_new(self, pdf_dir, output_dir="output", method="auto"):
+        pdf_files = glob.glob(os.path.join(pdf_dir, "*.pdf"))
+
+        for pdf_file in pdf_files:
+            base_name = os.path.splitext(os.path.basename(pdf_file))[0]
+            target_dir = os.path.join(output_dir, base_name)
+
+            if os.path.exists(target_dir):
+                print(f"Folder for {pdf_file} already exists in {output_dir}. Skipping conversion.")
+            else:
+                command = ["magic-pdf", "-p", pdf_file, "-o", output_dir, "-m", method]
+                try:
+                    subprocess.run(command, check=True)
+                    print(f"Successfully converted {pdf_file} to markdown format in {target_dir}.")
+                except subprocess.CalledProcessError as e:
+                    print(f"An error occurred: {e}")
+
+    def batch_convert_pdfs(pdf_files, output_dir="output", method="auto", max_workers=None):
+        # Create a process pool to run the conversion in parallel
+        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+            # Submit each PDF file to the process pool for conversion
+            futures = [executor.submit(convert_pdf_to_md, pdf, output_dir, method) for pdf in pdf_files]
+
+            # Optionally, you can monitor the status of each future as they complete
+            for future in futures:
+                try:
+                    future.result()  # This will raise any exceptions that occurred during the processing
+                except Exception as exc:
+                    print(f"An error occurred during processing: {exc}")
+
+    def extract_information_from_md(self, md_text):
+        title_match = re.search(r'^(.*?)(\n\n|\Z)', md_text, re.DOTALL)
+        title = title_match.group(1).strip() if title_match else "N/A"
+
+        authors_match = re.search(
+            r'\n\n(.*?)(\n\n[aA][\s]*[bB][\s]*[sS][\s]*[tT][\s]*[rR][\s]*[aA][\s]*[cC][\s]*[tT][^\n]*\n\n)', 
+            md_text, 
+            re.DOTALL
+        )
+        authors = authors_match.group(1).strip() if authors_match else "N/A"
+
+        abstract_match = re.search(
+            r'(\n\n[aA][\s]*[bB][\s]*[sS][\s]*[tT][\s]*[rR][\s]*[aA][\s]*[cC][\s]*[tT][^\n]*\n\n)(.*?)(\n\n|\Z)', 
+            md_text, 
+            re.DOTALL
+        )
+        abstract = abstract_match.group(0).strip() if abstract_match else "N/A"
+        abstract = re.sub(r'^[aA]\s*[bB]\s*[sS]\s*[tT]\s*[rR]\s*[aA]\s*[cC]\s*[tT][^\w]*', '', abstract)
+        abstract = re.sub(r'^[^a-zA-Z]*', '', abstract)
+
+        introduction_match = re.search(
+            r'\n\n([1I][\.\- ]?\s*)?[Ii]\s*[nN]\s*[tT]\s*[rR]\s*[oO]\s*[dD]\s*[uU]\s*[cC]\s*[tT]\s*[iI]\s*[oO]\s*[nN][\.\- ]?\s*\n\n(.*?)'
+            r'(?=\n\n(?:([2I][I]|\s*2)[^\n]*?\n\n|\n\n(?:[2I][I][^\n]*?\n\n)))',
+            md_text, 
+            re.DOTALL
+        )
+        introduction = introduction_match.group(2).strip() if introduction_match else "N/A"
+
+        main_content_match = re.search(
+            r'(.*?)(\n\n([3I][\.\- ]?\s*)?[Rr][Ee][Ff][Ee][Rr][Ee][Nn][Cc][Ee][Ss][^\n]*\n\n|\Z)', 
+            md_text, 
+            re.DOTALL
+        )
+        
+        if main_content_match:
+            main_content = main_content_match.group(1).strip()
+        else:
+            main_content = "N/A"
+
+        extracted_data = {
+            "title": title,
+            "authors": authors,
+            "abstract": abstract,
+            "introduction": introduction,
+            "main_content": main_content
+        }
+        return extracted_data
+    
+    def process_md_file(self, md_file_path, survey_id):
+        loader = UnstructuredMarkdownLoader(md_file_path)
+        data = loader.load()
+        assert len(data) == 1, "Expected exactly one document in the markdown file."
+        assert isinstance(data[0], Document), "The loaded data is not of type Document."
+        extracted_text = data[0].page_content
+        
+        extracted_data = self.extract_information_from_md(extracted_text)
+        if len(extracted_data["abstract"]) < 10:
+            extracted_data["abstract"] = extracted_data['title']
+
+        title = os.path.splitext(os.path.basename(md_file_path))[0]
+        title_new = title.strip()
+        invalid_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*', '_']
+        for char in invalid_chars:
+            title_new = title_new.replace(char, ' ')
+
+        os.makedirs(f'./src/static/data/txt/{survey_id}', exist_ok=True)
+        with open(f'./src/static/data/txt/{survey_id}/{title_new}.json', 'w', encoding='utf-8') as f:
+            json.dump(extracted_data, f, ensure_ascii=False, indent=4)
+        return extracted_data['introduction']
+    
+    def process_md_file_full(self, md_file_path, survey_id):
+        loader = UnstructuredMarkdownLoader(md_file_path)
+        data = loader.load()
+        assert len(data) == 1, "Expected exactly one document in the markdown file."
+        assert isinstance(data[0], Document), "The loaded data is not of type Document."
+        extracted_text = data[0].page_content
+        
+        extracted_data = self.extract_information_from_md(extracted_text)
+        if len(extracted_data["abstract"]) < 10:
+            extracted_data["abstract"] = extracted_data['title']
+
+        title = os.path.splitext(os.path.basename(md_file_path))[0]
+        title_new = title.strip()
+        invalid_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*', '_']
+        for char in invalid_chars:
+            title_new = title_new.replace(char, ' ')
+ 
+        os.makedirs(f'./src/static/data/txt/{survey_id}', exist_ok=True)
+        with open(f'./src/static/data/txt/{survey_id}/{title_new}.json', 'w', encoding='utf-8') as f:
+            json.dump(extracted_data, f, ensure_ascii=False, indent=4)
+        return extracted_data['abstract'] + extracted_data['introduction'] + extracted_data['main_content']
+
+    
+    def load_pdf(self, pdf_file, survey_id, mode):
+        os.makedirs(f'./src/static/data/md/{survey_id}', exist_ok=True)
+        output_dir = f"./src/static/data/md/{survey_id}"
+        base_name = os.path.splitext(os.path.basename(pdf_file))[0]
+        target_dir = os.path.join(output_dir, base_name, "auto")
+
+        # 1. Convert PDF to markdown if the folder doesn't exist
+        self.convert_pdf_to_md(pdf_file, output_dir)
+
+        # 2. Process the markdown file in the output directory
+        md_file_path = os.path.join(target_dir, f"{base_name}.md")
+        if not os.path.exists(md_file_path):
+            raise FileNotFoundError(f"Markdown file {md_file_path} does not exist. Conversion might have failed.")
+
+        if mode == "intro":
+            return self.process_md_file(md_file_path, survey_id)
+        elif mode == "full":
+            return self.process_md_file_full(md_file_path, survey_id)
+
+    # wrong, still being tested
+    def load_pdf_new(self, pdf_dir, survey_id):
+        os.makedirs(f'./src/static/data/md/{survey_id}', exist_ok=True)
+        output_dir = f"./src/static/data/md/{survey_id}"
+        self.convert_pdf_to_md_new(pdf_dir, output_dir)
+        markdown_files = glob.glob(os.path.join(output_dir, "*", "auto", "*.md"))
+        all_introductions = []
+        
+        for md_file_path in markdown_files:
+            try:
+                introduction = self.process_md_file(md_file_path, survey_id)
+                all_introductions.append(introduction)
+            except FileNotFoundError as e:
+                print(f"Markdown file {md_file_path} does not exist. Conversion might have failed.")
+        
+        return all_introductions
+
+
+
+    def parallel_load_pdfs(self, pdf_files, survey_id, max_workers=4):
+        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+            # Submit tasks for parallel execution
+            futures = [executor.submit(self.load_pdf, pdf, survey_id) for pdf in pdf_files]
+            
+            # Collect results
+            for future in futures:
+                try:
+                    result = future.result()
+                    print(f"Processed result: {result}")
+                except Exception as e:
+                    print(f"Error processing PDF: {e}")
+    
+    def ensure_non_empty_introduction(self, introduction, full_text):
+        """
+        Ensure introduction is not empty. If empty, replace with full text.
+        """
+        if introduction == "N/A" or len(introduction.strip()) < 50:
+            return full_text.strip()
+        return introduction
+
+    def extract_information_from_md_new(self, md_text):
+        # Title extraction
+        title_match = re.search(r'^(.*?)(\n\n|\Z)', md_text, re.DOTALL)
+        title = title_match.group(1).strip() if title_match else "N/A"
+
+        # Authors extraction
+        authors_match = re.search(
+            r'\n\n(.*?)(\n\n[aA][\s]*[bB][\s]*[sS][\s]*[tT][\s]*[rR][\s]*[aA][\s]*[cC][\s]*[tT][^\n]*\n\n)', 
+            md_text, 
+            re.DOTALL
+        )
+        authors = authors_match.group(1).strip() if authors_match else "N/A"
+
+        # Abstract extraction
+        abstract_match = re.search(
+            r'(\n\n[aA][\s]*[bB][\s]*[sS][\s]*[tT][\s]*[rR][\s]*[aA][\s]*[cC][\s]*[tT][^\n]*\n\n)(.*?)(\n\n|\Z)', 
+            md_text, 
+            re.DOTALL
+        )
+        abstract = abstract_match.group(0).strip() if abstract_match else "N/A"
+        abstract = re.sub(r'^[aA]\s*[bB]\s*[sS]\s*[tT]\s*[rR]\s*[aA]\s*[cC]\s*[tT][^\w]*', '', abstract)
+        abstract = re.sub(r'^[^a-zA-Z]*', '', abstract)
+
+        # Introduction extraction
+        introduction_match = re.search(
+            r'\n\n([1I][\.\- ]?\s*)?[Ii]\s*[nN]\s*[tT]\s*[rR]\s*[oO]\s*[dD]\s*[uU]\s*[cC]\s*[tT]\s*[iI]\s*[oO]\s*[nN][\.\- ]?\s*\n\n(.*?)',
+            md_text, re.DOTALL
+        )
+        introduction = introduction_match.group(2).strip() if introduction_match else "N/A"
+
+        # Ensure introduction is not empty
+        introduction = self.ensure_non_empty_introduction(introduction, md_text)
+
+        return {
+            "title": title,
+            "authors": authors,
+            "abstract": abstract,
+            "introduction": introduction
+        }

src/demo/asg_mindmap.py

@@ -0,0 +1,302 @@
+import json
+import re
+import textwrap
+from graphviz import Digraph
+import os
+
+def wrap_text(text, max_chars):
+    """
+    对文本进行自动换行包装，每行最大字符数为 max_chars。
+    """
+    return textwrap.fill(text, width=max_chars)
+
+def parse_md_refs(md_content):
+    """
+    解析 Markdown 内容，提取以 x.y.z 格式标题对应的引用。
+    
+    对于每个满足格式的 section，其内容中所有形如 [数字] 的引用
+    将被抽取出来，去重后按数字升序排序，生成类似 "[1,2,3]" 的引用字符串。
+    
+    如果遇到 undesired header（如 "6 Future Directions" 或 "7 Conclusion"），
+    则停止后续内容的解析，确保最后一个 section 仅包含到该 header 之前的内容。
+
+    返回字典，键为 section 编号（例如 "3.1.1"），值为引用字符串（例如 "[1,2,3]"）。
+    """
+    ref_dict = {}
+
+    # 处理 Markdown 内容（按行拆分）
+    lines = md_content.split("\n") if md_content else []
+
+    # 匹配 Markdown 标题中以 x.y.z 开头的叶节点（例如 "5.1.1 Neural Topic..."）
+    section_header_regex = re.compile(r'^\s*#+\s*(\d+\.\d+\.\d+).*')
+    # 匹配 undesired header，如 "6 Future Directions" 或 "7 Conclusion"
+    undesired_header_regex = re.compile(r'^\s*#+\s*(6 Future Directions|7 Conclusion)\b')
+    # 匹配引用，例如 [数字]
+    ref_pattern = re.compile(r'\[(\d+)\]')
+    
+    current_section = None
+    current_content = []
+
+    for line in lines:
+        # 如果检测到 undesired header，则先处理当前 section，再退出循环
+        if undesired_header_regex.match(line):
+            break
+
+        header_match = section_header_regex.match(line)
+        if header_match:
+            # 处理上一个 section
+            if current_section is not None:
+                all_refs = [int(num) for content_line in current_content for num in ref_pattern.findall(content_line)]
+                if all_refs:
+                    ref_dict[current_section] = "[" + ",".join(map(str, sorted(set(all_refs)))) + "]"
+            
+            # 更新当前 section
+            current_section = header_match.group(1)
+            current_content = []
+        else:
+            if current_section is not None:
+                current_content.append(line)
+
+    # 处理最后一个 section
+    if current_section is not None and current_content:
+        all_refs = [int(num) for content_line in current_content for num in ref_pattern.findall(content_line)]
+        if all_refs:
+            ref_dict[current_section] = "[" + ",".join(map(str, sorted(set(all_refs)))) + "]"
+
+    return ref_dict
+
+def generate_graphviz_png(json_path, output_png_path, md_content=None, title="Document Outline", max_root_chars=20):
+    """
+    从 JSON 文件中读取大纲，构造树状结构，并生成 mindmap 的 PNG 图片。
+
+    如果提供了 md_content，则根据 Markdown 内容中以 x.y.z 格式标题对应的引用，
+    在生成 mindmap 时，对于叶节点（没有子节点且标题以 x.y.z 开头）的标签，
+    在原文本后追加一个换行，然后添加引用信息（例如 "[1,2,3]"），
+    且引用经过数字排序。
+    
+    同时，仅对根节点文本进行自动换行包装，以限制根节点的最大宽度，
+    其它节点保持原始文本格式。
+
+    参数:
+      json_path: JSON 文件路径（包含大纲）
+      output_png_path: 输出 PNG 文件路径（不带后缀）
+      md_content: Markdown 文本内容（字符串），可选
+      title: 用于替换 mindmap 中根节点的标题，默认 "Document Outline"
+      max_root_chars: 限制根节点每行最大字符数，默认 20
+    """
+    # 解析 Markdown 内容的引用
+    ref_dict = parse_md_refs(md_content) if md_content else {}
+
+    # 读取 JSON 大纲
+    with open(json_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    outline_str = data.get("outline", "")
+
+    # 解析形如 [层级, '标题'] 的项
+    pattern = re.compile(r"\[(\d+),\s*'([^']+)'\]")
+    items = pattern.findall(outline_str)
+    items = [(int(level), title) for level, title in items]
+
+    # 不需要的标题关键词
+    undesired_keywords = {"Abstract", "Introduction", "Future Directions", "Conclusion"}
+    # 过滤掉不需要的条目
+    filtered_items = [
+        (lvl, title) for lvl, title in items
+        if not re.match(r"^\d+\s+(.+)", title) or re.match(r"^\d+\s+(.+)", title).group(1) not in undesired_keywords
+    ]
+
+    # 构造树状结构
+    tree = []
+    stack = []
+    for lvl, title_item in filtered_items:
+        node = {"title": title_item, "children": []}
+        while stack and lvl <= stack[-1][0]:
+            stack.pop()
+        if stack:
+            stack[-1][1]["children"].append(node)
+        else:
+            tree.append(node)
+        stack.append((lvl, node))
+
+    # 生成 Mindmap
+    dot = Digraph(comment=title, format='png', engine='dot')
+    dot.graph_attr.update(rankdir='LR', splines='ortho', bgcolor='white', dpi="150")
+    dot.attr('node', shape='box', style='rounded,filled', fillcolor='white', color='gray')
+    dot.edge_attr.update(arrowhead='none', color="black")
+    
+    # 处理根节点
+    wrapped_title = wrap_text(title, max_root_chars)
+    dot.node('root', label=wrapped_title, shape='ellipse', style='filled', fillcolor='lightgray')
+
+    node_counter = [0]
+    section_pattern = re.compile(r'^(\d+\.\d+\.\d+)')
+
+    def add_nodes(node, parent_id):
+        current_id = f'node_{node_counter[0]}'
+        node_counter[0] += 1
+        safe_label = node['title'].replace('"', r'\"')
+        
+        # 如果是叶节点且标题以 x.y.z 开头，则追加引用信息（如果存在）
+        if not node["children"]:
+            m = section_pattern.match(safe_label)
+            if m:
+                section_id = m.group(1)
+                if section_id in ref_dict:
+                    safe_label += "\n" + ref_dict[section_id]
+        
+        dot.node(current_id, label=safe_label)
+        dot.edge(parent_id, current_id)
+        for child in node.get("children", []):
+            add_nodes(child, current_id)
+
+    for top_node in tree:
+        add_nodes(top_node, "root")
+
+    dot.render(output_png_path, cleanup=True)
+    print("生成 PNG 文件：", output_png_path + ".png")
+    return output_png_path + ".png"
+
+
+
+def insert_outline_image(png_path, md_content, survey_title):
+    """
+    在给定的 Markdown 内容字符串中查找 "2 Introduction" 这一行，
+    然后在该位置之前插入 outline 图片的 HTML 代码块，确保渲染时
+    HTML 块与后续 Markdown 内容间有足够空行分隔开。
+
+    参数：
+      png_path: 要插入的 PNG 图片路径，将作为 img 的 src 属性值。
+      md_content: Markdown 文件内容字符串。
+      survey_title: 用于生成图片说明文字的问卷标题。
+
+    插入的 HTML 格式如下：
+
+      <div style="text-align:center">
+          <img src="{png_path}" alt="Outline" style="width:100%;"/>
+      </div>
+      <div style="text-align:center">
+          Fig 1. The outline of the {survey_title}
+      </div>
+
+    函数返回更新后的 Markdown 内容字符串。
+    """
+
+    # 将 Markdown 内容字符串分割成行（保留换行符）
+    lines = md_content.splitlines(keepends=True)
+    print(lines)
+
+    # 查找包含 "2 Introduction" 的行的索引
+    intro_index = None
+    for i, line in enumerate(lines):
+        if '2 Introduction' in line:
+            intro_index = i
+            break
+
+    if intro_index is None:
+        print("没有找到 '2 Introduction' 这一行！")
+        return md_content
+
+    # 确保路径中的反斜杠被替换成正斜杠
+    png_path_fixed = png_path.replace("\\", "/")
+    
+    # 构造需要插入的 HTML 代码块，在前后增加空行
+    html_snippet = (
+        "\n\n"  # 添加换行确保与上文/下文分隔
+        f'<div style="text-align:center">\n'
+        f'    <img src="{png_path_fixed}" alt="Outline" style="width:100%;"/>\n'
+        f'</div>\n'
+        f'<div style="text-align:center">\n'
+        f'    Fig 1. The outline of the {survey_title}\n'
+        f'</div>\n'
+        "\n"  # 再添加一个空行确保与下方内容分隔
+    )
+    
+    print(f"将在第 {intro_index} 行插入如下 HTML 代码块（插入在 '2 Introduction' 之前）：\n{html_snippet}")
+    
+    # 在找到的 "2 Introduction" 这一行之前插入 html_snippet
+    lines.insert(intro_index, html_snippet)
+
+    # 合并所有行，构造更新后的 Markdown 内容
+    updated_md = "".join(lines)
+    
+    print("已在 Markdown 内容中插入 outline 图片。")
+    return updated_md
+
+def insert_outline_figure(png_path, tex_content, survey_title):
+    """
+    在给定的 TeX 文件内容字符串中查找 "2 Introduction" 这一行，
+    然后在其之前插入一个跨页(双栏)的 figure* 环境，包括整页显示的图片。
+    它将生成类似如下 LaTeX 片段：
+    
+    \begin{figure*}[htbp]
+      \centering
+      \includegraphics[width=\textwidth]{path/to/xxx.png}
+      \caption{Fig 1. The outline of the XXX}
+    \end{figure*}
+
+    参数：
+      png_path: 要插入的 PNG 图片路径
+      tex_content: TeX 文件内容字符串
+      survey_title: 用于生成图片 caption 的文献/问卷标题
+
+    返回:
+      更新后的 TeX 文本字符串
+    """
+
+    # 将 TeX 内容逐行分割（保留换行符）
+    lines = tex_content.splitlines(keepends=True)
+
+    # 查找包含 "2 Introduction" 的行索引
+    intro_index = None
+    for i, line in enumerate(lines):
+        if 'Introduction' in line:
+            intro_index = i
+            break
+
+    # 如果找不到，就直接返回原文
+    if intro_index is None:
+        print("没有找到 'Introduction' 这一行，未执行插入。")
+        return tex_content
+
+    # 构造 TeX 的 figure* 代码块
+    # 为确保整页，可用 [p] 或者 [htbp]，具体可根据排���需要调整
+    # 也可替换成普通 \begin{figure} ... \end{figure}，如果不需要跨双栏
+    figure_block = (
+        "\n"  # 加一个空行，确保与上文分隔
+        "\\begin{figure*}[htbp]\n"
+        "  \\centering\n"
+        f"  \\includegraphics[width=\\textwidth]{{{png_path}}}\n"
+        f"  \\caption{{The outline of our survey: {survey_title}}}\n"
+        "\\end{figure*}\n\n"  # 再留一个空行分隔
+    )
+
+    # 在找到的 "2 Introduction" 所在行之前插入 figure 环境
+    lines.insert(intro_index, figure_block)
+
+    # 重新拼接所有行
+    updated_tex = "".join(lines)
+    return updated_tex
+# 使用示例：
+# if __name__ == "__main__":
+#     png_path = 'src/static/data/info/test_4/outline.png'
+#     md_content = ''
+#     survey_title = "My Survey Title"
+#     updated_md = insert_outline_image(png_path, md_content, survey_title)
+# --------------------------
+# 使用示例
+# --------------------------
+if __name__ == "__main__":
+    json_path = os.path.join("src", "static", "data", "txt", 'test_2', "outline.json")
+    output_png_path = os.path.join("src", "static", "data", "info", 'test_2', "outline")
+    md_path = os.path.join("src", "static", "data", "info", 'test_2', f"survey_{'test_2'}_processed.md")
+    flowchart_results_path = os.path.join("src", "static", "data", "info", 'test_2', "flowchart_results.json")
+    png_path = generate_graphviz_png(
+            json_path=json_path,
+            output_png_path=output_png_path,
+            md_path=md_path,
+            title='test',
+            max_root_chars=30
+        )
+
+    # generate_graphviz_png(json_file_path, output_png_file, md_file_path, title=mindmap_title, max_root_chars=20)

src/demo/asg_outline.py

@@ -0,0 +1,1029 @@
+import transformers
+import torch
+import os
+import json
+import re
+import ast
+from .survey_generator_api import *
+from .asg_abstract import AbstractGenerator
+from .asg_conclusion import ConclusionGenerator
+from .asg_retriever import *
+import pandas as df
+from .references import generate_references
+
+
+class OutlineGenerator():
+    def __init__(self, pipeline, df, cluster_names, mode='desp'):
+        self.pipeline = pipeline
+        # self.pipeline.model.load_adapter("technicolor/llama3.1_8b_outline_generation")
+        self.pipeline.model.set_adapter("outline")
+        self.df = df
+        self.cluster = [{'label': i, 'name': cluster_names[i]} for i in range(len(cluster_names))]
+        self._add_cluster_info()
+        self.mode = mode
+
+    def __init__(self, df, cluster_names, mode='desp'): #Without local llms
+        self.df = df
+        self.cluster = [{'label': i, 'name': cluster_names[i]} for i in range(len(cluster_names))]
+        self._add_cluster_info()
+        self.mode = mode
+
+    def _add_cluster_info(self):
+        label_to_info = {label: self.df[self.df['label'] == label] for label in range(len(self.cluster))}
+        for cluster in self.cluster:
+            cluster['info'] = label_to_info[cluster['label']]
+
+    def get_cluster_info(self):
+        return self.cluster
+
+    def generate_claims(self):
+        result = []
+        if self.mode == 'desp':
+            for i in range(len(self.cluster)):
+                cluster = self.cluster[i]
+                claims = ''
+                for j in range(len(cluster['info'])):
+                    claims = cluster['info'].iloc[j]['retrieval_result'] + '\n' + claims
+                    # claims = cluster['info'].iloc[j]['ref_title'] + '\n' + claims
+                result.append(claims)
+        else:
+            for i in range(len(self.cluster)):
+                cluster = self.cluster[i]
+                claims = ''
+                data = cluster['info']
+                for j in range(len(data)):
+                    entry = data.iloc[j]
+                    title = entry['title']
+                    abstract = entry['abstract']
+                    prompt = f'''
+                        Title:
+                        {title}
+                        Abstract:
+                        {abstract}
+                        Task:
+                        Conclude new findings and null findings from the abstract in one sentence in the atomic format. Do not separate
+                        new findings and null findings. The finding must be relevant to the title. Do not include any other information.
+                        Definition:
+                        A scientific claim is an atomic verifiable statement expressing a finding about one aspect of a scientific entity or
+                        process, which can be verified from a single source.'''
+                    
+                    messages = [
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {"role": "user", "content": prompt},
+                    ]
+                    
+                    outputs = self.pipeline(
+                        messages,
+                        max_new_tokens=256,
+                    )
+                    claim = outputs[0]["generated_text"][-1]['content']
+                    # print(claim)
+                    # print('+++++++++++++++++++++++++++++++++')
+                    claims = claims + '\n' + claim
+                result.append(claims)
+        return result
+    
+    
+    def generate_claims_qwen(self):
+        """
+        Generate claims for each cluster using Qwen API.
+
+        Returns:
+            list: A list of strings, where each string contains the claims generated
+                for a cluster.
+        """
+        result = []
+        openai_api_key = os.getenv("OPENAI_API_KEY")
+        openai_api_base = os.getenv("OPENAI_API_BASE")
+        client = OpenAI(
+            api_key=openai_api_key,
+            base_url=openai_api_base,
+        )
+
+        for i in range(len(self.cluster)):
+            cluster = self.cluster[i]
+            claims = ''
+            data = cluster['info']
+
+            for j in range(len(data)):
+                entry = data.iloc[j]
+                title = entry['title']
+                abstract = entry['abstract']
+                
+                # Construct the prompt for Qwen
+                prompt = f'''
+                    Title:
+                    {title}
+                    Abstract:
+                    {abstract}
+                    Task:
+                    Conclude new findings and null findings from the abstract in one sentence in the atomic format. Do not separate
+                    new findings and null findings. The finding must be relevant to the title. Do not include any other information.
+                    Definition:
+                    A scientific claim is an atomic verifiable statement expressing a finding about one aspect of a scientific entity or
+                    process, which can be verified from a single source.
+                '''
+                
+                # Define the input for Qwen
+                messages = [
+                    {"role": "system", "content": "You are a helpful assistant."},
+                    {"role": "user", "content": prompt},
+                ]
+
+                try:
+                    # Call Qwen API
+                    chat_response = client.chat.completions.create(
+                        model=os.environ.get("MODEL"),
+                        max_tokens=512,
+                        temperature=0.5,
+                        messages=messages
+                    )
+                    
+                    # Extract the generated claim from Qwen's response
+                    claim = ""
+                    for chunk in chat_response:
+                        if "content" in chunk.choices[0].delta:
+                            claim += chunk.choices[0].delta.content
+                    
+                    # Clean and append the claim
+                    claims = claims + '\n' + claim.strip()
+                    # print("Generated claim:", claim)
+                    # print("+++++++++++++++++++++++++++++++++")
+                
+                except Exception as e:
+                    print(f"Error generating claim for entry {j} in cluster {i}: {e}")
+                    continue
+
+            result.append(claims)
+
+        return result
+
+    def generate_outline(self, survey_title):
+        claims = self.generate_claims()
+        cluster_with_claims = ""
+        for i in range(len(self.cluster)):
+            cluster = self.cluster[i]
+            cluster_with_claims = cluster_with_claims + f'Cluster {i}: {cluster["name"]}\n' + "Descriptions for entities in this cluster: \n" + claims[i] + '\n\n'
+        # system_prompt = f'''
+        #     You are a helpful assistant who is helping a researcher to generate an outline for a survey paper.
+        #     The references used by this survey paper have been clustered into different categories.
+        #     The researcher will provides you with the title of the survey paper
+        #     together with the cluster names and the descriptions for entities in each cluster.
+        #     '''
+        system_prompt = f'''Generate the outline of the survey paper following the format of the example : [[1, '1 Introduction'], [1, '2 Perturbations of (co)differentials'], [2, '2.1 Derivations of the tensor algebra'], [more sections...]].\
+        The first element in the sub-list refers to the hierachy of the section name (from 1 to 3). Sections like Introduction and Conclusion should have the highest level (1)\
+        The second element in the sub-list refers to the section name.
+        '''
+
+        example_json = {"title":"A Survey of Huebschmann and Stasheff's Paper: Formal Solution of the Master Equation via HPT and Deformation Theory","outline":[{"title":"1 Introduction","outline":[]},{"title":"2 Perturbations of (co)differentials","outline":[{"title":"2.1 Derivations of the tensor algebra","outline":[]},{"title":"2.2 Coderivations of the tensor coalgebra","outline":[]},{"title":"2.3 Coderivations of the symmetric coalgebra","outline":[]},{"title":"2.4 DGLA\u2019s and perturbations of the codifferential","outline":[]},{"title":"2.5 Strongly homotopy Lie algebras","outline":[]},{"title":"2.6 The Hochschild chain complex and DGA\u2019s","outline":[]},{"title":"2.7 Strongly homotopy associative algebras","outline":[]}]},{"title":"3 Master equation","outline":[]},{"title":"4 Twisting cochain","outline":[{"title":"4.1 Differential on Hom","outline":[]},{"title":"4.2 Cup product and cup bracket","outline":[]},{"title":"4.3 Twisting cochain","outline":[]}]},{"title":"5 Homological perturbation theory (HPT)","outline":[{"title":"5.1 Contraction","outline":[]},{"title":"5.2 The first main theorem.","outline":[]}]},{"title":"6 Corollaries and the second main theorem","outline":[{"title":"6.1 Other corollaries of Theorem\u00a01.","outline":[]},{"title":"6.2 The second main theorem","outline":[]}]},{"title":"7 Differential Gerstenhaber and BV algebras","outline":[{"title":"7.1 Differential Gerstenhaber algebras","outline":[]},{"title":"7.2 Differential BV algebras","outline":[]},{"title":"7.3 Formality","outline":[{"title":"7.3.1 Formality of differential graded P\ud835\udc43Pitalic_P-algebras","outline":[]},{"title":"7.3.2 Examples","outline":[]}]},{"title":"7.4 Differential BV algebras and formality","outline":[]}]},{"title":"8 Deformation theory","outline":[]},{"title":"References","outline":[]}]}
+        # user_prompt = {"survey_title":survey_title, "claims":cluster_with_claims}
+        user_prompt = f'''Generate the outline of the survey paper given the title:{survey_title}, and three lists of sentences describing each cluster of the references used by this survey:{cluster_with_claims}'''
+
+        messages = [
+            {"role": "system", "content": system_prompt}, 
+            {"role": "user", "content": user_prompt},
+            {"role": "assistant", "content":"[[1, '1 Abstract'], [1, '2 Introduction'], "}
+            ] 
+
+        outputs = self.pipeline(
+            messages,
+            max_new_tokens=9192,
+        )
+        result = outputs[0]["generated_text"][-1]['content']
+
+        self.pipeline.model.disable_adapters()
+
+        return messages, result
+
+    def generate_outline_qwen(self, survey_title, cluster_num = 3):
+        claims = self.generate_claims()
+        cluster_with_claims = ""
+        cluster_names = []
+        for i in range(cluster_num):  # 改为 cluster_num
+            cluster = self.cluster[i]
+            cluster_with_claims += f'Cluster {i}: {cluster["name"]}\nDescriptions for reference papers in this cluster:\n{claims[i]}\n\n'
+            cluster_names.append(cluster["name"])
+        # system_prompt = f'''
+        #     You are a helpful assistant who is helping a researcher to generate an outline for a survey paper.
+        #     The references used by this survey paper have been clustered into different categories.
+        #     The researcher will provides you with the title of the survey paper
+        #     together with the cluster names and the descriptions for entities in each cluster.
+        #     '''
+        system_prompt = f'''Finish the outline of the survey paper following the format of the example : [[1, '1 Introduction'], [1, '2 Perturbations of (co)differentials'], [2, '2.1 Derivations of the tensor algebra'], [3, '2.2.1 ...']......].\
+        The first element in the sub-list refers to the hierachy of the section name (from 1 to 3). Sections like Introduction and Conclusion should have the highest level (1)\
+        The second element in the sub-list refers to the section name.
+        You are required to finish the second and third level subsections name under [1, '3 <Cluster 0's name>'], [1, '4 <Cluster 1's name>'] and [1, '5 <Cluster 2's name>']
+        You must not generate third level susections over *3* for each second level subsection, for example, [3, '3.1.4 xxx'], [3, '3.1.5 xxx'] are not allowed. 
+        *Try to conclude the main findings of each cluster in the second and third level subsections, use highly abstract terms and phrases to describe*
+        *Do not include colons, e.g. AutoSurvey: Large Language Models Can Automatically Write Surveys should be written in Large Language Models in Writing Surveys*
+        '''
+        # user_prompt = {"survey_title":survey_title, "claims":cluster_with_claims}
+        cluster_sections = "\n".join([f"[1, '{i+3} {cluster_names[i]}'], [level 2 and 3 sections to finish...]" for i in range(cluster_num)])
+
+        user_prompt = f'''Finish the outline of the survey paper given the title: {survey_title}, and lists of sentences describing each cluster of the references used by this survey:\n{cluster_with_claims}
+        The first level sections' hierarchy is given: [[1, '1 Abstract'], [1, '2 Introduction'], {cluster_sections}, [1, '{cluster_num+3} Future Directions'], [1, '{cluster_num+4} Conclusion']].
+        You are required to finish the second and third level subsections under each cluster section with [2, 'a.b xxx'] and [3, 'a.b.c xxx'].
+        You must not generate third level susections over *3* for each second level subsection, for example, [3, '3.1.4 xxx'], [3, '3.1.5 xxx'] are not allowed.
+        *Try to conclude the main findings of each cluster in the second and third level subsections, use highly abstract terms and phrases to describe*
+        *Do not include colons, e.g. AutoSurvey: Large Language Models Can Automatically Write Surveys should be written in Large Language Models in Writing Surveys*
+        '''
+
+        messages = [
+            {"role": "system", "content": system_prompt}, 
+            {"role": "user", "content": user_prompt},
+            ]
+        openai_api_key = os.getenv("OPENAI_API_KEY")
+        openai_api_base = os.getenv("OPENAI_API_BASE")
+        client = OpenAI(
+            # defaults to os.environ.get("OPENAI_API_KEY")
+            api_key = openai_api_key,
+            base_url = openai_api_base,
+        )
+        chat_response = client.chat.completions.create(
+            model=os.environ.get("MODEL"),
+            max_tokens=2048,
+            temperature=0.5,
+            stop="<|im_end|>",
+            stream=True,
+            messages= messages
+        )
+        # Stream the response to console
+        text = ""
+        for chunk in chat_response:
+            if chunk.choices[0].delta.content:
+                text += chunk.choices[0].delta.content
+        # print('The response is :', text)
+        pattern = r'\[(.*)\]'
+        match = re.search(pattern, text, re.DOTALL)  # re.DOTALL 允许 . 匹配换行符
+        text = match.group(1)
+        clean_text = re.sub(r'\s+', ' ', text).strip()
+        return messages, clean_text
+
+def parseOutline(survey_id):
+    file_path = f'./src/static/data/txt/{survey_id}/outline.json'
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            data = json.load(file)
+    except Exception as e:
+        print(f"Error loading JSON file {file_path}: {e}")
+        return []
+
+    response = data.get('outline', '')
+    if not response:
+        print("No outline content found in JSON.")
+        return []
+
+    # 提取文本中第一个 '[' 与最后一个 ']' 之间的内容
+    def extract_first_last(text):
+        first_match = re.search(r'\[', text)
+        last_match = re.search(r'\](?!.*\])', text)  # 使用负向前瞻查找最后一个 ']'
+        if first_match and last_match:
+            return '[' + text[first_match.start() + 1:last_match.start()] + ']'
+        return None
+
+    response_extracted = extract_first_last(response)
+    if not response_extracted:
+        print("Failed to extract a valid list string from the outline content.")
+        return []
+
+    # 检查提取结果是否为“列表的列表”格式（应该以 "[[" 开头）
+    fixed_str = response_extracted.strip()
+    if not fixed_str.startswith("[["):
+        # 如果不是，则去掉原有的首尾括号，再重新包装：[[ ... ]]
+        # 注意：这种方式假定内部结构是以逗号分隔的多个列表，而不是单个列表。
+        fixed_str = "[[" + fixed_str[1:-1] + "]]"
+        # 或者根据你的实际情况，也可简单包装外层括号：
+        # fixed_str = "[" + fixed_str + "]"
+
+    try:
+        outline_list = ast.literal_eval(fixed_str)
+    except Exception as e:
+        print(f"Error converting extracted outline to a list.\nExtracted text: {fixed_str}\nError: {e}")
+        return []
+
+    # 如果结果不是列表，则转换成列表
+    if not isinstance(outline_list, list):
+        outline_list = list(outline_list)
+
+    # 如果解析结果不是列表的列表，而是单个列表（例如 [a, b, c]），则将其包装成一个列表
+    if outline_list and not all(isinstance(item, list) for item in outline_list):
+        outline_list = [outline_list]
+
+    result = []
+    for item in outline_list:
+        result.append(item)
+    return result
+
+
+def generateOutlineHTML_qwen(survey_id):
+    outline_list = parseOutline(survey_id)
+    html = '''
+    <div class="container-fluid w-50 d-flex flex-column justify-content-center align-items-center">
+
+        <style>
+            /* 不同层级的样式 */
+            .level-1 {
+                font-size: 20px;
+                font-weight: bold;
+                position: relative;
+                padding-right: 40px; /* 为箭头留出空间 */
+            }
+            .level-2 {
+                font-size: 18px;
+                padding-left: 40px;
+            }
+            .level-3 {
+                font-size: 16px;
+                padding-left: 80px;
+            }
+            .list-group-item {
+                border: none;
+            }
+            
+            /* 自定义卡片样式 */
+            .custom-card {
+                background-color: #fff;
+                border-radius: 8px;
+                padding: 20px;
+                margin-top: 20px;
+                width: 100%;
+                max-width: 800px;
+                box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1), 
+                            0 6px 20px rgba(0, 0, 0, 0.1);
+            }
+
+            /* 自定义卡片主体样式 */
+            .custom-card-body {
+                padding: 20px;
+            }
+
+            /* 折叠图标样式 */
+            .collapse-icon {
+                background: none;
+                border: none;
+                padding: 0;
+                position: absolute;
+                right: 10px;
+                top: 50%;
+                transform: translateY(-50%) rotate(0deg);
+                cursor: pointer;
+                font-size: 16px;
+                /* 旋转过渡效果 */
+                transition: transform 0.2s;
+            }
+            /* 去除按钮聚焦时的轮廓 */
+            .collapse-icon:focus {
+                outline: none;
+            }
+            /* 当折叠展开时旋转图标 */
+            .collapsed .collapse-icon {
+                transform: translateY(-50%) rotate(0deg);
+            }
+            .in .collapse-icon {
+                transform: translateY(-50%) rotate(90deg);
+            }
+        </style>
+
+        <div class="custom-card">
+            <div class="custom-card-body" id="display-outline">
+                <ul class="list-group list-group-flush">
+    '''
+
+    # 添加默认的一级标题内容
+    default_items = []
+
+    # 将默认项与解析出的纲要列表合并
+    combined_list = default_items + outline_list
+
+    # 构建树形结构，以便检测一级标题是否有子标题
+    def build_outline_tree(outline_list):
+        sections = []
+        stack = []
+        for level, content in outline_list:
+            level = int(level)
+            node = {'level': level, 'content': content, 'subitems': []}
+            if level == 1:
+                sections.append(node)
+                stack = [node]
+            elif level == 2:
+                if stack:
+                    parent = stack[-1]
+                    parent['subitems'].append(node)
+                    # stack.append(node)
+                else:
+                    sections.append(node)
+            elif level == 3:
+                if stack:
+                    parent = stack[-1]
+                    parent['subitems'].append(node)
+                else:
+                    sections.append(node)
+        return sections
+
+    sections = build_outline_tree(combined_list)
+
+    # 生成 HTML
+    def generate_html_from_sections(sections):
+        html = ''
+        section_index = 1  # 用于生成唯一的 ID
+
+        def generate_node_html(node):
+            nonlocal section_index
+            level = node['level']
+            content = node['content']
+            has_subitems = len(node['subitems']) > 0
+            if level == 1:
+                # 一级标题
+                if has_subitems:
+                    # 如果有子标题，添加下拉图标和可折叠功能
+                    section_id = f"outline_collapseSection{section_index}"
+                    section_index += 1
+                    node_html = f'''
+                        <li class="list-group-item level-1">
+                            {content}
+                            <a class="collapsed" data-toggle="collapse" data-target="#{section_id}" aria-expanded="true" aria-controls="{section_id}">
+                                &#9654; <!-- 右箭头表示折叠状态 -->
+                            </a>
+                            <ul class="list-group collapse in" id="{section_id}">
+                    '''
+                    for subitem in node['subitems']:
+                        node_html += generate_node_html(subitem)
+                    node_html += '''
+                            </ul>
+                        </li>
+                    '''
+                else:
+                    # 如果没有子标题，不显示下拉图标
+                    node_html = f'''
+                        <li class="list-group-item level-1">
+                            {content}
+                        </li>
+                    '''
+            elif level == 2:
+                    node_html = f'<li class="list-group-item level-2">{content}</li>'
+            elif level == 3:
+                # 三级标题直接显示，已经在二级标题中处理
+                node_html = f'<li class="list-group-item level-3">{content}</li>'
+            return node_html
+
+        for section in sections:
+            html += generate_node_html(section)
+
+        return html
+    
+    def generate_list_html(combined_list, editable=True):
+        html = '<ul class="list-group list-group-flush">\n'  # 开始 <ul>
+        for level, content in combined_list:
+            # 根据层级添加对应的 class
+            if level == 1:  # Level 1 的输入框需要禁用
+                if editable:
+                    html += f'<li class="list-group-item level-1"><input type="text" class="form-control" value="{content}" disabled></li>\n'
+                else:
+                    html += f'<li class="list-group-item level-1">{content}</li>\n'
+            elif level == 2:
+                if editable:
+                    html += f'<li class="list-group-item level-2" style="padding-left: 20px;"><input type="text" class="form-control" value="{content}"></li>\n'
+                else:
+                    html += f'<li class="list-group-item level-2" style="padding-left: 20px;">{content}</li>\n'
+            elif level == 3:
+                if editable:
+                    html += f'<li class="list-group-item level-3" style="padding-left: 40px;"><input type="text" class="form-control" value="{content}"></li>\n'
+                else:
+                    html += f'<li class="list-group-item level-3" style="padding-left: 40px;">{content}</li>\n'
+        html += '</ul>'  # 结束 </ul>
+        return html
+
+    # 生成列表 HTML
+    list_html = generate_list_html(combined_list)
+    html += generate_html_from_sections(sections)
+
+    html += f'''
+                </ul>
+            </div>
+            <div class="custom-card-body" style="display: none" id="edit-outline">
+                {list_html}
+            </div>
+            <button type="button" class="btn btn-secondary btn-lg" id="edit-btn" onclick="editOutline()"><i class="bi bi-pen"></i></button>
+            <button type="button" class="btn btn-success btn-lg" id="confirm-btn" style="display: none;" onclick="confirmOutline()"><i class="bi bi-check"></i></button>
+        </div>
+        <!-- 添加 Bootstrap v3.3.0 的 JavaScript 来处理折叠功能 -->
+        <script>
+        $(document).ready(function(){{
+            $('.collapsed').click(function(){{
+                $(this).toggleClass('collapsed');
+            }});
+        }});
+        </script>
+
+    </div>
+    '''
+
+    html+='''
+        <script>
+        // 切换到编辑模式
+        function editOutline() {
+            document.getElementById("display-outline").style.display = "none"; // 隐藏不可编���部分
+            document.getElementById("edit-outline").style.display = "block";  // 显示可编辑部分
+
+            // 显示 "Confirm" 按钮，隐藏 "Edit" 按钮
+            document.getElementById("edit-btn").style.display = "none";
+            document.getElementById("confirm-btn").style.display = "inline-block";
+        }
+
+        // 确认编辑并提交数据
+function confirmOutline() {
+    const outlineData = []; // 用于存储提交到后端的数据
+
+    // 遍历所有的可编辑输入框
+    document.querySelectorAll("#edit-outline .list-group-item").forEach((item) => {
+        const level = item.classList.contains("level-1") ? 1 :
+                      item.classList.contains("level-2") ? 2 : 3; // 获取层级
+        const content = item.querySelector("input").value.trim(); // 获取编辑框的值
+
+        // 将数据转换为数组格式 [level, content]
+        outlineData.push([level, content]); 
+    });
+
+    console.log("Submitting to backend:", outlineData); // 打印提交数据以供调试
+
+    // 使用 AJAX 提交数据到后端
+    const csrftoken = getCookie("csrftoken"); // 获取 CSRF token
+    fetch("/save_outline/", {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            "X-CSRFToken": csrftoken, // Django 的 CSRF 令牌
+        },
+        body: JSON.stringify({ outline: outlineData }) // 将数据转换为 JSON 字符串
+    })
+        .then((response) => response.json())
+        .then((data) => {
+            if (data.status === "success") {
+                $('#sections_').html(data.html);
+                alert("Outline updated successfully!");
+            } else {
+                alert("Error updating outline: " + data.message);
+            }
+        })
+        .catch((error) => {
+            console.error("Error:", error);
+            alert("Error updating outline. Please check the console for details.");
+        });
+}
+        </script>
+        '''
+    return html
+
+def insert_section(content, section_header, section_content):
+    """
+    在 content 中找到以 section_header 开头的行，并在其后插入 section_content
+    section_header: 标题名称，例如 "Abstract" 或 "Conclusion"
+    section_content: 要插入的内容（字符串）
+    """
+    # 修改正则表达式，使得数字后的点是可选的
+    pattern = re.compile(
+        r'(^#\s+\d+\.?\s+' + re.escape(section_header) + r'\s*$)',
+        re.MULTILINE | re.IGNORECASE
+    )
+    replacement = r'\1\n\n' + section_content + '\n'
+    new_content, count = pattern.subn(replacement, content)
+    if count == 0:
+        print(f"警告: 未找到标题 '{section_header}'。无法插入内容。")
+    return new_content
+
+def generateOutlineHTML(survey_id):
+    outline_list = parseOutline(survey_id)
+    html = '''
+    <div class="container-fluid w-50 d-flex flex-column justify-content-center align-items-center">
+
+        <style>
+            /* 不同层级的样式 */
+            .level-1 {
+                font-size: 20px;
+                font-weight: bold;
+                position: relative;
+                padding-right: 40px; /* 为箭头留出空间 */
+            }
+            .level-2 {
+                font-size: 18px;
+                padding-left: 40px;
+            }
+            .level-3 {
+                font-size: 16px;
+                padding-left: 80px;
+            }
+            .list-group-item {
+                border: none;
+            }
+            
+            /* 自定义卡片样式 */
+            .custom-card {
+                background-color: #fff;
+                border-radius: 8px;
+                padding: 20px;
+                margin-top: 20px;
+                width: 100%;
+                max-width: 800px;
+                box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1), 
+                            0 6px 20px rgba(0, 0, 0, 0.1);
+            }
+
+            /* 自定义卡片主体样式 */
+            .custom-card-body {
+                padding: 20px;
+            }
+
+            /* 折叠图标样式 */
+            .collapse-icon {
+                background: none;
+                border: none;
+                padding: 0;
+                position: absolute;
+                right: 10px;
+                top: 50%;
+                transform: translateY(-50%) rotate(0deg);
+                cursor: pointer;
+                font-size: 16px;
+                /* 旋转过渡效果 */
+                transition: transform 0.2s;
+            }
+            /* 去除按钮聚焦时的轮廓 */
+            .collapse-icon:focus {
+                outline: none;
+            }
+            /* 当折叠展开时旋转图标 */
+            .collapsed .collapse-icon {
+                transform: translateY(-50%) rotate(0deg);
+            }
+            .in .collapse-icon {
+                transform: translateY(-50%) rotate(90deg);
+            }
+        </style>
+
+        <div class="custom-card">
+            <div class="custom-card-body">
+                <ul class="list-group list-group-flush">
+    '''
+
+    # 添加默认的一级标题内容
+    default_items = [[1, '1 Abstract'], [1, '2 Introduction']]
+
+    # 将默认项与解析出的纲要列表合并
+    combined_list = default_items + outline_list
+
+    # 构建树形结构，以便检测一级标题是否有子标题
+    def build_outline_tree(outline_list):
+        sections = []
+        stack = []
+        for level, content in outline_list:
+            level = int(level)
+            node = {'level': level, 'content': content, 'subitems': []}
+            if level == 1:
+                sections.append(node)
+                stack = [node]
+            elif level == 2:
+                if stack:
+                    parent = stack[-1]
+                    parent['subitems'].append(node)
+                    # stack.append(node)
+                else:
+                    sections.append(node)
+            elif level == 3:
+                if stack:
+                    parent = stack[-1]
+                    parent['subitems'].append(node)
+                else:
+                    sections.append(node)
+        return sections
+
+    sections = build_outline_tree(combined_list)
+
+    # 生成 HTML
+    def generate_html_from_sections(sections):
+        html = ''
+        section_index = 1  # 用于生成唯一的 ID
+
+        def generate_node_html(node):
+            nonlocal section_index
+            level = node['level']
+            content = node['content']
+            has_subitems = len(node['subitems']) > 0
+            if level == 1:
+                # 一级标题
+                if has_subitems:
+                    # 如果有子标题，添加下拉图标和可折叠功能
+                    section_id = f"outline_collapseSection{section_index}"
+                    section_index += 1
+                    node_html = f'''
+                        <li class="list-group-item level-1">
+                            {content}
+                            <a class="collapsed" data-toggle="collapse" data-target="#{section_id}" aria-expanded="true" aria-controls="{section_id}">
+                                &#9654; <!-- 右箭头表示折叠状态 -->
+                            </a>
+                            <ul class="list-group collapse in" id="{section_id}">
+                    '''
+                    for subitem in node['subitems']:
+                        node_html += generate_node_html(subitem)
+                    node_html += '''
+                            </ul>
+                        </li>
+                    '''
+                else:
+                    # 如果没有子标题，不显示下拉图标
+                    node_html = f'''
+                        <li class="list-group-item level-1">
+                            {content}
+                        </li>
+                    '''
+            elif level == 2:
+                    node_html = f'<li class="list-group-item level-2">{content}</li>'
+            elif level == 3:
+                # 三级标题直接显示，已经在二级标题中处理
+                node_html = f'<li class="list-group-item level-3">{content}</li>'
+            return node_html
+
+        for section in sections:
+            html += generate_node_html(section)
+
+        return html
+
+    html += generate_html_from_sections(sections)
+
+    html += '''
+                </ul>
+            </div>
+        </div>
+        <!-- 添加 Bootstrap v3.3.0 的 JavaScript 来处理折叠功能 -->
+        <script>
+        $(document).ready(function(){
+            // 切换箭头方向
+            $('.collapsed').click(function(){
+                $(this).toggleClass('collapsed');
+            });
+        });
+        </script>
+    </div>
+    '''
+    return html
+
+def insert_section(content, section_header, section_content):
+    """
+    在 content 中找到以 section_header 开头的行，并在其后插入 section_content
+    section_header: 标题名称，例如 "Abstract" 或 "Conclusion"
+    section_content: 要插入的内容（字符串）
+    """
+    # 修改正则表达式，使得数字后的点是可选的
+    pattern = re.compile(
+        r'(^#\s+\d+\.?\s+' + re.escape(section_header) + r'\s*$)',
+        re.MULTILINE | re.IGNORECASE
+    )
+    replacement = r'\1\n\n' + section_content + '\n'
+    new_content, count = pattern.subn(replacement, content)
+    if count == 0:
+        print(f"警告: 未找到标题 '{section_header}'。无法插入内容。")
+    return new_content
+
+def generateSurvey(survey_id, title, collection_list, pipeline):
+    outline = parseOutline(survey_id)
+    default_items = [[1, '1 Abstract'], [1, '2 Introduction'], [1, '3 Overview']]
+    outline = str(default_items + outline)
+    
+    client = getQwenClient()
+
+    context_list = generate_context_list(outline, collection_list)
+
+    temp = {
+        "survey_id": survey_id,
+        "outline": str(default_items), 
+        "survey_title": title,
+        "context": context_list, 
+        "abstract": "",
+        "introduction": "",
+        "content": "",
+        "conclusion": "",
+        "references": ""
+    }
+
+    generated_survey_paper = generate_survey_paper_new(outline, context_list, client)
+
+    generated_introduction = generate_introduction(generated_survey_paper, client)
+    # print("\nGenerated Introduction:\n", generated_introduction)
+
+    abs_generator = AbstractGenerator(pipeline)
+    abstract = abs_generator.generate(title, generated_introduction)
+    con_generator = ConclusionGenerator(pipeline)
+    conclusion = con_generator.generate(title, generated_introduction)
+
+    abstract = abstract.replace("Abstract:", "")
+    conclusion = conclusion.replace("Conclusion:", "")
+
+    temp["abstract"] = abstract
+    temp["introduction"] = generated_introduction
+    temp["content"] = generated_survey_paper
+    temp["conclusion"] = conclusion
+
+    temp["content"] = insert_section(temp["content"], "Abstract", temp["abstract"])
+    temp["content"] = insert_section(temp["content"], "Conclusion", temp["conclusion"])
+
+    output_path = f'./src/static/data/txt/{survey_id}/generated_result.json'
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(temp, f, ensure_ascii=False, indent=4)
+    print(f"Survey has been saved to {output_path}.")
+
+    return
+
+def generate_future_directions_qwen(client, title, intro):
+    system_prompt = f'''You are a helpful assistant that help to generate the future directions of the survey paper given the survey title and survey introduction.'''
+    # user_prompt = {"survey_title":survey_title, "claims":cluster_with_claims}
+    user_prompt = f'''Help me to generate the future directions of a survey paper given the title: *{title}*, and and the introduction:{intro} within 300 words.'''
+
+    messages = [
+        {"role": "system", "content": system_prompt}, 
+        {"role": "user", "content": user_prompt},
+        {"role": "assistant", "content":"Future Directions:"}
+        ] 
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        # defaults to os.environ.get("OPENAI_API_KEY")
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    chat_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=768,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages= messages
+    )
+    # Stream the response to console
+    text = ""
+    for chunk in chat_response:
+        if chunk.choices[0].delta.content:
+            text += chunk.choices[0].delta.content
+    return text
+
+def generateSurvey_qwen(survey_id, title, collection_list, pipeline):
+    outline = str(parseOutline(survey_id))
+    
+    client = getQwenClient()
+
+    context_list = generate_context_list(outline, collection_list)
+
+    temp = {
+        "survey_id": survey_id,
+        "outline": outline, 
+        "survey_title": title,
+        "context": context_list, 
+        "abstract": "",
+        "introduction": "",
+        "content": "",
+        "future_directions":"",
+        "conclusion": "",
+        "references": ""
+    }
+
+    generated_survey_paper = generate_survey_paper_new(title, outline, context_list, client)
+    # print("Generated Survey Paper:\n", generated_survey_paper)
+
+    generated_introduction = generate_introduction(generated_survey_paper, client)
+    # print("\nGenerated Introduction:\n", generated_introduction)
+    abs_generator = AbstractGenerator(pipeline)
+    abstract = abs_generator.generate(title, generated_introduction)
+    con_generator = ConclusionGenerator(pipeline)
+    # conclusion = con_generator.generate(title, generated_introduction)
+    #New version: 12/03
+    conclusion = generate_conclusion(generated_survey_paper, client)
+    abstract = abstract.replace("Abstract:", "")
+    conclusion = conclusion.replace("Conclusion:", "")
+    # future_directions =  generate_future_directions_qwen(client, title, generated_introduction).replace("Future Directions:","")
+    #New version: 12/03
+    future_directions = generate_future_work(generated_survey_paper, client)
+    # references = generate_references_dir('./src/static/data/txt/'+survey_id)
+    temp["abstract"] = abstract
+    temp["introduction"] = generated_introduction
+    temp["content"] = generated_survey_paper
+    temp["conclusion"] = conclusion
+    temp["future_directions"] = future_directions
+    # temp["references"] = "\n\n".join([f"{ref}" for i, ref in enumerate(references)])
+    temp["content"] = insert_section(temp["content"], "Abstract", temp["abstract"])
+    temp["content"] = insert_section(temp["content"], "Conclusion", temp["conclusion"])
+    temp["content"] = insert_section(temp["content"], "Future Directions", temp["future_directions"])
+    output_path = f'./src/static/data/txt/{survey_id}/generated_result.json'
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(temp, f, ensure_ascii=False, indent=4)
+    print(f"Survey has been saved to {output_path}.")
+    return
+
+# wza
+def generateSurvey_qwen_new(survey_id, title, collection_list, pipeline, citation_data_list):
+    outline = str(parseOutline(survey_id))
+    client = getQwenClient()
+    context_list = generate_context_list(outline, collection_list)
+
+    temp = {
+        "survey_id": survey_id,
+        "outline": outline, 
+        "survey_title": title,
+        "context": context_list, 
+        "abstract": "",
+        "introduction": "",
+        "content": "",
+        "future_directions": "",
+        "conclusion": "",
+        "references": ""
+    }
+
+    # 调用generate_survey_paper_new时传入citation_data_list
+    generated_survey_paper = generate_survey_paper_new(title, outline, context_list, client, citation_data_list)
+
+    generated_introduction = generate_introduction_alternate(title, generated_survey_paper, client)
+    # generated_introduction = introduction_with_citations(generated_introduction, citation_data_list)
+    # print("\nGenerated Introduction:\n", generated_introduction)
+    # abs_generator = AbstractGenerator(pipeline)
+    # abstract = abs_generator.generate(title, generated_introduction)
+    abstract = generate_abstract(generated_survey_paper, client)
+    # con_generator = ConclusionGenerator(pipeline)
+    # conclusion = con_generator.generate(title, generated_introduction)
+    conclusion = generate_conclusion(generated_survey_paper, client)
+    abstract = abstract.replace("Abstract:", "")
+    conclusion = conclusion.replace("Conclusion:", "")
+    # future_directions =  generate_future_directions_qwen(client, title, generated_introduction).replace("Future Directions:","")
+    #New version: 12/03
+    future_directions = generate_future_work(generated_survey_paper, client)
+    # references = generate_references_dir('./src/static/data/txt/'+survey_id)
+    temp["abstract"] = abstract
+    temp["introduction"] = generated_introduction
+    temp["content"] = generated_survey_paper
+    temp["conclusion"] = conclusion
+    temp["future_directions"] = future_directions
+    # temp["references"] = "\n\n".join([f"{ref}" for i, ref in enumerate(references)])
+    temp["content"] = insert_section(temp["content"], "Abstract", temp["abstract"])
+    temp["content"] = insert_section(temp["content"], "Conclusion", temp["conclusion"])
+    temp["content"] = insert_section(temp["content"], "Future Directions", temp["future_directions"])
+    output_path = f'./src/static/data/txt/{survey_id}/generated_result.json'
+    with open(output_path, 'w', encoding='utf-8') as f:
+        json.dump(temp, f, ensure_ascii=False, indent=4)
+    print(f"Survey has been saved to {output_path}.")
+    return
+
+
+def generate_references_dir(dir):
+    client = getQwenClient()
+    papers_info = []
+    for file in os.listdir(dir):
+        if file.endswith(".json"):
+            file_path = os.path.join(dir, file)
+            with open(file_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+
+                papers_info.append({
+                    "file_path": file_path,
+                    "title": data.get("title", "Unknown Title"),
+                    "authors": data.get("authors", "Unknown Author")
+                })
+                print("The length of papers_info is: ", len(papers_info))
+    references = generate_references(papers_info, client)
+    return references
+
+if __name__ == '__main__':
+    model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+
+    context = '''
+    Many paradigms have been proposed to asses informativeness of data samples for active learning. One of the popular approaches is selecting the most uncertain data sample, i.e the data sample in which current classifier is least confident. Some other approaches are selecting the sample which yields a model with minimum risk or the data sample which yields fastest convergence in gradient based methods.//
+    An active under-sampling approach is presented in this paper to change the data distribution of training datasets, and improve the classification accuracy of minority classes while maintaining overall classification performance.//
+    In this paper, we propose an uncertainty-based active learning algorithm which requires only samples of one class and a set of unlabeled data in order to operate.//
+    The principal contribution of our work is twofold: First, we use Bayes’ rule and density estimation to avoid the need to have a model of all classes for computing the uncertainty measure.//
+    This technique reduces the number of input parameters of the problem. At the rest of this paper, we first review recent related works in the fields of active learning and active one-class learning (section II).//
+    The classifier predicts that all the samples are non-fraud, it will have a quite high accuracy. However, for problems like fraud detection, minority class classification accuracy is more critical.//
+    The algorithm used and the features selected are always the key points at design time, and many experiments are needed to select the final algorithm and the best suited feature set.//
+    Active learning works by selecting among unlabeled data, the most informative data sample. The informativeness of a sample is the amount of accuracy gain achieved after adding it to the training set.//
+    Some other approaches are selecting the sample which yields a model with minimum risk or the data sample which yields fastest convergence in gradient based methods.//
+    In this paper, we propose a novel approach reducing each within group error, BABoost, that is a variant of AdaBoost.//
+    Simulations on different unbalanced distribution data and experiments performed on several real datasets show that the new method is able to achieve a lower within group error.//
+    Active learning with early stopping can achieve a faster and scalable solution without sacrificing prediction performance.//
+    We also propose an efficient Support Vector Machine (SVM) active learning strategy which queries a small pool of data at each iterative step instead of querying the entire dataset.//
+    The second part consists of applying a treatment method and inducing a classifier for each class distribution.//
+    This time we measured the percentage of the performance loss that was recovered by the treatment method.//
+    We used two well-known over-sampling methods, random over-sampling and SMOTE.//
+    We tested our proposed technique on a sample of three representative functional genomic problems: splice site, protein subcellular localization and phosphorylation site prediction problems.//
+    Among the possible PTMs, phosphorylation is the most studied and perhaps the most important.//
+    The second part consists of applying a treatment method and inducing a classifier for each class distribution.//
+    We show that Active Learning (AL) strategy can be a more efficient alternative to resampling methods to form a balanced training set for the learner in early stages of the learning.//
+    '''
+
+    collection_list = ['activelearningfrompositiveandunlabeleddata', ]
+
+    Global_pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_id,
+        model_kwargs={"torch_dtype": torch.bfloat16},
+        token = os.getenv('HF_API_KEY'),
+        device_map="auto",
+    )
+    Global_pipeline.model.load_adapter(peft_model_id = "technicolor/llama3.1_8b_outline_generation", adapter_name="outline")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_conclusion_generation", adapter_name="conclusion")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_abstract_generation", adapter_name="abstract")
+
+    generateSurvey("test", "Predictive modeling of imbalanced data", collection_list, Global_pipeline)

src/demo/asg_query.py

@@ -0,0 +1,326 @@
+import os
+from openai import OpenAI
+from datetime import datetime, timedelta
+import re
+
+def generate_abstract_qwen(topic):
+    
+    # Initialize the OpenAI client using environment variables
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    
+    ###########################
+    # Step 1: Generate a survey abstract for the given topic.
+    ###########################
+    system_prompt_abstract = """
+You are a skilled research survey writer. Your task is to generate a survey abstract on the given topic. The abstract should cover the main challenges, key concepts, and research directions associated with the topic. Write in clear, concise academic English.
+"""
+    user_prompt_abstract = f"""
+Topic: {topic}
+
+Please generate a comprehensive survey abstract for this topic. Include discussion of core challenges, key terminologies, and emerging methodologies that are critical in the field. The total length of the abstract should be around 300–500 words.
+"""
+    messages_abstract = [
+        {"role": "system", "content": system_prompt_abstract},
+        {"role": "user", "content": user_prompt_abstract}
+    ]
+    
+    abstract_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=2048,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages_abstract
+    )
+    
+    abstract_text = ""
+    for chunk in abstract_response:
+        if chunk.choices[0].delta.content:
+            abstract_text += chunk.choices[0].delta.content
+    abstract_text = abstract_text.strip()
+    # print("The abstract is:", abstract_text)
+
+    return abstract_text
+
+def generate_entity_lists_qwen(topic, abstract_text):
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    system_prompt_abstract = f"""
+    You are an AI assistant specializing in natural language processing and entity recognition. Your task is to extract key entities and core concepts from a given abstract based on a specified topic.  
+
+    You should return two distinct lists:  
+    1. **Entity list**:  5 Entities that are synonymous or closely related to the given topic (nouns only). These should be concise (no more than two words) and simplified to their root forms (e.g., removing suffixes like "-ing", "-ed") such as llm for large language model.  
+    2. **Concept list**: Core concepts from the abstract that are highly relevant to the topic. These should also be concise (no more than two words) and in their simplest form, one single word is preferred unless the term is inseparable.  
+
+    Ensure that your response follows this exact format:
+    Entity list: [entity1, entity2, entity3, entity4, entity5]
+    Concept list: [concept1, concept2, concept3, ...concept n]
+    Do not include any explanations or additional text.  
+
+    ### **Example**  
+    #### **Input:**  
+    Topic: Large Language Models  
+    Abstract: Ever since the Turing Test was proposed in the 1950s, humans have explored the mastering of language intelligence by machine. Language is essentially a complex, intricate system of human expressions governed by grammatical rules. It poses a significant challenge to develop capable artificial intelligence (AI) algorithms for comprehending and grasping a language. As a major approach, language modeling has been widely studied for language understanding and generation in the past two decades, evolving from statistical language models to neural language models. Recently, pre-trained language models (PLMs) have been proposed by pretraining Transformer models over large-scale corpora, showing strong capabilities in solving various natural language processing (NLP) tasks. Since the researchers have found that model scaling can lead to an improved model capacity, they further investigate the scaling effect by increasing the parameter scale to an even larger size. Interestingly, when the parameter scale exceeds a certain level, these enlarged language models not only achieve a significant performance improvement, but also exhibit some special abilities (e.g., in-context learning) that are not present in small-scale language models (e.g., BERT). To discriminate the language models in different parameter scales, the research community has coined the term large language models (LLM) for the PLMs of significant size (e.g., containing tens or hundreds of billions of parameters). Recently, the research on LLMs has been largely advanced by both academia and industry, and a remarkable progress is the launch of ChatGPT (a powerful AI chatbot developed based on LLMs), which has attracted widespread attention from society. The technical evolution of LLMs has been making an important impact on the entire AI community, which would revolutionize the way how we develop and use AI algorithms. Considering this rapid technical progress, in this survey, we review the recent advances of LLMs by introducing the background, key findings, and mainstream techniques. In particular, we focus on four major aspects of LLMs, namely pre-training, adaptation tuning, utilization, and capacity evaluation. Furthermore, we also summarize the available resources for developing LLMs and discuss the remaining issues for future directions. This survey provides an up-to-date review of the literature on LLMs, which can be a useful resource for both researchers and engineers.  
+
+    #### **Expected Output:**
+    "entity list": ["language model", "plm", "large language", "llm", "llms"]  
+    "concept list": ["turing", "language intelligence", "ai", "generation", "statistical", "neural", "pre-train", "transformer", "corpora", "nlp", "in-context", "bert", "chatgpt", "adaptation", "utilization"]
+    Make sure to strictly follow this format in your response.
+    """
+
+    user_prompt_abstract = f"""
+    Topic: {topic}  
+    Abstract: {abstract_text}  
+
+    Based on the given topic and abstract, extract the following:  
+    1. A **list of 5 most key entities (nouns)** that are synonymous or closely related to the topic. Keep each entity under two words and in its simplest form.  
+    2. A **list of core concepts (terms) as many as possible** from the abstract that are highly relevant to the topic. Keep each concept under two words and in its simplest form.     
+    """
+
+    messages_abstract = [
+        {"role": "system", "content": system_prompt_abstract},
+        {"role": "user", "content": user_prompt_abstract}
+    ]
+    
+    entity_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=2048,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages_abstract
+    )
+    
+    entity_list = ""
+    for chunk in entity_response:
+        if chunk.choices[0].delta.content:
+            entity_list += chunk.choices[0].delta.content
+    entity_list = entity_list.strip()
+    # print("The entity lists are:", entity_list)
+
+    return entity_list
+
+def generate_query_qwen(topic):
+    # Calculate date range for the arXiv query (last 5 years)
+    abstract_text = generate_abstract_qwen(topic)
+    entity_list = generate_entity_lists_qwen(topic, abstract_text)
+    today = datetime.now()
+    five_years_ago = today - timedelta(days=10 * 365)  # approximate calculation
+    start_date = five_years_ago.strftime('%Y%m%d')
+    end_date = today.strftime('%Y%m%d')
+
+
+    # System prompt: Focus on how to extract keywords from the abstract.
+    system_prompt_query = """
+    You are a research assistant specializing in constructing effective arXiv search queries. Your task is to generate a structured search query using **pre-extracted entity and concept lists** from a given abstract and the topic. Follow these instructions exactly:
+
+    1. **Input Data:**
+    - **Entity List:** A list of entities that are synonymous or closely related to the given topic.
+    - **Concept List:** A list of core concepts from the abstract that are highly relevant to the topic.
+
+    2. **Ensure Minimum Keyword Count:**
+    - **Entity List** must contain at least **3** nouns of entities. If there are fewer, intelligently supplement additional relevant terms, ensuring that entities are synonyms or closely related to the key entity in the topic (e.g., "LLM" for "Large Language Model").
+    - **Concept List** must contain **12-15** domain-specific terms. If there are fewer, intelligently supplement additional relevant terms. Avoid broad terms like "combine" or "introduce."
+
+    3. **Standardize Formatting:**
+    - Convert all terms to their **base form** without adding any wildcard (`*`).
+    - All terms must be **in lowercase**.
+
+    4. **Construct the Final Query:**
+    - The query must follow this exact structure:
+        ```
+        (abs:"<Term1>" AND abs:"<Term2>") AND
+        (abs:"<Entity1>" OR abs:"<Entity2>" OR abs:"<Entity3>" OR abs:"<Entity4>" OR abs:"<Entity5>") AND 
+        (abs:"<Concept1>" OR abs:"<Concept2>" OR ... OR abs:"<Concept12>")
+        ```
+    - **Terms are 2 or 3 keywords or phrases extracted from the topic that you think **must** occur in the abstract of the searching results and are grouped together using `AND` in the first part.** (most important)
+    - **Entities are grouped together using `OR` in the second part.**
+    - **Concepts are grouped together using `OR` in the third part.**
+    - **The two groups are combined using `AND`.**
+    - **For compound words with hyphens (e.g., "in-context"), replace `-` with a space, resulting in `"in context"`.**
+    - **Do not include any explanations or extra text. Output only the final query.**
+    """
+
+    # User prompt: Provide examples of topics with corresponding query formats.
+    # User prompt: Provide examples of topics with corresponding query formats.
+    # User prompt: Uses pre-extracted entities and concepts, ensures minimum count, and applies stemming + wildcards.
+    user_prompt_query = f"""
+    Below are the pre-extracted keywords for constructing the final arXiv query.
+
+    **Topic:** {topic}  
+    **Entity list and Concept list:** {entity_list}
+
+    ### **Processing Rules Applied:**
+    - **Ensure the key terms in the topic are included**.
+    - **Ensure at least 5 entities** (if fewer, supplement additional relevant terms).
+    - **Ensure 12-15 concepts** (if fewer, supplement additional relevant terms).
+    - **Convert all terms to lowercase.**
+    - **For compound words with hyphens (e.g., "in-context"), replace `-` with a space, resulting in `"in context"`**.
+    - **Output only the final query with no extra text.**
+
+    ### **Example Query Format:**
+
+    1. **Topic:** Large Language Models in Recommendation Systems  
+    **Transformed Entity List:** ["language model", "plm", "large language", "llm", "deep model"]  
+    **Transformed Concept List:** ["tur", "language intelligence", "ai", "generation", "statistical", "neural", "pretraining", "transformer", "corpora", "nlp", "in context", "bert", "chatgpt", "adaptation", "utilization"]  
+    **Query:**  
+    (abs:"large language model" AND abs:"recommendation") AND (abs:"language model" OR abs:"plm" OR abs:"large language" OR abs:"llm" OR abs:"deep model") AND (abs:"tur" OR abs:"language intelligence" OR abs:"ai" OR abs:"generation" OR abs:"statistical" OR abs:"neural" OR abs:"pretraining" OR abs:"transformer" OR abs:"corpora" OR abs:"nlp" OR abs:"in context" OR abs:"bert" OR abs:"chatgpt" OR abs:"adaptation" OR abs:"utilization")
+
+    2. **Topic:** Quantum Computing in Physics  
+    **Transformed Entity List:** ["quantum computing", "qubit", "qc", "quantum device", "topological computing"]  
+    **Transformed Concept List:** ["decoherence", "entanglement", "error", "topology", "annealing", "photon", "superconducting", "algorithm", "optimization", "verification", "fault tolerance", "noise", "circuit", "quantum machine", "measurement"]  
+    **Query:**  
+    (abs:"quantum computing" AND abs:"physics") AND (abs:"quantum computing" OR abs:"qubit" OR abs:"qc" OR abs:"quantum device" OR abs:"topological computing") AND (abs:"decoherence" OR abs:"entanglement" OR abs:"error" OR abs:"topology" OR abs:"annealing" OR abs:"photon" OR abs:"superconducting" OR abs:"algorithm" OR abs:"optimization" OR abs:"verification" OR abs:"fault tolerance" OR abs:"noise" OR abs:"circuit" OR abs:"quantum machine" OR abs:"measurement")
+
+    ---
+
+    ### **Now Generate the Query for This Topic:**
+    **Topic:** {topic}  
+    Using the provided **Entity List** and **Concept List**, apply the following steps:
+    1. **Ensure Entity List contains at least 5 items.** If fewer, supplement additional relevant terms.
+    2. **Ensure Concept List contains 12-15 items.** If fewer, supplement additional relevant terms.
+    3. **Convert all terms to lowercase.**
+    4. **For compound words with hyphens (`-`), replace `-` with a space, e.g., `"in-context"` → `"in context"`.**
+    5. **Construct the arXiv search query in the same format as the examples above.**
+    6. **Return only the final query. Do not include explanations or additional text.**
+    All the terms in query should not exceed 2 words!
+    """
+
+    # Initialize the OpenAI API client
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    messages = [
+        {"role": "system", "content": system_prompt_query},
+        {"role": "user", "content": user_prompt_query}
+    ]
+    
+    response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=512,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages
+    )
+    
+    output_query = ""
+    for chunk in response:
+        if chunk.choices[0].delta.content:
+            output_query += chunk.choices[0].delta.content
+    match = re.search(r'\(.*\)', output_query, re.DOTALL)
+
+    if match:
+        extracted_query = match.group(0)  # 保留匹配到的整个括号内容
+    else:
+        extracted_query = output_query.strip()  # 如果匹配失败，使用原始查询
+
+    # 重新拼接 `submittedDate`
+    # updated_query = f"{extracted_query} AND submittedDate:[{start_date} TO {end_date}]"
+    updated_query = f"{extracted_query}"
+    print('The response is :', updated_query)
+    return updated_query.strip()
+
+def generate_generic_query_qwen(original_query, topic):
+    """
+    Transforms an overly strict arXiv query into a simplified, more generic version.
+    
+    The new query must be in the format:
+        (abs:"<GenericTerm1>" AND abs:"<GenericTerm2>") OR (abs:"<GenericTerm3>" AND abs:"<GenericTerm4>")
+        
+    Here, <GenericTerm1> and <GenericTerm2> represent two generic and common keywords,
+    while <GenericTerm3> and <GenericTerm4> are synonyms or closely related terms to the first two.
+    related to the given topic. If the terms in the original query are too strict,
+    replace them with broader terms that improve matching against arXiv articles.
+    
+    Parameters:
+        original_query (str): The output query from generate_query_qwen() which is too strict.
+        topic (str): The research topic.
+    
+    Returns:
+        str: The simplified arXiv query.
+    """
+    
+    system_prompt = """
+    You are a research assistant specializing in constructing effective and broad arXiv search queries.
+    Your job is to transform an overly strict query into a simplified, generic one.
+    
+    Instructions:
+    1. Input:
+       - A strict query that might be too specific.
+       - A topic which the query intends to capture.
+    
+    2. Requirements:
+       - Create a new query that only has the structure:
+         (abs:"<GenericTerm1>" AND abs:"<GenericTerm2>") OR (abs:"<GenericTerm3>" AND abs:"<GenericTerm4>")
+       - Replace <GenericTerm1> and <GenericTerm2> with two generic and common keywords for the topic.
+       - Replace <GenericTerm3> and <GenericTerm4> with the synonyms or closely related terms to the <GenericTerm1> and <GenericTerm2>.
+       - If the terms from the original query are too narrow, modify them to more broadly represent the given topic.
+       - All keywords must be in lowercase and in their base form.
+       - Each term should be one or two words.
+    
+    3. Output:
+       - Return only the final query in the exact format with no extra explanations.
+    """
+    
+    user_prompt = f"""
+    Original Query: {original_query}
+    Topic: {topic}
+    
+    The original query may be too strict and fails to match a broad range of arXiv articles.
+    Please generate a new query in the format:
+        (abs:"<GenericTerm1>" AND abs:"<GenericTerm2>") OR (abs:"<GenericTerm3>" AND abs:"<GenericTerm4>")
+    Replace <GenericTerm1> and <GenericTerm2> with more generic and commonly used terms that represent the topic.
+    Output only the final query.
+    """
+    
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    
+    # Initialize the OpenAI API client (assuming a similar interface as before)
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=512,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages
+    )
+    
+    output_query = ""
+    for chunk in response:
+        if chunk.choices[0].delta.content:
+            output_query += chunk.choices[0].delta.content
+            
+    # Use regex to extract the new simplified query in the exact required format
+    match = re.search(r'\(.*\)', output_query, re.DOTALL)
+    if match:
+        extracted_query = match.group(0)
+    else:
+        extracted_query = output_query.strip()
+    
+    print('The response is :', extracted_query)
+    return extracted_query.strip()

src/demo/asg_retriever.py

@@ -0,0 +1,364 @@
+import torch
+import uuid
+import re
+import os
+import json
+import chromadb
+from .asg_splitter import TextSplitting
+from langchain_huggingface import HuggingFaceEmbeddings
+import time
+import concurrent.futures
+
+class Retriever:
+    client = None
+    cur_dir = os.getcwd()
+    chromadb_path = os.path.join(cur_dir, "chromadb")
+
+    def __init__ (self):
+        self.client = chromadb.PersistentClient(path=self.chromadb_path)
+
+    def create_collection_chroma(self, collection_name: str):
+        """
+        The Collection will be created with collection_name, the name must follow the rules:\n
+        0. Collection name must be unique, if the name exists then try to get this collection\n
+        1. The length of the name must be between 3 and 63 characters.\n
+        2. The name must start and end with a lowercase letter or a digit, and it can contain dots, dashes, and underscores in between.\n
+        3. The name must not contain two consecutive dots.\n
+        4. The name must not be a valid IP address.\n
+        """
+        try: 
+            self.client.create_collection(name=collection_name)
+        except chromadb.db.base.UniqueConstraintError: 
+            self.get_collection_chroma(collection_name)
+        return collection_name
+
+    def get_collection_chroma (self, collection_name: str):
+        collection = self.client.get_collection(name=collection_name)
+        return collection
+
+    def add_documents_chroma (self, collection_name: str, embeddings_list: list[list[float]], documents_list: list[dict], metadata_list: list[dict]) :
+        """
+        Please make sure that embeddings_list and metadata_list are matched with documents_list\n
+        Example of one metadata: {"doc_name": "Test2.pdf", "page": "9"}\n
+        The id will be created automatically as uuid v4 
+        The chunks content and metadata will be logged (appended) into ./logs/<collection_name>.json
+        """
+        collection = self.get_collection_chroma(collection_name)
+        num = len(documents_list)
+        ids=[str(uuid.uuid4()) for i in range(num) ]
+
+        collection.add(
+            documents= documents_list,
+            metadatas= metadata_list,
+            embeddings= embeddings_list,
+            ids=ids 
+        )
+        logpath = os.path.join(self.cur_dir, "logs", f"{collection_name}.json")
+        os.makedirs(os.path.dirname(logpath), exist_ok=True)
+        logs = []
+        try:  
+            with open (logpath, 'r', encoding="utf-8") as chunklog:
+                logs = json.load(chunklog)
+        except (FileNotFoundError, json.decoder.JSONDecodeError):
+            logs = []
+       
+        added_log= [{"chunk_id": ids[i], "metadata": metadata_list[i], "page_content": documents_list[i]} \
+                       for i in range(num)]
+      
+        logs.extend(added_log)
+
+        # write back
+        with open (logpath, "w", encoding="utf-8") as chunklog:
+            json.dump(logs, chunklog, indent=4)
+        print(f"Logged document information to '{logpath}'.")
+    
+    def query_chroma(self, collection_name: str, query_embeddings: list[list[float]], n_results: int = 5) -> dict:
+        # return n closest results (chunks and metadatas) in order
+        collection = self.get_collection_chroma(collection_name)
+        result = collection.query(
+            query_embeddings=query_embeddings,
+            n_results=n_results,
+        )
+        return result
+
+    def update_chroma (self, collection_name: str, id_list: list[str], embeddings_list: list[list[float]], documents_list: list[str], metadata_list: list[dict]):
+        collection = self.get_collection_chroma(collection_name)
+        num = len(documents_list)
+        collection.update(
+            ids=id_list,
+            embeddings=embeddings_list,
+            metadatas=metadata_list,
+            documents=documents_list,
+        )
+        update_list = [{"chunk_id": id_list[i], "metadata": metadata_list[i], "page_content": documents_list[i]} for i in range(num)]
+       
+        # update the chunk log 
+        logs = []
+
+        logpath = os.path.join(self.cur_dir, "logs", f"{collection_name}.json")
+        try:  
+            with open (logpath, 'r', encoding="utf-8") as chunklog:
+                logs = json.load(chunklog)
+        except (FileNotFoundError, json.decoder.JSONDecodeError):
+            logs = [] # old_log does not exist or empty, then no need to update
+        else:
+            for i in range(num):
+                for log in logs:
+                    if (log["chunk_id"] == update_list[i]["chunk_id"]):
+                        log["metadata"] = update_list[i]["metadata"]
+                        log["page_content"] = update_list[i]["page_content"]
+                        break
+
+        with open (logpath, "w", encoding="utf-8") as chunklog:
+            json.dump(logs, chunklog, indent=4)
+        print(f"Updated log file at '{logpath}'.")
+
+    def delete_collection_entries_chroma(self, collection_name: str, id_list: list[str]):
+        collection = self.get_collection_chroma(collection_name)
+        collection.delete(ids=id_list)
+        print(f"Deleted entries with ids: {id_list} from collection '{collection_name}'.")
+
+    def delete_collection_chroma(self, collection_name: str):
+        print(f"The collection {collection_name} will be deleted forever!")    
+        self.client.delete_collection(collection_name)
+        try:
+            logpath = os.path.join(self.cur_dir, "logs", f"{collection_name}.json")
+            print(f"Collection {collection_name} has been removed, deleting log file of this collection")
+            os.remove(logpath)
+        except FileNotFoundError:
+            print("The log of this collection did not exist!")
+
+    def list_collections_chroma(self):
+        collections = self.client.list_collections()
+
+# Generate a legal collection name from a PDF filename
+def legal_pdf(filename: str) -> str:
+    pdf_index = filename.lower().rfind('.pdf')
+    if pdf_index != -1:
+        name_before_pdf = filename[:pdf_index]
+    else:
+        name_before_pdf = filename
+    name_before_pdf = name_before_pdf.strip()
+    name = re.sub(r'[^a-zA-Z0-9._-]', '', name_before_pdf)
+    name = name.lower()
+    while '..' in name:
+        name = name.replace('..', '.')
+    name = name[:63]
+    if len(name) < 3:
+        name = name.ljust(3, '0')  # fill with '0' if the length is less than 3
+    if not re.match(r'^[a-z0-9]', name):
+        name = 'a' + name[1:]
+    if not re.match(r'[a-z0-9]$', name):
+        name = name[:-1] + 'a'
+    ip_pattern = re.compile(r'^(\d{1,3}\.){3}\d{1,3}$')
+    if ip_pattern.match(name):
+        name = 'ip_' + name
+    return name
+
+def process_pdf(file_path: str, survey_id: str, embedder: HuggingFaceEmbeddings, mode: str):
+    # Load and split the PDF
+    split_start_time = time.time()
+    splitters = TextSplitting().mineru_recursive_splitter(file_path, survey_id, mode)
+
+    documents_list = [document.page_content for document in splitters]
+    for i in range(len(documents_list)):
+        documents_list[i] = documents_list[i].replace('\n', ' ')
+    print(f"Splitting took {time.time() - split_start_time} seconds.")
+
+    # Embed the documents
+    # embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    embed_start_time = time.time()
+    doc_results = embedder.embed_documents(documents_list)
+    if isinstance(doc_results, torch.Tensor):
+        embeddings_list = doc_results.tolist()
+    else:
+        embeddings_list = doc_results
+    print(f"Embedding took {time.time() - embed_start_time} seconds.")
+
+    # Prepare metadata
+    metadata_list = [{"doc_name": os.path.basename(file_path)} for i in range(len(documents_list))]
+
+    title = os.path.splitext(os.path.basename(file_path))[0]
+    
+
+    title_new = title.strip()
+    invalid_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*','_']
+    for char in invalid_chars:
+        title_new = title_new.replace(char, ' ')
+    collection_name = legal_pdf(title_new)
+
+    retriever = Retriever()
+    retriever.list_collections_chroma()
+    retriever.create_collection_chroma(collection_name)
+    retriever.add_documents_chroma(
+        collection_name=collection_name,
+        embeddings_list=embeddings_list,
+        documents_list=documents_list,
+        metadata_list=metadata_list
+    )
+
+    return collection_name, embeddings_list, documents_list, metadata_list,title_new
+
+def query_embeddings(collection_name: str, query_list: list):
+    embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    retriever = Retriever()
+
+    final_context = ""
+
+    seen_chunks = set()
+    for query_text in query_list:
+        query_embeddings = embedder.embed_query(query_text)
+        query_result = retriever.query_chroma(collection_name=collection_name, query_embeddings=[query_embeddings], n_results=2)
+
+        query_result_chunks = query_result["documents"][0]
+        # query_result_ids = query_result["ids"][0]
+
+        for chunk in query_result_chunks:
+            if chunk not in seen_chunks:
+                final_context += chunk.strip() + "//\n"
+                seen_chunks.add(chunk)
+    return final_context
+
+# new, may be in parallel
+def query_embeddings_new(collection_name: str, query_list: list):
+    embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    retriever = Retriever()
+
+    final_context = ""
+
+    seen_chunks = set()
+    def process_query(query_text):
+        query_embeddings = embedder.embed_query(query_text)
+        query_result = retriever.query_chroma(
+            collection_name=collection_name,
+            query_embeddings=[query_embeddings],
+            n_results=2
+        )
+        query_result_chunks = query_result["documents"][0]
+        return query_result_chunks
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {executor.submit(process_query, query_text): query_text for query_text in query_list}
+        for future in concurrent.futures.as_completed(futures):
+            query_result_chunks = future.result()
+            for chunk in query_result_chunks:
+                if chunk not in seen_chunks:
+                    final_context += chunk.strip() + "//\n"
+                    seen_chunks.add(chunk)
+    return final_context
+
+# wza
+def query_embeddings_new_new(collection_name: str, query_list: list):
+    embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    retriever = Retriever()
+
+    final_context = ""  # Stores concatenated context
+    citation_data_list = []  # Stores chunk content and collection name as source
+    seen_chunks = set()  # Ensures unique chunks are added
+
+    def process_query(query_text):
+        # Embed the query text and retrieve relevant chunks
+        query_embeddings = embedder.embed_query(query_text)
+        query_result = retriever.query_chroma(
+            collection_name=collection_name,
+            query_embeddings=[query_embeddings],
+            n_results=5  # Fixed number of results
+        )
+        return query_result
+
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        future_to_query = {executor.submit(process_query, q): q for q in query_list}
+        for future in concurrent.futures.as_completed(future_to_query):
+            query_text = future_to_query[future]
+            try:
+                query_result = future.result()
+            except Exception as e:
+                print(f"Query '{query_text}' failed with exception: {e}")
+                continue
+
+            if "documents" not in query_result or "distances" not in query_result:
+                continue
+            if not query_result["documents"] or not query_result["distances"]:
+                continue
+            docs_list = query_result["documents"][0] if query_result["documents"] else []
+            dist_list = query_result["distances"][0] if query_result["distances"] else []
+
+            if len(docs_list) != len(dist_list):
+                continue
+
+            for chunk, distance in zip(docs_list, dist_list):
+                processed_chunk = chunk.strip()
+                if processed_chunk not in seen_chunks:
+                    final_context += processed_chunk + "//\n"
+                    seen_chunks.add(processed_chunk)
+                    citation_data_list.append({
+                        "source": collection_name,
+                        "distance": distance,
+                        "content": processed_chunk,
+                    })
+
+    return final_context, citation_data_list
+
+# concurrent version for both collection names and queries
+def query_multiple_collections(collection_names: list[str], query_list: list[str], survey_id: str) -> dict:
+    """
+    Query multiple collections in parallel and return the combined results.
+
+    Args:
+        collection_names (list[str]): List of collection names to query.
+        query_list (list[str]): List of queries to execute on each collection.
+
+    Returns:
+        dict: Combined results from all collections, grouped by collection.
+    """
+    # Define embedder inside the function
+    embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+    retriever = Retriever()
+
+    def query_single_collection(collection_name: str):
+        """
+        Query a single collection for all queries in the query_list.
+        """
+        final_context = ""
+        seen_chunks = set()
+
+        def process_query(query_text):
+            # Embed the query
+            query_embeddings = embedder.embed_query(query_text)
+            # Query the collection
+            query_result = retriever.query_chroma(
+                collection_name=collection_name,
+                query_embeddings=[query_embeddings],
+                n_results=5
+            )
+            query_result_chunks = query_result["documents"][0]
+            return query_result_chunks
+
+        # Process all queries in parallel for the given collection
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = {executor.submit(process_query, query_text): query_text for query_text in query_list}
+            for future in concurrent.futures.as_completed(futures):
+                query_result_chunks = future.result()
+                for chunk in query_result_chunks:
+                    if chunk not in seen_chunks:
+                        final_context += chunk.strip() + "//\n"
+                        seen_chunks.add(chunk)
+
+        return final_context
+
+    # Outer parallelism for multiple collections
+    results = {}
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = {executor.submit(query_single_collection, collection_name): collection_name for collection_name in collection_names}
+        for future in concurrent.futures.as_completed(futures):
+            collection_name = futures[future]
+            results[collection_name] = future.result()
+
+    # Automatically save the results to a JSON file
+    file_path = f'./src/static/data/info/{survey_id}/retrieved_context.json'
+    with open(file_path, 'w', encoding='utf-8') as f:
+        json.dump(results, f, ensure_ascii=False, indent=4)
+
+    print(f"Results saved to {file_path}")
+
+    return results

src/demo/asg_splitter.py

@@ -0,0 +1,25 @@
+from .asg_loader import DocumentLoading
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+class TextSplitting:
+    def mineru_recursive_splitter(self, file_path, survey_id, mode):
+        docs = DocumentLoading().load_pdf(file_path, survey_id, mode)
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=400,
+            chunk_overlap=30,
+            length_function=len,
+            is_separator_regex=False,
+        )
+        texts = text_splitter.create_documents([docs])
+        return texts
+
+    def pypdf_recursive_splitter(self, file_path, survey_id):
+        docs = DocumentLoading().pypdf_loader(file_path, survey_id)
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=300,
+            chunk_overlap=20,
+            length_function=len,
+            is_separator_regex=False,
+        )
+        texts = text_splitter.create_documents([docs])
+        return texts

src/demo/category_and_tsne.py

@@ -0,0 +1,231 @@
+from sklearn.metrics import silhouette_score
+
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import matplotlib.pyplot as plt
+from sklearn.manifold import TSNE
+from sklearn.cluster import AgglomerativeClustering
+import json
+
+IMG_PATH = './src/static/img/'
+
+plt.switch_backend('agg')
+device = 0
+# tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased", model_max_length = 128)
+# model = AutoModel.from_pretrained("bert-base-uncased").to(device)
+
+from sentence_transformers import SentenceTransformer
+from umap import UMAP
+from sklearn.decomposition import PCA
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.feature_extraction.text import CountVectorizer
+from bertopic.vectorizers import ClassTfidfTransformer
+from bertopic.representation import KeyBERTInspired
+from bertopic import BERTopic
+
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.manifold import TSNE
+import seaborn as sns
+
+class DimensionalityReduction:
+    def fit(self, X):
+        return self
+
+    def transform(self, X):
+        return X
+
+class ClusteringWithTopic:
+    def __init__(self, df, n_topics=3):
+        embedding_model = SentenceTransformer("nomic-ai/nomic-embed-text-v1", trust_remote_code=True)
+        # umap_model = DimensionalityReduction()
+        umap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', init = 'pca')
+        hdbscan_model = AgglomerativeClustering(n_clusters=n_topics)
+        vectorizer_model = CountVectorizer(stop_words="english", min_df=1, ngram_range=(1, 2))
+        ctfidf_model = ClassTfidfTransformer(reduce_frequent_words=False)# True
+        keybert_model = KeyBERTInspired()
+
+        self.df = df
+        self.embeddings = embeddings = embedding_model.encode(df, show_progress_bar=True)
+
+        representation_model = {
+        "KeyBERT": keybert_model,
+        # "OpenAI": openai_model,  # Uncomment if you will use OpenAI
+        # "MMR": mmr_model,
+        # "POS": pos_model
+    }
+        self.topic_model = BERTopic(
+
+        # Pipeline models
+        embedding_model=embedding_model,
+        umap_model=umap_model,
+        hdbscan_model=hdbscan_model,
+        vectorizer_model=vectorizer_model,
+        ctfidf_model=ctfidf_model,
+        representation_model=representation_model,
+
+        # Hyperparameters
+        top_n_words=10,
+        verbose=True
+        )
+
+    def __init__(self, df, n_topics_list):
+        """
+        初始化 ClusteringWithTopic，接受一个 n_topics_list，其中包含多个聚类数目，
+        选取 silhouette_score 最高的结果。
+        """
+        embedding_model = SentenceTransformer("nomic-ai/nomic-embed-text-v1", trust_remote_code=True)
+        self.embeddings = embedding_model.encode(df, show_progress_bar=True)
+        
+        self.df = df
+        self.n_topics_list = n_topics_list
+
+        self.embedding_model = embedding_model
+        self.umap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine',init ='pca')
+        self.vectorizer_model = CountVectorizer(stop_words="english", min_df=1, ngram_range=(1, 2))
+        self.ctfidf_model = ClassTfidfTransformer(reduce_frequent_words=False)
+        self.keybert_model = KeyBERTInspired()
+        self.representation_model = {"KeyBERT": self.keybert_model}
+        
+        # 用于存储不同聚类数目的结果
+        self.best_n_topics = None
+        self.best_labels = None
+        self.best_score = -1 
+    # def fit_and_get_labels(self, X):
+    #     topics, probs = self.topic_model.fit_transform(self.df, self.embeddings)
+    #     return topics 
+    def fit_and_get_labels(self):
+        """
+        对不同的 n_topics 进行聚类，计算 silhouette_score，选取最佳的 n_topics 进行后续操作。
+        """
+        for n_topics in self.n_topics_list:
+            hdbscan_model = AgglomerativeClustering(n_clusters=n_topics)
+
+            topic_model = BERTopic(
+                embedding_model= self.embedding_model,
+                umap_model=self.umap_model,
+                hdbscan_model=hdbscan_model,
+                vectorizer_model=self.vectorizer_model,
+                ctfidf_model=self.ctfidf_model,
+                representation_model=self.representation_model,
+                top_n_words=10,
+                verbose=False
+            )
+
+            topics, _ = topic_model.fit_transform(self.df, self.embeddings)
+            
+            # 计算 silhouette_score
+            if len(set(topics)) > 1:  # silhouette_score 需要至少 2 个类别
+                score = silhouette_score(self.embeddings, topics)
+            else:
+                score = -1  # 单个类别时，silhouette_score 无意义
+
+            print(f"n_topics={n_topics}, silhouette_score={score}")
+
+            # 记录最佳的 n_topics
+            if score > self.best_score:
+                self.best_score = score
+                self.best_n_topics = n_topics
+                self.best_labels = topics
+                self.best_topic_model = topic_model
+        
+        print(f"Best n_topics={self.best_n_topics}, Best silhouette_score={self.best_score}")
+        return self.best_labels, self.best_topic_model, self.best_n_topics
+
+def clustering(df, n_cluster, survey_id):
+    text = df['retrieval_result'].astype(str)
+    clustering = ClusteringWithTopic(text, n_cluster)
+    df['label'] = clustering.fit_and_get_labels(text)
+    
+    print("The clustering result is: ")
+    for col in df.columns:
+        print(f"{col}: {df.iloc[0][col]}")
+    
+    # Save topic model information as JSON
+    topic_json = clustering.topic_model.get_topic_info().to_json()
+    with open(f'./src/static/data/info/{survey_id}/topic.json', 'w', encoding="utf-8") as file:
+        file.write(topic_json)
+    
+    # Create a dictionary from 'ref_title' and 'retrieval_result' columns
+    description_dict = dict(zip(df['ref_title'], df['retrieval_result']))
+    
+    # Save the dictionary to description.json
+    with open(f'./src/static/data/info/{survey_id}/description.json', 'w', encoding="utf-8") as file:
+        json.dump(description_dict, file, ensure_ascii=False, indent=4)
+    # df['top_n_words'] = clustering.topic_model.get_topic_info()['Representation'].tolist()
+    # df['topic_word'] = clustering.topic_model.get_topic_info()['KeyBERT'].tolist()
+
+
+    X = np.array(clustering.embeddings)
+    perplexity = 10
+    if X.shape[0] <= perplexity:
+        perplexity = max(1, X.shape[0] // 2)   
+
+    tsne = TSNE(n_components=2, init='pca', perplexity=perplexity, random_state=42)
+    X_tsne = tsne.fit_transform(X)
+    colors = scatter(X_tsne, df['label'])
+
+    plt.savefig(IMG_PATH + 'tsne_' + survey_id + '.png', dpi=800, transparent=True)
+
+    plt.close()
+    output_tsv_filename = "./src/static/data/tsv/" + survey_id + '.tsv'
+    df.to_csv(output_tsv_filename, sep='\t')
+    return df, colors
+
+def clustering(df, n_topics_list, survey_id):
+    text = df['retrieval_result'].astype(str)
+    clustering = ClusteringWithTopic(text, n_topics_list)
+    df['label'], topic_model, best_n_topics = clustering.fit_and_get_labels()
+
+    print("The clustering result is: ")
+    for col in df.columns:
+        print(f"{col}: {df.iloc[0][col]}")
+
+    # 保存 topic model 信息
+    topic_json = topic_model.get_topic_info().to_json()
+    with open(f'./src/static/data/info/{survey_id}/topic.json', 'w', encoding="utf-8") as file:
+        file.write(topic_json)
+
+    # 创建描述信息
+    description_dict = dict(zip(df['ref_title'], df['retrieval_result']))
+    with open(f'./src/static/data/info/{survey_id}/description.json', 'w', encoding="utf-8") as file:
+        json.dump(description_dict, file, ensure_ascii=False, indent=4)
+
+    # t-SNE 降维可视化
+    X = np.array(clustering.embeddings)
+    perplexity = min(10, max(1, X.shape[0] // 2))  # 避免 perplexity 过大
+
+    tsne = TSNE(n_components=2, init='pca', perplexity=perplexity, random_state=42)
+    X_tsne = tsne.fit_transform(X)
+
+    colors = scatter(X_tsne, df['label'])  # 计算颜色
+
+    plt.savefig(IMG_PATH + 'tsne_' + survey_id + '.png', dpi=800, transparent=True)
+
+    plt.close()
+    output_tsv_filename = "./src/static/data/tsv/" + survey_id + '.tsv'
+    df.to_csv(output_tsv_filename, sep='\t')
+    return df, colors, best_n_topics
+
+def scatter(x, colors):
+    sns.set_style('whitegrid')
+    sns.set_palette('Set1')
+    sns.set_context("notebook", font_scale=1.5,
+                    rc={"lines.linewidth": 2.5})
+    # We choose a color palette with seaborn.
+    palette = np.array(sns.hls_palette(8, l=0.4, s=.8))
+    color_hex = sns.color_palette(sns.hls_palette(8, l=0.4, s=.8)).as_hex()
+    # We create a scatter plot.
+    f = plt.figure(figsize=(8, 8))
+    ax = plt.subplot(aspect='equal')
+    sc = ax.scatter(x[:, 0], x[:, 1], lw=0, s=1,
+                    c=palette[colors.astype(np.int32)])
+    c = [palette[x] if x >= 0 else (0.0, 0.0, 0.0) for x in colors]
+    for i in range(x.shape[0]):
+        ax.text(x[i, 0], x[i, 1], '[' + str(i) + ']', fontsize=20, color=c[i], weight='1000')
+    plt.xlim(-25, 25)
+    plt.ylim(-25, 25)
+    ax.axis('off')
+    ax.axis('tight')
+    return color_hex[:colors.nunique()]

src/demo/count_files.py

@@ -0,0 +1,20 @@
+import os
+
+def count_files_in_folders(parent_folder):
+    folder_counts = {}
+
+    for folder in os.listdir(parent_folder):
+        folder_path = os.path.join(parent_folder, folder)
+        if os.path.isdir(folder_path):
+            num_files = len([f for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f))])
+            folder_counts[folder] = num_files
+
+    return folder_counts
+
+# 指定arxiv_downloads文件夹路径
+parent_folder = "arxiv_downloads"
+counts = count_files_in_folders(parent_folder)
+
+# 输出统计结果
+for folder, num_files in counts.items():
+    print(f"{folder}: {num_files} files")

src/demo/download.py

@@ -0,0 +1,225 @@
+import os
+import requests
+import xml.etree.ElementTree as ET
+import urllib.parse
+from tqdm import tqdm
+import time
+import re
+
+import requests
+from asg_query import generate_generic_query_qwen, generate_query_qwen
+from dotenv import load_dotenv
+
+load_dotenv()
+
+PARENT_FOLDER = "arxiv_downloads_new_new_new"
+os.makedirs(PARENT_FOLDER, exist_ok=True)
+
+def sanitize_filename(filename):
+    filename = filename.replace("\n", "").strip()
+    filename = re.sub(r'[\/:*?"<>|]', '_', filename)
+    return filename[:100] + ".pdf"
+
+def search_arxiv_papers(topic, max_results=50):
+    query_qwen = generate_query_qwen(topic)
+    encoded_query = urllib.parse.quote_plus(query_qwen)
+    url = f"https://export.arxiv.org/api/query?search_query={encoded_query}&start=0&max_results={max_results}&sortBy=submittedDate"
+
+    # base_url = "http://export.arxiv.org/api/query?"
+    # query = f"search_query=all:{topic.replace(' ', '+')}&start=0&max_results={max_results}&sortBy=relevance&sortOrder=descending"
+    # url = base_url + query
+    
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error fetching data for {topic}: {response.status_code}")
+        return []
+    
+    root = ET.fromstring(response.text)
+    entries = root.findall("{http://www.w3.org/2005/Atom}entry")
+    
+    papers = []
+    for entry in entries:
+        title = entry.find("{http://www.w3.org/2005/Atom}title").text
+        pdf_link = entry.find("{http://www.w3.org/2005/Atom}id").text.replace("abs", "pdf")
+        papers.append({"title": title, "pdf_link": pdf_link})
+    
+    return papers
+
+def download_pdf(url, folder, filename):
+    file_path = os.path.join(folder, filename)
+    
+    response = requests.get(url, stream=True)
+    if response.status_code == 200:
+        with open(file_path, 'wb') as file:
+            for chunk in response.iter_content(chunk_size=1024):
+                file.write(chunk)
+    else:
+        print(f"Failed to download {url}")
+
+def download_arxiv_papers(topic, max_results=50):
+    folder_name = os.path.join(PARENT_FOLDER, topic.replace(" ", "_"))
+    os.makedirs(folder_name, exist_ok=True)
+
+    papers = search_arxiv_papers(topic, max_results)
+
+    if not papers:
+        print(f"No papers found for topic: {topic}")
+        return
+    
+    print(f"Downloading {len(papers)} papers for topic: {topic}")
+
+    for paper in tqdm(papers, total=len(papers)):
+        filename = sanitize_filename(paper['title'])
+        pdf_link = paper["pdf_link"]
+        download_pdf(pdf_link, folder_name, filename)
+        time.sleep(2)
+        
+    print(f"Download complete. Papers saved in: {folder_name}")
+
+
+def search_arxiv_with_query(query, max_results=50):
+    """
+    Query the arXiv API with a given query string.
+    
+    Parameters:
+      query (str): The query string (URL-unencoded).
+      max_results (int): Maximum number of results to request.
+      
+    Returns:
+      list: A list of dictionaries containing paper metadata.
+    """
+    encoded_query = urllib.parse.quote_plus(query)
+    url = f"https://export.arxiv.org/api/query?search_query={encoded_query}&start=0&max_results={max_results}&sortBy=submittedDate"
+    
+    response = requests.get(url)
+    if response.status_code != 200:
+        print(f"Error fetching data with query: {query} | status code: {response.status_code}")
+        return []
+    
+    try:
+        root = ET.fromstring(response.text)
+    except Exception as e:
+        print("Error parsing XML:", e)
+        return []
+    
+    entries = root.findall("{http://www.w3.org/2005/Atom}entry")
+    papers = []
+    for entry in entries:
+        title = entry.find("{http://www.w3.org/2005/Atom}title").text.strip()
+        pdf_link = entry.find("{http://www.w3.org/2005/Atom}id").text.replace("abs", "pdf")
+        papers.append({"title": title, "pdf_link": pdf_link})
+    return papers
+
+def download_arxiv_papers_new(topic, max_results=50, min_results=10):
+    """
+    Download arXiv papers for a given topic.
+    
+    Process:
+      1. Use a strict query generated by generate_query_qwen(topic) to query arXiv.
+      2. If the number of results is fewer than `min_results`, then generate a more generic query
+         using generate_generic_query_qwen() and run additional searches.
+      3. Combine non-duplicate papers (filtered by title) until reaching max_results or exhausting attempts.
+      4. Download the PDF of each paper.
+      
+    Parameters:
+      topic (str): The research topic.
+      max_results (int): Total maximum number of papers to download (default is 50).
+      min_results (int): Minimum acceptable number of papers from the first query (default is 10).
+    """
+    folder_name = os.path.join(PARENT_FOLDER, topic.replace(" ", "_"))
+    os.makedirs(folder_name, exist_ok=True)
+    
+    # 1. Initial strict query.
+    strict_query = generate_query_qwen(topic)
+    papers = search_arxiv_with_query(strict_query, max_results=max_results)
+    
+    # Use a dict keyed by title to avoid duplicates.
+    total_papers = {paper["title"]: paper for paper in papers}
+    print(f"[Strict Query] Found {len(total_papers)} papers for topic: {topic}")
+    
+    # 2. If the strict query returns fewer than min_results papers,
+    #    use the generic query to broaden the search.
+    attempts = 0
+    MAX_ATTEMPTS = 5  # Limit attempts to avoid infinite loops.
+    while len(total_papers) < max_results and len(total_papers) < min_results and attempts < MAX_ATTEMPTS:
+        # Generate a less strict (generic) query
+        generic_query = generate_generic_query_qwen(strict_query, topic)
+        print(f"[Generic Query Attempt {attempts + 1}] Using generic query: {generic_query}")
+        generic_papers = search_arxiv_with_query(generic_query, max_results=max_results)
+        
+        new_count = 0
+        for paper in generic_papers:
+            if paper["title"] not in total_papers:
+                total_papers[paper["title"]] = paper
+                new_count += 1
+            if len(total_papers) >= max_results:
+                break
+        
+        attempts += 1
+        strict_query = generic_query  # Update the query for the next iteration.
+
+    total_paper_list = list(total_papers.values())[:max_results]
+    
+    if not total_paper_list:
+        print(f"No papers found for topic: {topic}")
+        return
+    
+    print(f"Downloading {len(total_paper_list)} papers for topic: {topic}")
+    for paper in tqdm(total_paper_list, total=len(total_paper_list)):
+        filename = sanitize_filename(paper['title'])
+        pdf_link = paper["pdf_link"]
+        download_pdf(pdf_link, folder_name, filename)
+        time.sleep(2)  # Delay to avoid overwhelming the arXiv API
+        
+    print(f"Download complete. Papers saved in: {folder_name}")
+
+first_topics = [
+    "quantum computing: bqp, quantum supremacy, and related concepts",
+    "fixed-parameter tractability and related concepts in computational complexity",
+    "fundamental concepts in computational complexity theory",
+    "pcp theorem and its implications in approximation and complexity theory",
+    "interconnections in theoretical computer science: seth, 3sum, apsp, and related concepts",
+    "nosql database systems for flexible and scalable data management",
+    "temporal databases, real-time databases, and data management systems",
+    "large language model integration with databases for enhanced data management and survey analysis",
+    "ai-driven database management",
+    "distributed systems and databases: key concepts and technologies",
+    "graph databases and query languages: traversal, indexing, and analytics",
+    "graph databases: models, data modeling, and applications",
+    "multi-model databases: mongodb, arangodb, and jsonb",
+    "time-series data management and analytics",
+    "advanced data management and retrieval techniques",
+    "vector databases and their role in modern data management and retrieval",
+    "content delivery networks: technologies and strategies for optimization",
+    "lpwan technologies: lora, zigbee 3.0, 6lowpan, and related protocols in iot",
+    "network slicing and emerging technologies in 6g networks",
+    "advanced concepts and technologies in software-defined networking and network function virtualization",
+    "battery electrolyte formulation in lithium-ion batteries",
+    "flow batteries as energy storage systems",
+    "internal consistency, self-feedback, and reliability in large language models",
+    "attention mechanisms in large language models",
+    "controlled text generation with large language models in natural language processing",
+    "domain adaptation and specialized nlp applications",
+    "evaluation of large language models for natural language processing",
+    "information extraction and large language models in natural language processing",
+    "techniques for low-resource natural language processing",
+    "model compression techniques for transformer models",
+    "multi-agent offline policy reinforcement learning: decentralized learning and cooperative policy optimization",
+    "multimodal learning and its applications",
+    "reasoning capabilities of large language models",
+    "transformer models in natural language processing"
+]
+
+second_topics = [
+    "semi-supervised learning",
+    "out-of-distribution detection",
+    "in-context learning"
+]
+
+if __name__ == '__main__':
+    for topic in first_topics:
+        print(f"\nProcessing topic (first list): {topic}")
+        download_arxiv_papers_new(topic, max_results=50, min_results=20)
+    for topic in second_topics:
+        print(f"\nProcessing topic (second list): {topic}")
+        download_arxiv_papers_new(topic, max_results=50, min_results=20)

src/demo/latex_template/acl.sty

@@ -0,0 +1,312 @@
+% This is the LaTex style file for *ACL.
+% The official sources can be found at
+%
+%     https://github.com/acl-org/acl-style-files/
+%
+% This package is activated by adding
+%
+%    \usepackage{acl}
+%
+% to your LaTeX file. When submitting your paper for review, add the "review" option:
+%
+%    \usepackage[review]{acl}
+
+\newif\ifacl@finalcopy
+\newif\ifacl@anonymize
+\newif\ifacl@linenumbers
+\newif\ifacl@pagenumbers
+\DeclareOption{final}{\acl@finalcopytrue\acl@anonymizefalse\acl@linenumbersfalse\acl@pagenumbersfalse}
+\DeclareOption{review}{\acl@finalcopyfalse\acl@anonymizetrue\acl@linenumberstrue\acl@pagenumberstrue}
+\DeclareOption{preprint}{\acl@finalcopytrue\acl@anonymizefalse\acl@linenumbersfalse\acl@pagenumberstrue}
+\ExecuteOptions{final} % final copy is the default
+
+% include hyperref, unless user specifies nohyperref option like this:
+% \usepackage[nohyperref]{acl}
+\newif\ifacl@hyperref
+\DeclareOption{hyperref}{\acl@hyperreftrue}
+\DeclareOption{nohyperref}{\acl@hyperreffalse}
+\ExecuteOptions{hyperref} % default is to use hyperref
+\ProcessOptions\relax
+
+\typeout{Conference Style for ACL}
+
+\usepackage{xcolor}
+
+\ifacl@linenumbers
+  % Add draft line numbering via the lineno package
+  % https://texblog.org/2012/02/08/adding-line-numbers-to-documents/
+  \usepackage[switch,mathlines]{lineno}
+
+  % Line numbers in gray Helvetica 8pt
+  \font\aclhv = phvb at 8pt
+  \renewcommand\linenumberfont{\aclhv\color{lightgray}}
+
+  % Zero-fill line numbers
+  % NUMBER with left flushed zeros  \fillzeros[<WIDTH>]<NUMBER>
+  \newcount\cv@tmpc@ \newcount\cv@tmpc
+  \def\fillzeros[#1]#2{\cv@tmpc@=#2\relax\ifnum\cv@tmpc@<0\cv@tmpc@=-\cv@tmpc@\fi
+    \cv@tmpc=1 %
+    \loop\ifnum\cv@tmpc@<10 \else \divide\cv@tmpc@ by 10 \advance\cv@tmpc by 1 \fi
+      \ifnum\cv@tmpc@=10\relax\cv@tmpc@=11\relax\fi \ifnum\cv@tmpc@>10 \repeat
+    \ifnum#2<0\advance\cv@tmpc1\relax-\fi
+    \loop\ifnum\cv@tmpc<#1\relax0\advance\cv@tmpc1\relax\fi \ifnum\cv@tmpc<#1 \repeat
+    \cv@tmpc@=#2\relax\ifnum\cv@tmpc@<0\cv@tmpc@=-\cv@tmpc@\fi \relax\the\cv@tmpc@}%
+  \renewcommand\thelinenumber{\fillzeros[3]{\arabic{linenumber}}}
+  \linenumbers
+
+  \setlength{\linenumbersep}{1.6cm}
+
+  % Bug: An equation with $$ ... $$ isn't numbered, nor is the previous line.
+
+  % Patch amsmath commands so that the previous line and the equation itself
+  % are numbered. Bug: multline has an extra line number.
+  % https://tex.stackexchange.com/questions/461186/how-to-use-lineno-with-amsmath-align
+  \usepackage{etoolbox} %% <- for \pretocmd, \apptocmd and \patchcmd
+
+  \newcommand*\linenomathpatch[1]{%
+    \expandafter\pretocmd\csname #1\endcsname {\linenomath}{}{}%
+    \expandafter\pretocmd\csname #1*\endcsname {\linenomath}{}{}%
+    \expandafter\apptocmd\csname end#1\endcsname {\endlinenomath}{}{}%
+    \expandafter\apptocmd\csname end#1*\endcsname {\endlinenomath}{}{}%
+  }
+  \newcommand*\linenomathpatchAMS[1]{%
+    \expandafter\pretocmd\csname #1\endcsname {\linenomathAMS}{}{}%
+    \expandafter\pretocmd\csname #1*\endcsname {\linenomathAMS}{}{}%
+    \expandafter\apptocmd\csname end#1\endcsname {\endlinenomath}{}{}%
+    \expandafter\apptocmd\csname end#1*\endcsname {\endlinenomath}{}{}%
+  }
+
+  %% Definition of \linenomathAMS depends on whether the mathlines option is provided
+  \expandafter\ifx\linenomath\linenomathWithnumbers
+    \let\linenomathAMS\linenomathWithnumbers
+    %% The following line gets rid of an extra line numbers at the bottom:
+    \patchcmd\linenomathAMS{\advance\postdisplaypenalty\linenopenalty}{}{}{}
+  \else
+    \let\linenomathAMS\linenomathNonumbers
+  \fi
+
+  \AtBeginDocument{%
+    \linenomathpatch{equation}%
+    \linenomathpatchAMS{gather}%
+    \linenomathpatchAMS{multline}%
+    \linenomathpatchAMS{align}%
+    \linenomathpatchAMS{alignat}%
+    \linenomathpatchAMS{flalign}%
+  }
+\else
+  % Hack to ignore these commands, which review mode puts into the .aux file.
+  \newcommand{\@LN@col}[1]{}
+  \newcommand{\@LN}[2]{}
+  \newcommand{\nolinenumbers}{}
+\fi
+
+\PassOptionsToPackage{a4paper,margin=2.5cm,heightrounded=true}{geometry}
+\RequirePackage{geometry}
+
+\setlength\columnsep{0.6cm}
+\newlength\titlebox
+\setlength\titlebox{11\baselineskip}
+% \titlebox should be a multiple of \baselineskip so that
+% column height remaining fits an exact number of lines of text
+
+\flushbottom \twocolumn \sloppy
+
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip@sandia-2
+\def\addcontentsline#1#2#3{}
+
+\ifacl@pagenumbers
+    \pagenumbering{arabic}
+\else
+    \thispagestyle{empty}
+    \pagestyle{empty}
+\fi
+
+%% Title and Authors %%
+
+\let\Thanks\thanks % \Thanks and \thanks used to be different, but keep this for backwards compatibility.
+
+\newcommand\outauthor{%
+    \begin{tabular}[t]{c}
+    \ifacl@anonymize
+        \bfseries Anonymous ACL submission
+    \else
+        \bfseries\@author
+    \fi
+    \end{tabular}}
+
+% Mostly taken from deproc.
+\AtBeginDocument{
+\def\maketitle{\par
+ \begingroup
+   \def\thefootnote{\fnsymbol{footnote}}
+   \twocolumn[\@maketitle]
+   \@thanks
+ \endgroup
+ \setcounter{footnote}{0}
+ \let\maketitle\relax
+ \let\@maketitle\relax
+ \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
+\def\@maketitle{\vbox to \titlebox{\hsize\textwidth
+ \linewidth\hsize \vskip 0.125in minus 0.125in \centering
+ {\Large\bfseries \@title \par} \vskip 0.2in plus 1fil minus 0.1in
+ {\def\and{\unskip\enspace{\rmfamily and}\enspace}%
+  \def\And{\end{tabular}\hss \egroup \hskip 1in plus 2fil
+           \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bfseries}%
+  \def\AND{\end{tabular}\hss\egroup \hfil\hfil\egroup
+          \vskip 0.25in plus 1fil minus 0.125in
+           \hbox to \linewidth\bgroup\large \hfil\hfil
+             \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bfseries}
+  \hbox to \linewidth\bgroup\large \hfil\hfil
+    \hbox to 0pt\bgroup\hss
+  \outauthor
+   \hss\egroup
+    \hfil\hfil\egroup}
+  \vskip 0.3in plus 2fil minus 0.1in
+}}
+}
+
+% margins and font size for abstract
+\renewenvironment{abstract}%
+  {\begin{center}\large\textbf{\abstractname}\end{center}%
+    \begin{list}{}%
+      {\setlength{\rightmargin}{0.6cm}%
+        \setlength{\leftmargin}{0.6cm}}%
+      \item[]\ignorespaces%
+      \@setsize\normalsize{12pt}\xpt\@xpt
+  }%
+  {\unskip\end{list}}
+
+% Resizing figure and table captions - SL
+% Support for interacting with the caption, subfigure, and subcaption packages - SL
+\RequirePackage{caption}
+\DeclareCaptionFont{10pt}{\fontsize{10pt}{12pt}\selectfont}
+\captionsetup{font=10pt}
+
+\RequirePackage{natbib}
+% for citation commands in the .tex, authors can use:
+% \citep, \citet, and \citeyearpar for compatibility with natbib, or
+% \cite, \newcite, and \shortcite for compatibility with older ACL .sty files
+\renewcommand\cite{\citep}  % to get "(Author Year)" with natbib
+\newcommand\shortcite{\citeyearpar}% to get "(Year)" with natbib
+\newcommand\newcite{\citet} % to get "Author (Year)" with natbib
+\newcommand{\citeposs}[1]{\citeauthor{#1}'s (\citeyear{#1})} % to get "Author's (Year)"
+
+\bibliographystyle{acl_natbib}
+
+% Bibliography
+
+% Don't put a label in the bibliography at all.  Just use the unlabeled format
+% instead.
+\def\thebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{References\@mkboth
+ {References}{References}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthebibliography=\endlist
+
+
+% Allow for a bibliography of sources of attested examples
+\def\thesourcebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{Sources of Attested Examples\@mkboth
+ {Sources of Attested Examples}{Sources of Attested Examples}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthesourcebibliography=\endlist
+
+% sections with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+    -0.5ex minus -.2ex}{1.5ex plus 0.3ex minus .2ex}{\large\bfseries\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-1.8ex plus
+    -0.5ex minus -.2ex}{0.8ex plus .2ex}{\normalsize\bfseries\raggedright}}
+%% changed by KO to - values to get the initial parindent right
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{-1.5ex plus
+   -0.5ex minus -.2ex}{0.5ex plus .2ex}{\normalsize\bfseries\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+   0.5ex minus .2ex}{-1em}{\normalsize\bfseries}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\parindent}{1.5ex plus
+   0.5ex minus .2ex}{-1em}{\normalsize\bfseries}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 1em
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em \leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+   \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+   \topsep 2pt plus 1pt minus 0.5pt
+   \parsep 1pt plus 0.5pt minus 0.5pt
+   \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+    \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+    \topsep 1pt plus 0.5pt minus 0.5pt
+    \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+    \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+     \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+     \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+     \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip  0pt plus3pt%
+\belowdisplayshortskip  4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
+
+% The hyperref manual (section 9) says hyperref should be loaded after natbib
+\ifacl@hyperref
+  \PassOptionsToPackage{breaklinks}{hyperref}
+  \RequirePackage{hyperref}
+  % make links dark blue
+  \definecolor{darkblue}{rgb}{0, 0, 0.5}
+  \hypersetup{colorlinks=true, citecolor=darkblue, linkcolor=darkblue, urlcolor=darkblue}
+\else
+  % This definition is used if the hyperref package is not loaded.
+  % It provides a backup, no-op definiton of \href.
+  % This is necessary because \href command is used in the acl_natbib.bst file.
+  \def\href#1#2{{#2}}
+  \usepackage{url}
+\fi

src/demo/latex_template/template.tex

@@ -0,0 +1,22 @@
+\documentclass[11pt]{article}
+
+\usepackage[final]{acl}
+\usepackage{times}
+\usepackage{latexsym}
+\usepackage[T1]{fontenc}
+\usepackage{microtype}
+\usepackage{graphicx}
+\usepackage{amsmath}  
+
+\author{
+  InteractiveSurvey \\
+  Affiliation Line 1 \\
+  Affiliation Line 2 \\
+  \texttt{InteractiveSurvey@domain} \\
+}
+
+\begin{document}
+\maketitle
+
+
+\end{document}

src/demo/main.py

@@ -0,0 +1,448 @@
+import csv
+import json
+import os
+
+import pandas as pd
+from langchain_huggingface import HuggingFaceEmbeddings
+from asg_retriever import legal_pdf
+from asg_loader import DocumentLoading
+from asg_retriever import Retriever, query_embeddings_new_new
+from asg_generator import generate_sentence_patterns, generate
+from category_and_tsne import clustering
+from langchain_text_splitters  import RecursiveCharacterTextSplitter
+import time
+import torch
+import re
+import transformers
+from dotenv import load_dotenv
+from asg_clustername import generate_cluster_name_new
+from asg_outline import OutlineGenerator, generateSurvey_qwen_new
+import os
+from markdown_pdf import MarkdownPdf, Section  # Assuming you are using markdown_pdf
+from typing import Any
+
+def clean_str(input_str):
+    input_str = str(input_str).strip().lower()
+    if input_str == "none" or input_str == "nan" or len(input_str) == 0:
+        return ""
+    input_str = input_str.replace('\\n',' ').replace('\n',' ').replace('\r',' ').replace('——',' ').replace('——',' ').replace('__',' ').replace('__',' ').replace('........','.').replace('....','.').replace('....','.').replace('..','.').replace('..','.').replace('..','.').replace('. . . . . . . . ','. ').replace('. . . . ','. ').replace('. . . . ','. ').replace('. . ','. ').replace('. . ','. ')
+    input_str = re.sub(r'\\u[0-9a-z]{4}', ' ', input_str).replace('  ',' ').replace('  ',' ')
+    return input_str
+
+def remove_invalid_citations(text, valid_collection_names):
+    """
+    只保留 [xxx\] 中的 xxx 属于 valid_collection_names 的引用，
+    其余的引用标记一律删除。
+    """
+    pattern = r"\[(.*?)\\\]"  # 匹配形如 [xxx\] 的内容
+    all_matches = re.findall(pattern, text)
+
+    new_text = text
+    for match in all_matches:
+        cleaned_match = match.rstrip('\\')  # 去除末尾的 \
+        if cleaned_match not in valid_collection_names:
+            new_text = new_text.replace(f"[{match}\\]", "")
+    return new_text
+def normalize_citations_with_mapping(paper_text):
+    # 使用正则表达式匹配所有引用标记（形如 [citation1]）
+    citations = re.findall(r'\[.*?\]', paper_text)
+    # 去重并保持顺序
+    unique_citations = list(dict.fromkeys(citations))
+    # 生成引用映射表，把原始引用标记映射为数字引用
+    citation_mapping = {citation: f'[{i + 1}]' for i, citation in enumerate(unique_citations)}
+
+    # 在文本中替换老引用为新引用
+    normalized_text = paper_text
+    for old_citation, new_citation in citation_mapping.items():
+        normalized_text = normalized_text.replace(old_citation, new_citation)
+
+    # 生成从数字到原始引用标记的反向映射
+    # 用 rstrip('\\') 去掉末尾的反斜杠
+    reverse_mapping = {
+        i + 1: unique_citations[i].strip('[]').rstrip('\\')
+        for i in range(len(unique_citations))
+    }
+
+    return normalized_text, reverse_mapping
+def generate_references_section(citation_mapping, collection_pdf_mapping):
+    
+    references = ["# References"]  # 生成引用部分
+    for num in sorted(citation_mapping.keys()):
+        collection_name = citation_mapping[num]
+        pdf_name = collection_pdf_mapping.get(collection_name, "Unknown PDF")
+        if pdf_name.endswith(".pdf"):
+            pdf_name = pdf_name[:-4]
+        # 在每一行末尾添加两个空格以确保换行
+        references.append(f"[{num}] {pdf_name}  ")
+
+    return "\n".join(references)
+def fix_citation_punctuation_md(text):
+    """
+    把类似于 'some text. \[1]' 或 'some text. \[2]' 调整为 'some text \[1].'
+    仅针对已经变成 \[1], \[2] 之类数字引用的 Markdown 情况有效。
+    如果还没有变成 \[数字]，则需先经过 normalize_citations_with_mapping。
+    """
+    # 正则表达式匹配点号后带有空格或无空格，紧接 \[数字] 的情况
+    pattern = r'\.\s*(\\\[\d+\])'
+    replacement = r' \1.'
+    fixed_text = re.sub(pattern, replacement, text)
+    return fixed_text
+def finalize_survey_paper(paper_text, 
+                          Global_collection_names, 
+                          Global_file_names):
+
+    # 1) 删除所有不想要的旧引用（包括 [数字]、[Sewon, 2021] 等）
+    paper_text = remove_invalid_citations(paper_text, Global_collection_names)
+
+    # 2) 规范化引用 => [1][2]...
+    normalized_text, citation_mapping = normalize_citations_with_mapping(paper_text)
+    
+    # 3) 修复标点，比如 .[1] => [1].
+    normalized_text = fix_citation_punctuation_md(normalized_text)
+
+    # 4) 构造 {collection_name: pdf_file_name} 字典
+    collection_pdf_mapping = dict(zip(Global_collection_names, Global_file_names))
+    
+    # 5) 生成 References
+    references_section = generate_references_section(citation_mapping, collection_pdf_mapping)
+    
+    # 6) 合并正文和 References
+    final_paper = normalized_text.strip() + "\n\n" + references_section
+    return final_paper
+
+class ASG_system:
+    def __init__(self, root_path: str, survey_id:str, pdf_path: str, survey_title: str, cluster_standard: str) -> None:
+        load_dotenv()
+        self.pdf_path = pdf_path
+        self.txt_path = root_path + "/txt"
+        self.tsv_path = root_path + "/tsv"
+        self.md_path = root_path + "/md"
+        self.info_path = root_path + "/info"
+        self.result_path = root_path + "/result"
+
+        self.survey_id = survey_id
+        self.survey_title = survey_title
+        self.cluster_standard = cluster_standard
+
+        self.collection_names = []
+        self.file_names = []
+        self.citation_data = []
+        self.description_list = []
+        self.ref_list = []
+        self.cluster_names = []
+        self.collection_names_clustered = []
+        self.df_selected = ''
+
+
+        model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+        self.embedder = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+        self.pipeline = transformers.pipeline(
+            "text-generation",
+            model=model_id,
+            model_kwargs={"torch_dtype": torch.bfloat16},
+            token = os.getenv('HF_API_KEY'),
+            device_map="auto",
+        )
+        self.pipeline.model.load_adapter(peft_model_id = "technicolor/llama3.1_8b_outline_generation", adapter_name="outline")
+        self.pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_abstract_generation", adapter_name="abstract")
+        self.pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_conclusion_generation", adapter_name="conclusion")
+
+        os.makedirs(self.txt_path, exist_ok=True)
+        os.makedirs(f'{self.txt_path}/{self.survey_id}', exist_ok=True)
+
+        os.makedirs(self.tsv_path, exist_ok=True)
+
+        os.makedirs(self.md_path, exist_ok=True)
+        os.makedirs(f'{self.md_path}/{self.survey_id}', exist_ok=True)
+
+        os.makedirs(self.info_path, exist_ok=True)
+        os.makedirs(f'{self.info_path}/{self.survey_id}', exist_ok=True)
+
+        os.makedirs(self.result_path, exist_ok=True)
+        os.makedirs(f'{self.result_path}/{self.survey_id}', exist_ok=True)
+
+    def parsing_pdfs(self, mode="intro") -> None:
+        pdf_files = os.listdir(self.pdf_path)
+        loader = DocumentLoading()
+        
+
+        for pdf_file in pdf_files:
+
+            pdf_file = os.path.join(self.pdf_path, pdf_file)
+
+            split_start_time = time.time()
+
+            base_name = os.path.splitext(os.path.basename(pdf_file))[0]
+            target_dir = os.path.join(self.md_path, self.survey_id, base_name, "auto")
+            md_dir = os.path.join(self.md_path, self.survey_id)
+
+            loader.convert_pdf_to_md(pdf_file, md_dir)
+
+            md_file_path = os.path.join(target_dir, f"{base_name}.md")
+            print(md_file_path)
+            print("*"*24)
+            if not os.path.exists(md_file_path):
+                raise FileNotFoundError(f"Markdown file {md_file_path} does not exist. Conversion might have failed.")
+
+            if mode == "intro":
+                doc = loader.process_md_file(md_file_path, self.survey_id, self.txt_path)
+            elif mode == "full":
+                doc = loader.process_md_file_full(md_file_path, self.survey_id,self.txt_path)
+            
+            text_splitter = RecursiveCharacterTextSplitter(
+                chunk_size=400,
+                chunk_overlap=30,
+                length_function=len,
+                is_separator_regex=False,
+            )
+            splitters = text_splitter.create_documents([doc])
+            documents_list = [document.page_content for document in splitters]
+            for i in range(len(documents_list)):
+                documents_list[i] = documents_list[i].replace('\n', ' ')
+            print(f"Splitting took {time.time() - split_start_time} seconds.")
+
+            embed_start_time = time.time()
+
+            doc_results = self.embedder.embed_documents(documents_list)
+            if isinstance(doc_results, torch.Tensor):
+                embeddings_list = doc_results.tolist()
+            else:
+                embeddings_list = doc_results
+            print(f"Embedding took {time.time() - embed_start_time} seconds.")
+
+            # Prepare metadata
+            metadata_list = [{"doc_name": os.path.basename(pdf_file)} for i in range(len(documents_list))]
+
+            title = os.path.splitext(os.path.basename(pdf_file))[0]
+            
+
+            title_new = title.strip()
+            invalid_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*','_']
+            for char in invalid_chars:
+                title_new = title_new.replace(char, ' ')
+            print("============================")
+            print(title_new)
+
+            # New logic to create collection_name
+            # filename = os.path.basename(file_path)
+            collection_name = legal_pdf(title_new)
+
+            retriever = Retriever()
+            retriever.list_collections_chroma()
+            retriever.create_collection_chroma(collection_name)
+            retriever.add_documents_chroma(
+                collection_name=collection_name,
+                embeddings_list=embeddings_list,
+                documents_list=documents_list,
+                metadata_list=metadata_list
+            )
+
+            self.collection_names.append(collection_name)
+            self.file_names.append(title_new)
+        print(self.collection_names)
+        print(self.file_names)
+
+        json_files = os.listdir(os.path.join(self.txt_path, self.survey_id))
+        ref_paper_num = len(json_files)
+        print(f'The length of the json files is {ref_paper_num}')
+
+
+        json_data_pd = pd.DataFrame()   
+        for _ in json_files:
+            file_path = os.path.join(self.txt_path, self.survey_id, _)
+
+            with open(file_path, 'r', encoding="utf-8") as file:
+                data = json.load(file)
+
+                # Extract necessary information
+                title = data.get("title", "")
+                abstract = data.get("abstract", "")
+                authors = data.get("authors", "")
+                introduction = data.get("introduction", "")
+                new_data = {
+                    "reference paper title": title,
+                    "reference paper citation information (can be collected from Google scholar/DBLP)": authors,
+                    "reference paper abstract (Please copy the text AND paste here)": abstract,
+                    "reference paper introduction (Please copy the text AND paste here)": introduction,
+                    "reference paper doi link (optional)": "",
+                    "reference paper category label (optional)": ""
+                }
+
+                # 将新数据转换为 DataFrame
+                new_data_df = pd.DataFrame([new_data])
+
+                # 使用 pd.concat 而不是 append
+                json_data_pd = pd.concat([json_data_pd, new_data_df], ignore_index=True)
+
+        # Save the DataFrame to a variable for further use
+        input_pd = json_data_pd
+
+        if ref_paper_num>0:
+                
+            ## change col name
+            input_pd['ref_title'] = [filename for filename in self.file_names]
+            input_pd["ref_context"] = [""]*ref_paper_num
+            input_pd["ref_entry"] = input_pd["reference paper citation information (can be collected from Google scholar/DBLP)"]
+            input_pd["abstract"] = input_pd["reference paper abstract (Please copy the text AND paste here)"].apply(lambda x: clean_str(x) if len(str(x))>0 else 'Invalid abstract')
+            input_pd["intro"] = input_pd["reference paper introduction (Please copy the text AND paste here)"].apply(lambda x: clean_str(x) if len(str(x))>0 else 'Invalid introduction')
+
+            # optional columns
+            input_pd["label"] = input_pd["reference paper category label (optional)"].apply(lambda x: str(x) if len(str(x))>0 else '')
+            #input_pd["label"] = input_pd["reference paper category id (optional)"].apply(lambda x: str(x) if len(str(x))>0 else '')
+            ## output tsv
+            # output_tsv_filename = self.tsv_path + self.survey_id + '.tsv'
+            output_tsv_filename = os.path.join(self.tsv_path, self.survey_id + '.tsv') 
+
+            #output_df = input_pd[["ref_title","ref_context","ref_entry","abstract","intro","description"]]
+            output_df = input_pd[["ref_title","ref_context","ref_entry","abstract","intro", 'label']]
+            # print(output_df)
+
+            #pdb.set_trace()
+            output_df.to_csv(output_tsv_filename, sep='\t')
+
+    def description_generation(self) -> None:
+        query= self.cluster_standard
+        query_list = generate_sentence_patterns(query)
+        for name in self.collection_names:
+            context, citation_data = query_embeddings_new_new(name, query_list)
+            self.citation_data.extend(citation_data)
+
+            description = generate(context, query, name)
+            self.description_list.append(description)
+
+        citation_path = f'{self.info_path}/{self.survey_id}/citation_data.json'
+        os.makedirs(os.path.dirname(citation_path), exist_ok=True)
+        with open(citation_path, 'w', encoding="utf-8") as outfile:
+            json.dump(self.citation_data, outfile, indent=4, ensure_ascii=False)
+        
+        file_path = f'{self.tsv_path}/{self.survey_id}.tsv'
+
+        with open(file_path, 'r', newline='', encoding='utf-8') as infile:
+            reader = csv.reader(infile, delimiter='\t')
+            rows = list(reader)
+        if rows:
+            headers = rows[0]
+            headers.append('retrieval_result')
+
+            updated_rows = [headers]
+            for row, description in zip(rows[1:], self.description_list):
+                row.append(description)
+                updated_rows.append(row)
+
+            with open(file_path, 'w', newline='', encoding='utf-8') as outfile:
+                writer = csv.writer(outfile, delimiter='\t')
+                writer.writerows(updated_rows)
+
+            print('Updated file has been saved to', file_path)
+        else:
+            print('Input file is empty.')
+
+    def agglomerative_clustering(self) -> None:
+        df = pd.read_csv(f'{self.tsv_path}/{self.survey_id}.tsv', sep='\t', index_col=0, encoding='utf-8')
+        df_selected = df
+
+        df_selected, _ = clustering(df_selected, 3, self.survey_id, self.info_path, self.tsv_path)
+        self.df_selected = df_selected
+
+        df_tmp = df_selected.reset_index()
+        df_tmp['index'] = df_tmp.index
+        ref_titles = list(df_tmp.groupby(df_tmp['label'])['ref_title'].apply(list))
+        # ref_indexs = list(df_tmp.groupby(df_tmp['label'])['index'].apply(list))
+
+        category_label_summarized = generate_cluster_name_new(f"{self.tsv_path}/{self.survey_id}.tsv", self.survey_title)
+        self.cluster_names = category_label_summarized
+
+        cluster_info = {category_label_summarized[i]:ref_titles[i] for i in range(len(category_label_summarized))}
+        for key, value in cluster_info.items():
+            temp = [legal_pdf(i) for i in value]
+            cluster_info[key] = temp
+            self.collection_names_clustered.append(temp)
+        cluster_info_path = f'{self.info_path}/{self.survey_id}/cluster_info.json'
+        with open(cluster_info_path, 'w', encoding="utf-8") as outfile:
+            json.dump(cluster_info, outfile, indent=4, ensure_ascii=False)        
+
+    def outline_generation(self) -> None:
+        print(self.df_selected)
+        print(self.cluster_names)
+        outline_generator = OutlineGenerator(self.pipeline, self.df_selected, self.cluster_names)
+        outline_generator.get_cluster_info()
+        messages, outline = outline_generator.generate_outline_qwen(self.survey_title)
+        outline_json = {'messages':messages, 'outline': outline}
+        output_path = f'{self.info_path}/{self.survey_id}/outline.json'
+        os.makedirs(os.path.dirname(output_path), exist_ok=True)
+        with open(output_path, 'w', encoding="utf-8") as outfile:
+            json.dump(outline_json, outfile, indent=4, ensure_ascii=False)
+        
+
+    def section_generation(self) -> None:
+        generateSurvey_qwen_new(self.survey_id, self.survey_title, self.collection_names_clustered, self.pipeline, self.citation_data, './txt','./info')
+
+    def citation_generation(self) -> None:
+        """
+        Generate citation Markdown and PDF files from JSON and store them in the specified result path.
+        """
+
+        json_filepath = os.path.join(self.info_path, self.survey_id, "generated_result.json")
+
+        markdown_dir = f'{self.result_path}/{self.survey_id}'
+        markdown_filename = f'survey_{self.survey_id}.md'
+        markdown_filepath = os.path.join(markdown_dir, markdown_filename)
+        pdf_filename = f'survey_{self.survey_id}.pdf'
+        pdf_filepath = os.path.join(markdown_dir, pdf_filename)
+
+        markdown_content = self.get_markdown_content(json_filepath)
+        if not markdown_content:
+            raise ValueError("Markdown content is empty. Cannot generate citation files.")
+
+        try:
+            with open(markdown_filepath, 'w', encoding='utf-8', encoding="utf-8") as markdown_file:
+                markdown_file.write(markdown_content)
+            print(f"Markdown content saved to: {markdown_filepath}")
+        except Exception as e:
+            raise RuntimeError(f"Failed to save Markdown file: {e}")
+
+        try:
+            pdf = MarkdownPdf()
+            pdf.meta["title"] = "Citation Results" 
+            pdf.add_section(Section(markdown_content, toc=False))  
+            pdf.save(pdf_filepath) 
+            print(f"PDF content saved to: {pdf_filepath}")
+        except Exception as e:
+            raise RuntimeError(f"Failed to generate PDF file: {e}")
+        print(f"Files generated successfully: \nMarkdown: {markdown_filepath}\nPDF: {pdf_filepath}")
+
+    def get_markdown_content(self, json_filepath: str) -> str:
+        """
+        Read a JSON file and generate Markdown content based on its data.
+
+        :param json_filepath: Path to the JSON file containing survey data.
+        :return: A string containing the generated Markdown content.
+        """
+        try:
+            with open(json_filepath, 'r', encoding='utf-8', encoding="utf-8") as json_file:
+                survey_data = json.load(json_file)
+        except Exception as e:
+            raise RuntimeError(f"Failed to read JSON file: {e}")
+
+        topic = survey_data.get('survey_title', 'Default Topic')
+        content = survey_data.get('content', 'No content available.')
+
+        survey_title_markdown = f"# A Survey of {topic}\n\n"
+        survey_content_markdown = content + "\n\n"
+
+        markdown_content = survey_title_markdown + survey_content_markdown
+        markdown_content = finalize_survey_paper(markdown_content, self.collection_names, self.file_names)
+        return markdown_content
+
+if __name__ == "__main__":
+    root_path = "."
+    pdf_path = "./pdfs/test"
+    survey_title = "Automating Literature Review Generation with LLM"
+    cluster_standard = "method"
+    asg_system = ASG_system(root_path, 'test', pdf_path, survey_title, cluster_standard)
+    asg_system.parsing_pdfs()
+    asg_system.description_generation()
+    asg_system.agglomerative_clustering()
+    asg_system.outline_generation()
+    asg_system.section_generation()
+    asg_system.citation_generation()

src/demo/models.py

@@ -0,0 +1,14 @@
+from django.db import models
+
+# Create your models here.
+
+# class Choose_Topic(models.Model):
+#     question_text = models.CharField(max_length=200)
+#     pub_date = models.DateTimeField('date published')
+#
+#
+# class Choice(models.Model):
+#     question = models.ForeignKey(Question, on_delete=models.CASCADE)
+#     choice_text = models.CharField(max_length=200)
+#     votes = models.IntegerField(default=0)
+

src/demo/postprocess.py

@@ -0,0 +1,39 @@
+import re
+
+def reindex_citations(content):
+    """
+    将content中所有形如[collection_name]的引用标记，全局重编号为[1]、[2]、[3]...。
+    返回:
+        new_content: 替换后的文本
+        source_map: {collection_name: index, ...}
+    """
+    pattern = r"\[([^\[\]]+)\]"
+    source_map = {}
+    current_index = 1
+
+    def replace_func(match):
+        source = match.group(1)
+        nonlocal current_index
+        if source not in source_map:
+            source_map[source] = current_index
+            current_index += 1
+        return f"[{source_map[source]}]"
+
+    new_content = re.sub(pattern, replace_func, content)
+    return new_content, source_map
+
+def generate_references_section(source_map):
+    """
+    根据source_map生成References部分的文本。
+    source_map: {collection_name: index, ...}
+
+    返回值:
+        str: 
+        "References\n1 collection_name_1\n2 collection_name_2\n..."
+    """
+    # 将source_map按index排序
+    index_to_source = sorted(source_map.items(), key=lambda x: x[1])
+    refs_lines = ["References"]
+    for source, idx in index_to_source:
+        refs_lines.append(f"{idx} {source}")
+    return "\n".join(refs_lines)

src/demo/query1.py

@@ -0,0 +1,234 @@
+import os
+from openai import OpenAI
+from datetime import datetime, timedelta
+import re
+
+def generate_abstract_qwen(topic):
+    
+    # Initialize the OpenAI client using environment variables
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    
+    ###########################
+    # Step 1: Generate a survey abstract for the given topic.
+    ###########################
+    system_prompt_abstract = """
+You are a skilled research survey writer. Your task is to generate a survey abstract on the given topic. The abstract should cover the main challenges, key concepts, and research directions associated with the topic. Write in clear, concise academic English.
+"""
+    user_prompt_abstract = f"""
+Topic: {topic}
+
+Please generate a comprehensive survey abstract for this topic. Include discussion of core challenges, key terminologies, and emerging methodologies that are critical in the field. The total length of the abstract should be around 300–500 words.
+"""
+    messages_abstract = [
+        {"role": "system", "content": system_prompt_abstract},
+        {"role": "user", "content": user_prompt_abstract}
+    ]
+    
+    abstract_response = client.chat.completions.create(
+        model="Qwen2.5-72B-Instruct",
+        max_tokens=2048,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages_abstract
+    )
+    
+    abstract_text = ""
+    for chunk in abstract_response:
+        if chunk.choices[0].delta.content:
+            abstract_text += chunk.choices[0].delta.content
+    abstract_text = abstract_text.strip()
+    print("The abstract is:", abstract_text)
+
+    return abstract_text
+
+def generate_entity_lists_qwen(topic, abstract_text):
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    system_prompt_abstract = f"""
+    You are an AI assistant specializing in natural language processing and entity recognition. Your task is to extract key entities and core concepts from a given abstract based on a specified topic.  
+
+    You should return two distinct lists:  
+    1. **Entity list**: Entities that are synonymous or closely related to the given topic. These should be concise (no more than two words) and simplified to their root forms (e.g., removing suffixes like "-ing", "-ed").  
+    2. **Concept list**: Core concepts from the abstract that are highly relevant to the topic. These should also be concise (no more than two words) and in their simplest form.  
+
+    Ensure that your response follows this exact format:
+    Entity list: [entity1, entity2, entity3, ...]
+    Concept list: [concept1, concept2, concept3, ...]
+    Do not include any explanations or additional text.  
+
+    ### **Example**  
+    #### **Input:**  
+    Topic: Large Language Models  
+    Abstract: Ever since the Turing Test was proposed in the 1950s, humans have explored the mastering of language intelligence by machine. Language is essentially a complex, intricate system of human expressions governed by grammatical rules. It poses a significant challenge to develop capable artificial intelligence (AI) algorithms for comprehending and grasping a language. As a major approach, language modeling has been widely studied for language understanding and generation in the past two decades, evolving from statistical language models to neural language models. Recently, pre-trained language models (PLMs) have been proposed by pretraining Transformer models over large-scale corpora, showing strong capabilities in solving various natural language processing (NLP) tasks. Since the researchers have found that model scaling can lead to an improved model capacity, they further investigate the scaling effect by increasing the parameter scale to an even larger size. Interestingly, when the parameter scale exceeds a certain level, these enlarged language models not only achieve a significant performance improvement, but also exhibit some special abilities (e.g., in-context learning) that are not present in small-scale language models (e.g., BERT). To discriminate the language models in different parameter scales, the research community has coined the term large language models (LLM) for the PLMs of significant size (e.g., containing tens or hundreds of billions of parameters). Recently, the research on LLMs has been largely advanced by both academia and industry, and a remarkable progress is the launch of ChatGPT (a powerful AI chatbot developed based on LLMs), which has attracted widespread attention from society. The technical evolution of LLMs has been making an important impact on the entire AI community, which would revolutionize the way how we develop and use AI algorithms. Considering this rapid technical progress, in this survey, we review the recent advances of LLMs by introducing the background, key findings, and mainstream techniques. In particular, we focus on four major aspects of LLMs, namely pre-training, adaptation tuning, utilization, and capacity evaluation. Furthermore, we also summarize the available resources for developing LLMs and discuss the remaining issues for future directions. This survey provides an up-to-date review of the literature on LLMs, which can be a useful resource for both researchers and engineers.  
+
+    #### **Expected Output:**
+    "entity list": ["language model", "plm", "large language", "llm"]  
+    "concept list": ["turing", "language intelligence", "ai", "generation", "statistical", "neural", "pre-train", "transformer", "corpora", "nlp", "in-context", "bert", "chatgpt", "adaptation", "utilization"]
+    Make sure to strictly follow this format in your response.
+    """
+
+    user_prompt_abstract = f"""
+    Topic: {topic}  
+    Abstract: {abstract_text}  
+
+    Based on the given topic and abstract, extract the following:  
+    1. A **list of entities** that are synonymous or closely related to the topic. Keep each entity under two words and in its simplest form.  
+    2. A **list of core concepts** from the abstract that are highly relevant to the topic. Keep each concept under two words and in its simplest form.     
+    """
+
+    messages_abstract = [
+        {"role": "system", "content": system_prompt_abstract},
+        {"role": "user", "content": user_prompt_abstract}
+    ]
+    
+    entity_response = client.chat.completions.create(
+        model="Qwen2.5-72B-Instruct",
+        max_tokens=2048,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages_abstract
+    )
+    
+    entity_list = ""
+    for chunk in entity_response:
+        if chunk.choices[0].delta.content:
+            entity_list += chunk.choices[0].delta.content
+    entity_list = entity_list.strip()
+    print("The entity lists are:", entity_list)
+
+    return entity_list
+
+
+def generate_query_qwen(topic):
+    # Calculate date range for the arXiv query (last 5 years)
+    abstract_text = generate_abstract_qwen(topic)
+    entity_list = generate_entity_lists_qwen(topic, abstract_text)
+    today = datetime.now()
+    five_years_ago = today - timedelta(days=10 * 365)  # approximate calculation
+    start_date = five_years_ago.strftime('%Y%m%d')
+    end_date = today.strftime('%Y%m%d')
+
+
+    # System prompt: Focus on how to extract keywords from the abstract.
+    system_prompt_query = """
+    You are a research assistant specializing in constructing effective arXiv search queries. Your task is to generate a structured search query using **pre-extracted entity and concept lists** from a given abstract. Follow these instructions exactly:
+
+    1. **Input Data:**
+    - **Entity List:** A list of entities that are synonymous or closely related to the given topic.
+    - **Concept List:** A list of core concepts from the abstract that are highly relevant to the topic.
+
+    2. **Ensure Minimum Keyword Count:**
+    - **Entity List** must contain at least **5** terms. If there are fewer, intelligently supplement additional relevant terms.
+    - **Concept List** must contain **12-15** terms. If there are fewer, intelligently supplement additional relevant terms.
+
+    3. **Standardize Formatting:**
+    - Convert all terms to their **base form** and ensure they end with a wildcard `*`.  
+        - Examples: `verification → verif*`, `optimization → optim*`, `retrieval → retriev*`, `embedding → embed*`
+    - All terms must be **in lowercase**.
+
+    4. **Construct the Final Query:**
+    - The query must follow this exact structure:
+        ```
+        (abs:"<Entity1*>" OR abs:"<Entity2*>" OR abs:"<Entity3*>" OR abs:"<Entity4*>" OR abs:"<Entity5*>") AND 
+        (abs:"<Concept1*>" OR abs:"<Concept2*>" OR ... OR abs:"<Concept12*>")
+        ```
+    - **Entities are grouped together using `OR` in the first part.**
+    - **Concepts are grouped together using `OR` in the second part.**
+    - **The two groups are combined using `AND`.**
+    - **Do not include any explanations or extra text. Output only the final query.**
+    """
+
+    # User prompt: Provide examples of topics with corresponding query formats.
+    # User prompt: Provide examples of topics with corresponding query formats.
+# User prompt: Uses pre-extracted entities and concepts, ensures minimum count, and applies stemming + wildcards.
+    user_prompt_query = f"""
+    Below are the pre-extracted keywords for constructing the final arXiv query.
+
+    **Topic:** {topic}  
+    **Entity list and Concept list:** {entity_list}
+
+    ### **Processing Rules Applied:**
+    - **Ensure at least 5 entities** (if fewer, supplement additional relevant terms).
+    - **Ensure 12-15 concepts** (if fewer, supplement additional relevant terms).
+    - **Convert all terms to base form and append wildcard `*`.**
+    - **Output only the final query with no extra text.**
+
+    ### **Example Query Format:**
+
+    1. **Topic:** Large Language Models  
+    **Transformed Entity List:** ["languag model*", "plm*", "larg languag*", "llm*", "deep model*"]  
+    **Transformed Concept List:** ["tur*", "languag intellig*", "ai*", "gener*", "statist*", "neural*", "pre-train*", "transform*", "corpora*", "nlp*", "in-context*", "bert*", "chatgpt*", "adapt*", "utiliz*"]  
+    **Query:**  
+    (abs:"languag model*" OR abs:"plm*" OR abs:"larg languag*" OR abs:"llm*" OR abs:"deep model*") AND (abs:"tur*" OR abs:"languag intellig*" OR abs:"ai*" OR abs:"gener*" OR abs:"statist*" OR abs:"neural*" OR abs:"pre-train*" OR abs:"transform*" OR abs:"corpora*" OR abs:"nlp*" OR abs:"in-context*" OR abs:"bert*" OR abs:"chatgpt*" OR abs:"adapt*" OR abs:"utiliz*")
+    2. **Topic:** Quantum Computing  
+    **Transformed Entity List:** ["quant comput*", "qubit*", "qc*", "quant devic*", "topolog comput*"]  
+    **Transformed Concept List:** ["decoheren*", "entangl*", "error*", "topolog*", "anneal*", "photon*", "superconduct*", "algorithm*", "optim*", "verif*", "fault-toler*", "nois*", "cirquit*", "quant machin*", "measur*"]  
+    **Query:**
+    (abs:"quant comput*" OR abs:"qubit*" OR abs:"qc*" OR abs:"quant devic*" OR abs:"topolog comput*") AND (abs:"decoheren*" OR abs:"entangl*" OR abs:"error*" OR abs:"topolog*" OR abs:"anneal*" OR abs:"photon*" OR abs:"superconduct*" OR abs:"algorithm*" OR abs:"optim*" OR abs:"verif*" OR abs:"fault-toler*" OR abs:"nois*" OR abs:"cirquit*" OR abs:"quant machin*" OR abs:"measur*")
+    ---
+
+    ### **Now Generate the Query for This Topic:**
+    Using the provided **Entity List** and **Concept List**, apply the following steps:
+    1. **Ensure Entity List contains at least 5 items.** If fewer, supplement additional relevant terms.
+    2. **Ensure Concept List contains 12-15 items.** If fewer, supplement additional relevant terms.
+    3. **Convert all terms to their base form and append `*`.**
+    4. **Construct the arXiv search query in the same format as the examples above.**
+    5. **Return only the final query. Do not include explanations or additional text.**
+    """
+
+    # Initialize the OpenAI API client
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    messages = [
+        {"role": "system", "content": system_prompt_query},
+        {"role": "user", "content": user_prompt_query}
+    ]
+    
+    response = client.chat.completions.create(
+        model="Qwen2.5-72B-Instruct",
+        max_tokens=512,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages
+    )
+    
+    output_query = ""
+    for chunk in response:
+        if chunk.choices[0].delta.content:
+            output_query += chunk.choices[0].delta.content
+    match = re.search(r'\(.*\)', output_query, re.DOTALL)
+
+    if match:
+        extracted_query = match.group(0)  # 保留匹配到的整个括号内容
+    else:
+        extracted_query = output_query.strip()  # 如果匹配失败，使用原始查询
+
+    # 重新拼接 `submittedDate`
+    updated_query = f"{extracted_query} AND submittedDate:[{start_date} TO {end_date}]"
+    print('The response is :', updated_query)
+    return updated_query.strip()
+
+
+# Example usage:
+if __name__ == "__main__":
+    topic = "Quantum Computing"
+    final_query = generate_arxiv_query_chain_of_thought(topic)
+    print("\nFinal Query Returned:")
+    print(final_query)

src/demo/references.py

@@ -0,0 +1,83 @@
+import os
+import re
+from openai import OpenAI
+
+def getQwenClient(): 
+    openai_api_key = os.environ.get("OPENAI_API_KEY")
+    openai_api_base = os.environ.get("OPENAI_API_BASE")
+    
+    client = OpenAI(
+        # defaults to os.environ.get("OPENAI_API_KEY")
+        api_key = openai_api_key,
+        base_url = openai_api_base,
+    )
+    return client
+
+def generateResponse(client, prompt):
+    chat_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=1536,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=[{"role": "user", "content": prompt}]
+    )
+    # Stream the response to console
+    text = ""
+    for chunk in chat_response:
+        if chunk.choices[0].delta.content:
+            text += chunk.choices[0].delta.content
+    return text
+
+def generate_references(papers_info, client):
+
+    # In-Context Learning
+    examples = '''
+Example1:
+Authors: Armen Aghajanyan, Armen Aghajanyan, Anchit Gupta, Akshat Shrivastava, Xilun Chen, Luke Zettlemoyer, and Sonal Gupta
+Title: Muppet: Massive multi-task representations with pre-finetuning
+Reference: Armen Aghajanyan, Anchit Gupta, Akshat Shrivastava, Xilun Chen, Luke Zettlemoyer, and Sonal Gupta. Muppet: Massive multi-task representations with pre-finetuning
+
+Example2:
+Authors: Ari Holtzman1, Peter West222, Vered Shwartz3, Yejin Choi4, Luke Zettlemoyer12001
+Title:  Surface form competition: Why the highest probability answer isn't always right.
+Reference: Ari Holtzman, Peter West, Vered Shwartz, Yejin Choi, Luke Zettlemoyer. Surface form competition: Why the highest probability answer isn't always right.
+
+Example3:
+Authors: Mikel Artetxe, Shruti Bhosale, Naman Goyal, Todor Mihaylov, Myle Ott, Sam Shleifer, Xi Victoria Lin, Jingfei Du, Srinivasan Iyer, Ramakanth Pasunuru, Giri Anantharaman, Xian Li, Shuohui Chen, Halil Akin, Mandeep Baines, Louis Martin, Xing Zhou, Punit Singh Koura, Brian O'Horo, Jeff Wang, Luke Zettlemoyer, Mona Diab, Zornitsa Kozareva, Ves Stoyanov
+Title: Efficient large scale language modeling with mixtures of experts.
+Reference: Mikel Artetxe, Shruti Bhosale, Naman Goyal, Todor Mihaylov, Myle Ott, Sam Shleifer, Xi Victoria Lin, Jingfei Du, Srinivasan Iyer, Ramakanth Pasunuru, et al. Efficient large scale language modeling with mixtures of experts.
+'''
+
+    prompt = f'''
+Based on the following examples, generate the references based on the provided paper information.
+The generated references should be clear, legal and properly formatted.
+If the authors are many, list the first few authors followed by "et al.".
+
+Please include the "Reference:" label before each reference as shown in the examples.
+
+{examples}
+Now, please generate the references:
+
+'''
+
+    for idx, paper in enumerate(papers_info):
+        authors = paper['authors']
+        title = paper['title']
+        prompt += f'''
+Paper{idx+1}:
+Authors: {authors}
+Title: {title}
+Reference:'''
+    
+    response = generateResponse(client, prompt)
+    references = []
+    pattern = r'Reference:(.*?)(?=\n\n|$)'
+    matches = re.findall(pattern, response, re.S)
+    
+    for match in matches:
+        reference = match.strip()
+        if reference:
+            references.append(reference)
+
+    return references

src/demo/survey_generation_pipeline/asg_abstract.py

@@ -0,0 +1,247 @@
+import os
+
+
+class AbstractGenerator:
+    def __init__(self, pipeline):
+        self.pipeline = pipeline
+        
+    def generate(self, title, intro, mode='lora'):
+        if mode == 'lora' or mode == 'test':
+            if mode == 'lora':
+                self.pipeline.model.set_adapter("abstract")
+
+            system_prompt = f'''You are a helpful assistant that help to generate the abstract of the survey paper given the survey title and survey introduction.'''
+            # user_prompt = {"survey_title":survey_title, "claims":cluster_with_claims}
+            user_prompt = f'''Help me to generate the abstract of a survey paper given the title: *{title}*, and and the introduction:{intro}'''
+
+            messages = [
+                {"role": "system", "content": system_prompt}, 
+                {"role": "user", "content": user_prompt},
+                {"role": "assistant", "content":"Abstract: This survey "}
+                ] 
+
+            outputs = self.pipeline(
+                messages,
+                max_new_tokens=4096,
+            )
+            result = outputs[0]["generated_text"][-1]['content']
+            return result
+        else:
+            raise ValueError('mode not supported')
+
+if __name__ == '__main__':
+    from transformers import pipeline
+    import torch
+    import transformers
+
+    model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
+    Global_pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_id,
+        model_kwargs={"torch_dtype": torch.bfloat16},
+        token = os.getenv('HF_API_KEY'),
+        device_map="auto",
+    )
+    Global_pipeline.model.load_adapter(peft_model_id = "technicolor/llama3.1_8b_outline_generation", adapter_name="outline")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_abstract_generation", adapter_name="abstract")
+    Global_pipeline.model.load_adapter(peft_model_id ="technicolor/llama3.1_8b_conclusion_generation", adapter_name="conclusion")
+    title = "A Survey of Large Language Models"
+    intro = '''L
+ANGUAGE is a prominent ability in human beings to
+express and communicate, which develops in early
+childhood and evolves over a lifetime [3, 4]. Machines,
+however, cannot naturally grasp the abilities of understanding and communicating in the form of human language,
+unless equipped with powerful artificial intelligence (AI)
+algorithms. It has been a longstanding research challenge
+to achieve this goal, to enable machines to read, write, and
+communicate like humans [5].
+Technically, language modeling (LM) is one of the major
+approaches to advancing language intelligence of machines.
+In general, LM aims to model the generative likelihood
+of word sequences, so as to predict the probabilities of
+future (or missing) tokens. The research of LM has received
+extensive attention in the literature, which can be divided
+into four major development stages:
+• Statistical language models (SLM). SLMs [6–9] are developed based on statistical learning methods that rose in
+the 1990s. The basic idea is to build the word prediction
+model based on the Markov assumption, e.g., predicting the
+next word based on the most recent context. The SLMs with
+a fixed context length n are also called n-gram language
+models, e.g., bigram and trigram language models. SLMs
+have been widely applied to enhance task performance
+in information retrieval (IR) [10, 11] and natural language
+processing (NLP) [12–14]. However, they often suffer from
+the curse of dimensionality: it is difficult to accurately
+estimate high-order language models since an exponential
+number of transition probabilities need to be estimated.
+Thus, specially designed smoothing strategies such as backoff estimation [15] and Good–Turing estimation [16] have
+been introduced to alleviate the data sparsity problem.
+• Neural language models (NLM). NLMs [1, 17, 18] characterize the probability of word sequences by neural networks,
+e.g., multi-layer perceptron (MLP) and recurrent neural networks (RNNs). As a remarkable contribution, the work in
+[1] introduced the concept of distributed representation of
+words and built the word prediction function conditioned
+on the aggregated context features (i.e., the distributed
+word vectors). By extending the idea of learning effective
+features for text data, a general neural network approach
+was developed to build a unified, end-to-end solution for
+various NLP tasks [2]. Furthermore, word2vec [19, 20] was
+proposed to build a simplified shallow neural network
+for learning distributed word representations, which were
+demonstrated to be very effective across a variety of NLP
+tasks. These studies have initiated the use of language
+models for representation learning (beyond word sequence
+modeling), having an important impact on the field of NLP.
+• Pre-trained language models (PLM). As an early attempt, ELMo [21] was proposed to capture context-aware
+word representations by first pre-training a bidirectional
+LSTM (biLSTM) network (instead of learning fixed word
+representations) and then fine-tuning the biLSTM network
+according to specific downstream tasks. Furthermore, based
+on the highly parallelizable Transformer architecture [22]
+with self-attention mechanisms, BERT [23] was proposed by
+pre-training bidirectional language models with specially
+designed pre-training tasks on large-scale unlabeled corpora. These pre-trained context-aware word representations
+are very effective as general-purpose semantic features,
+which have largely raised the performance bar of NLP
+tasks. This study has inspired a large number of follow-up
+work, which sets the “pre-training and fine-tuning” learning
+paradigm. Following this paradigm, a great number of studies on PLMs have been developed, introducing either different architectures [24, 25] (e.g., GPT-2 [26] and BART [24]) or
+improved pre-training strategies [27–29]. In this paradigm, it
+often requires fine-tuning the PLM for adapting to different
+downstream tasks.
+• Large language models (LLM). Researchers find that
+scaling PLM (e.g., scaling model size or data size) often
+leads to an improved model capacity on downstream tasks
+(i.e., following the scaling law [30]). A number of studies
+have explored the performance limit by training an ever
+larger PLM (e.g., the 175B-parameter GPT-3 and the 540Bparameter PaLM). Although scaling is mainly conducted
+in model size (with similar architectures and pre-training
+tasks), these large-sized PLMs display different behaviors
+from smaller PLMs (e.g., 330M-parameter BERT and 1.5Bparameter GPT-2) and show surprising abilities (called emergent abilities [31]) in solving a series of complex tasks. For
+example, GPT-3 can solve few-shot tasks through in-context
+learning, whereas GPT-2 cannot do well. Thus, the research
+community coins the term “large language models (LLM)”
+1
+for these large-sized PLMs [32–35], which attract increasing
+research attention (See Figure 1). A remarkable application
+of LLMs is ChatGPT2
+that adapts the LLMs from the GPT
+series for dialogue, which presents an amazing conversation
+ability with humans. We can observe a sharp increase of the
+arXiv papers that are related to LLMs after the release of
+ChatGPT in Figure 1.
+As discussed before, language model is not a new technical concept specially for LLMs, but has evolved with the
+advance of artificial intelligence over the decades. Early language models mainly aim to model and generate text data,
+while latest language models (e.g., GPT-4) focus on complex
+task solving. From language modeling to task solving, it is an
+important leap in scientific thinking, which is the key to
+understand the development of language models in the research history. From the perspective of task solving, the four
+generations of language models have exhibited different levels of model capacities. In Figure 2, we describe the evolution process of language models in terms of the task solving
+capacity. At first, statistical language models mainly assisted
+in some specific tasks (e.g., retrieval or speech tasks), in
+which the predicted or estimated probabilities can enhance
+the performance of task-specific approaches. Subsequently,
+neural language models focused on learning task-agnostic
+representations (e.g., features), aiming to reduce the efforts
+for human feature engineering. Furthermore, pre-trained
+language models learned context-aware representations that
+can be optimized according to downstream tasks. For the
+latest generation of language model, LLMs are enhanced by
+exploring the scaling effect on model capacity, which can be
+considered as general-purpose task solvers. To summarize,
+in the evolution process, the task scope that can be solved
+by language models have been greatly extended, and the
+task performance attained by language models have been
+significantly enhanced.
+In the existing literature, PLMs have been widely discussed and surveyed [36–39], while LLMs are seldom reviewed in a systematic way. To motivate our survey, we first
+highlight three major differences between LLMs and PLMs.
+First, LLMs display some surprising emergent abilities that
+may not be observed in previous smaller PLMs. These abilities are key to the performance of language models on complex tasks, making AI algorithms unprecedently powerful
+and effective. Second, LLMs would revolutionize the way
+that humans develop and use AI algorithms. Unlike small
+PLMs, the major approach to accessing LLMs is through
+the prompting interface (e.g., GPT-4 API). Humans have to
+understand how LLMs work and format their tasks in a way
+that LLMs can follow. Third, the development of LLMs no
+longer draws a clear distinction between research and engineering. The training of LLMs requires extensive practical
+experiences in large-scale data processing and distributed
+parallel training. To develop capable LLMs, researchers
+have to solve complicated engineering issues, working with
+engineers or being engineers.
+Nowadays, LLMs are posing a significant impact on
+the AI community, and the advent of ChatGPT and GPT-4
+leads to the rethinking of the possibilities of artificial general
+intelligence (AGI). OpenAI has published a technical article
+entitled “Planning for AGI and beyond”, which discusses
+the short-term and long-term plans to approach AGI [40],
+and a more recent paper has argued that GPT-4 might be
+considered as an early version of an AGI system [41]. The
+research areas of AI are being revolutionized by the rapid
+progress of LLMs. In the field of NLP, LLMs can serve as a
+general-purpose language task solver (to some extent), and
+the research paradigm has been shifting towards the use
+of LLMs. In the field of IR, traditional search engines are
+challenged by the new information seeking way through AI
+chatbots (i.e., ChatGPT), and New Bing3 presents an initial
+attempt that enhances the search results based on LLMs. In
+the field of CV, the researchers try to develop ChatGPT-like
+vision-language models that can better serve multimodal
+dialogues [42–45], and GPT-4 [46] has supported multimodal input by integrating the visual information. This new
+wave of technology would potentially lead to a prosperous
+ecosystem of real-world applications based on LLMs. For
+instance, Microsoft 365 is being empowered by LLMs (i.e.,
+Copilot) to automate the office work, and OpenAI supports
+the use of plugins in ChatGPT for implementing special
+functions.
+Despite the progress and impact, the underlying principles of LLMs are still not well explored. Firstly, it is
+mysterious why emergent abilities occur in LLMs, instead of
+smaller PLMs. As a more general issue, there lacks a deep,
+detailed investigation of the key factors that contribute to
+the superior abilities of LLMs. It is important to study when
+and how LLMs obtain such abilities [47]. Although there are
+some meaningful discussions about this problem [31, 47],
+more principled investigations are needed to uncover the
+“secrets“ of LLMs. Secondly, it is difficult for the research
+community to train capable LLMs. Due to the huge demand of computation resources, it is very costly to carry
+out repetitive, ablating studies for investigating the effect
+of various strategies for training LLMs. Indeed, LLMs are
+mainly trained by industry, where many important training
+details (e.g., data collection and cleaning) are not revealed
+to the public. Thirdly, it is challenging to align LLMs with
+human values or preferences. Despite the capacities, LLMs
+are also likely to produce toxic, fictitious, or harmful contents. It requires effective and efficient control approaches
+to eliminating the potential risk of the use of LLMs [46].
+Faced with both opportunities and challenges, it needs
+more attention on the research and development of LLMs. In
+order to provide a basic understanding of LLMs, this survey
+conducts a literature review of the recent advances in LLMs
+from four major aspects, including pre-training (how to pretrain a capable LLM), adaptation (how to effectively adapt
+pre-trained LLMs for better use), utilization (how to use
+LLMs for solving various downstream tasks) and capability
+evaluation (how to evaluate the abilities of LLMs and existing
+empirical findings). We thoroughly comb the literature and
+summarize the key findings, techniques, and methods of
+LLMs. For this survey, we also create a GitHub project
+website by collecting the supporting resources for LLMs, at
+the link https://github.com/RUCAIBox/LLMSurvey. We
+are also aware of several related review articles on PLMs
+or LLMs [32, 36, 38, 39, 43, 48–54]. These papers either
+discuss PLMs or some specific (or general) aspects of LLMs.
+Compared with them, we focus on the techniques and
+methods to develop and use LLMs and provide a relatively
+comprehensive reference to important aspects of LLMs.
+The remainder of this survey is organized as follows:
+Section 2 introduces the background for LLMs and the evolution of GPT-series models, followed by the summarization
+of available resources for developing LLMs in Section 3.
+Sections 4, 5, 6, and 7 review and summarize the recent
+progress from the four aspects of pre-training, adaptation,
+utilization, and capacity evaluation, respectively. Then, Section 8 discusses the practical guide for prompt design,
+and Section 9 reviews the applications of LLMs in several
+representative domains. Finally, we conclude the survey in
+Section 10 by summarizing the major findings and discuss
+the remaining issues for future work.
+'''
+
+
+    abstract_generator = AbstractGenerator(Global_pipeline)
+    with_lora = abstract_generator.generate(title, intro, mode='lora')
+    with_test = abstract_generator.generate(title, intro, mode='test')

src/demo/survey_generation_pipeline/asg_clustername.py

@@ -0,0 +1,228 @@
+import os
+import pandas as pd
+import re  # Import the regular expressions module
+from openai import OpenAI
+import ast
+
+def generate_cluster_name_qwen_sep(tsv_path, survey_title):
+    data = pd.read_csv(tsv_path, sep='\t')
+    
+    # Define the system prompt once, outside the loop
+    system_prompt = f'''You are a research assistant working on a survey paper. The survey paper is about "{survey_title}". \
+    '''
+    
+    result = []  # Initialize the result list
+
+    for i in range(3):  # Assuming labels are 0, 1, 2
+        sentence_list = []  # Reset sentence_list for each label
+        for j in range(len(data)):
+            if data['label'][j] == i:
+                sentence_list.append(data['retrieval_result'][j])
+        
+        # Convert the sentence list to a string representation
+        user_prompt = f'''
+        Given a list of descriptions of sentences about an aspect of the survey, you need to use one phrase (within 8 words) to summarize it and treat it as a section title of your survey paper. \
+Your response should be a list with only one element and without any other information, for example, ["Post-training of LLMs"]  \
+Your response must contain one keyword of the survey title, unspecified or irrelevant results are not allowed. \
+The description list is:{sentence_list}'''
+        
+        messages = [
+            {"role": "system", "content": system_prompt}, 
+            {"role": "user", "content": user_prompt},
+        ]
+        
+        openai_api_key = os.getenv("OPENAI_API_KEY")
+        openai_api_base = os.getenv("OPENAI_API_BASE")
+        client = OpenAI(
+            api_key=openai_api_key,
+            base_url=openai_api_base,
+        )
+        
+        chat_response = client.chat.completions.create(
+            model=os.environ.get("MODEL"),
+            max_tokens=768,
+            temperature=0.5,
+            stop="<|im_end|>",
+            stream=True,
+            messages=messages
+        )
+        
+        # Stream the response to a single text string
+        text = ""
+        for chunk in chat_response:
+            if chunk.choices[0].delta.content:
+                text += chunk.choices[0].delta.content
+        
+        # Use regex to extract the first content within []
+        match = re.search(r'\[(.*?)\]', text)
+        if match:
+            cluster_name = match.group(1).strip()  # Extract and clean the cluster name
+            # 去除集群名称两侧的引号（如果存在）
+            cluster_name = cluster_name.strip('"').strip("'")
+            result.append(cluster_name)
+        else:
+            result.append("No Cluster Name Found")  # Handle cases where pattern isn't found
+    # print("The generated cluster names are:")
+    # print(result)
+    return result  # This will be a list with three elements
+    
+# Example usage:
+# result = generate_cluster_name_qwen_sep('path_to_your_file.tsv', 'Your Survey Title')
+# print(result)  # Output might look like ["Cluster One", "Cluster Two", "Cluster Three"]
+
+def refine_cluster_name(cluster_names, survey_title):
+    cluster_names = str(cluster_names)  # Convert to string to handle list input
+    # Define the system prompt to set the context
+    system_prompt = f'''You are a research assistant tasked with optimizing and refining a set of section titles for a survey paper. The survey paper is about "{survey_title}". 
+'''
+    
+    # Construct the user prompt, including all cluster names
+    user_prompt = f'''
+Here is a set of section titles generated for the survey topic "{survey_title}":
+{cluster_names}
+Please ensure that all cluster names are coherent and consistent with each other, and that each name is clear, concise, and accurately reflects the corresponding section.
+Notice to remove the overlapping information between the cluster names.
+Each cluster name should be within 8 words and include a keyword from the survey title.
+Response with a list of section titles in the following format without any other irrelevant information,
+For example, ["Refined Title 1", "Refined Title 2", "Refined Title 3"]
+'''
+    
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    # Initialize OpenAI client
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    try:
+        chat_response = client.chat.completions.create(
+            model=os.environ.get("MODEL"),
+            max_tokens=256,
+            temperature=0.5,
+            stop="<|im_end|>",
+            stream=True,
+            messages=messages
+        )
+    
+        # Stream the response and concatenate into a complete text
+        text = ""
+        for chunk in chat_response:
+            if chunk.choices[0].delta.content:
+                text += chunk.choices[0].delta.content
+
+        # print("The raw response text is:")
+        # print(text)
+    
+        # Use regex to extract content within square brackets
+        match = re.search(r'\[(.*?)\]', text)
+        if match:
+            refined_cluster_names = match.group(1).strip()  # Extract and clean the cluster name
+        else:
+            refined_cluster_names = [
+                survey_title + ": Definition",
+                survey_title + ": Methods",
+                survey_title + ": Evaluation"
+            ]  # Handle cases where pattern isn't found
+    
+    except Exception as e:
+        print(f"An error occurred while refining cluster names: {e}")
+        refined_cluster_names = ["Refinement Error"] * len(cluster_names)
+    
+    refined_cluster_names = ast.literal_eval(refined_cluster_names)  # Convert string to list
+    
+    # print("The refined cluster names are:")
+    # print(refined_cluster_names)
+    return refined_cluster_names  # Returns a list with the refined cluster names、
+
+
+
+
+def generate_cluster_name_new(tsv_path, survey_title, cluster_num = 3):
+    data = pd.read_csv(tsv_path, sep='\t')
+    desp=[]
+
+
+    for i in range(cluster_num):  # Assuming labels are 0, 1, 2
+        sentence_list = []  # Initialize the sentence list
+        for j in range(len(data)):
+            if data['label'][j] == i:
+                sentence_list.append(data['retrieval_result'][j])
+        desp.append(sentence_list)
+
+    system_prompt = f'''
+    You are a research assistant working on a survey paper. The survey paper is about "{survey_title}". '''
+    
+    cluster_info = "\n".join([f'Cluster {i+1}: "{desp[i]}"' for i in range(cluster_num)])
+
+    user_prompt = f'''
+    Your task is to generate {cluster_num} distinctive cluster names (e.g., "Pre-training of LLMs") of the given clusters of reference papers, each reference paper is described by a sentence.
+
+    The clusters of reference papers are: 
+    {cluster_info}
+
+    Your output should be a single list of {cluster_num} cluster names, e.g., ["Pre-training of LLMs", "Fine-tuning of LLMs", "Evaluation of LLMs"]
+    Do not output any other text or information.
+    '''
+
+    messages = [
+        {"role": "system", "content": system_prompt}, 
+        {"role": "user", "content": user_prompt},
+    ]
+    
+    openai_api_key = os.getenv("OPENAI_API_KEY")
+    openai_api_base = os.getenv("OPENAI_API_BASE")
+    client = OpenAI(
+        api_key=openai_api_key,
+        base_url=openai_api_base,
+    )
+    
+    chat_response = client.chat.completions.create(
+        model=os.environ.get("MODEL"),
+        max_tokens=768,
+        temperature=0.5,
+        stop="<|im_end|>",
+        stream=True,
+        messages=messages
+    )
+    
+    # Stream the response to a single text string
+    text = ""
+    for chunk in chat_response:
+        if chunk.choices[0].delta.content:
+            text += chunk.choices[0].delta.content
+    # print("The raw response text is:")
+    # print(text)
+
+    # Use regex to extract content within square brackets
+    match = re.search(r'\[(.*?)\]', text)
+    if match:
+        refined_cluster_names = match.group(1).strip()  # Extract and clean the cluster name
+    else:
+        predefined_sections = [
+            "Definition", "Methods", "Evaluation", "Applications",
+            "Challenges", "Future Directions", "Comparisons", "Case Studies"
+        ]
+        
+        # 根据 cluster_num 选择前 cluster_num 个预定义类别
+        refined_cluster_names = [
+            f"{survey_title}: {predefined_sections[i]}" for i in range(cluster_num)
+        ]
+    
+    refined_cluster_names = ast.literal_eval(refined_cluster_names)  # Convert string to list
+    
+    # print("The refined cluster names are:")
+    # print(refined_cluster_names)
+    return refined_cluster_names  # Returns a list with the refined cluster names、
+
+
+if __name__ == "__main__":
+    refined_result = refine_cluster_name(["Pre-training of LLMs", "Fine-tuning of LLMs", "Evaluation of LLMs"], 'Survey of LLMs')
+    # print(refined_result)
+        
+