update scripts

environment.yml

@@ -0,0 +1,166 @@
+name: scipip
+channels:
+  - pytorch
+  - nvidia
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - blas=1.0=mkl
+  - brotli-python=1.0.9=py310h6a678d5_8
+  - bzip2=1.0.8=h5eee18b_6
+  - ca-certificates=2024.9.24=h06a4308_0
+  - certifi=2024.8.30=py310h06a4308_0
+  - cffi=1.17.1=py310h1fdaa30_0
+  - charset-normalizer=3.3.2=pyhd3eb1b0_0
+  - cryptography=43.0.0=py310hdda0065_0
+  - cuda-cudart=11.8.89=0
+  - cuda-cupti=11.8.87=0
+  - cuda-libraries=11.8.0=0
+  - cuda-nvrtc=11.8.89=0
+  - cuda-nvtx=11.8.86=0
+  - cuda-runtime=11.8.0=0
+  - cuda-version=12.6=3
+  - filelock=3.13.1=py310h06a4308_0
+  - gmp=6.2.1=h295c915_3
+  - gmpy2=2.1.2=py310heeb90bb_0
+  - idna=3.7=py310h06a4308_0
+  - intel-openmp=2023.1.0=hdb19cb5_46306
+  - jinja2=3.1.4=py310h06a4308_0
+  - ld_impl_linux-64=2.40=h12ee557_0
+  - libcublas=11.11.3.6=0
+  - libcufft=10.9.0.58=0
+  - libcufile=1.11.1.6=0
+  - libcurand=10.3.7.77=0
+  - libcusolver=11.4.1.48=0
+  - libcusparse=11.7.5.86=0
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libnpp=11.8.0.86=0
+  - libnvjpeg=11.9.0.86=0
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libuuid=1.41.5=h5eee18b_0
+  - llvm-openmp=14.0.6=h9e868ea_0
+  - markupsafe=2.1.3=py310h5eee18b_0
+  - mkl=2023.1.0=h213fc3f_46344
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py310h06a4308_0
+  - ncurses=6.4=h6a678d5_0
+  - networkx=3.3=py310h06a4308_0
+  - openssl=3.0.15=h5eee18b_0
+  - pip=24.2=py310h06a4308_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyopenssl=24.2.1=py310h06a4308_0
+  - pysocks=1.7.1=py310h06a4308_0
+  - python=3.10.13=h955ad1f_0
+  - pytorch=2.1.0=py3.10_cuda11.8_cudnn8.7.0_0
+  - pytorch-cuda=11.8=h7e8668a_5
+  - pytorch-mutex=1.0=cuda
+  - pyyaml=6.0=py310h5eee18b_1
+  - readline=8.2=h5eee18b_0
+  - requests=2.31.0=py310h06a4308_0
+  - setuptools=68.0.0=py310h06a4308_0
+  - sqlite=3.45.3=h5eee18b_0
+  - sympy=1.13.2=py310h06a4308_0
+  - tbb=2021.8.0=hdb19cb5_0
+  - tk=8.6.14=h39e8969_0
+  - torchtriton=2.1.0=py310
+  - tqdm=4.65.0=py310h2f386ee_0
+  - typing_extensions=4.11.0=py310h06a4308_0
+  - urllib3=1.26.19=py310h06a4308_0
+  - wheel=0.44.0=py310h06a4308_0
+  - xz=5.4.6=h5eee18b_1
+  - yaml=0.2.5=h7b6447c_0
+  - yaml-cpp=0.7.0=h295c915_1
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - annotated-types==0.7.0
+      - antlr4-python3-runtime==4.9.3
+      - anyio==4.6.2.post1
+      - arrow==1.3.0
+      - beautifulsoup4==4.12.3
+      - bibtexparser==1.4.2
+      - blis==0.7.11
+      - cachetools==5.5.0
+      - catalogue==2.0.10
+      - click==8.1.7
+      - cloudpathlib==0.16.0
+      - confection==0.1.5
+      - cymem==2.0.8
+      - deprecated==1.2.14
+      - distro==1.9.0
+      - exceptiongroup==1.2.2
+      - free-proxy==1.1.2
+      - fsspec==2024.10.0
+      - h11==0.14.0
+      - httpcore==1.0.6
+      - httpx==0.27.2
+      - huggingface-hub==0.26.2
+      - interchange==2021.0.4
+      - joblib==1.4.2
+      - langcodes==3.4.1
+      - language-data==1.2.0
+      - loguru==0.7.2
+      - lxml==5.3.0
+      - marisa-trie==1.2.1
+      - markdown-it-py==3.0.0
+      - mdurl==0.1.2
+      - monotonic==1.6
+      - murmurhash==1.0.10
+      - neo4j==5.21.0
+      - numpy==1.26.0
+      - omegaconf==2.3.0
+      - openai==1.12.0
+      - outcome==1.3.0.post0
+      - pandas==2.2.3
+      - pansi==2020.7.3
+      - pillow==10.4.0
+      - preshed==3.0.9
+      - py2neo==2021.2.4
+      - pydantic==2.9.2
+      - pydantic-core==2.23.4
+      - pygments==2.18.0
+      - pyjwt==2.8.0
+      - pymupdf==1.24.5
+      - pymupdfb==1.24.3
+      - pyparsing==3.2.0
+      - pyphen==0.17.0
+      - python-dateutil==2.9.0.post0
+      - pytz==2024.2
+      - regex==2024.9.11
+      - rich==13.9.3
+      - safetensors==0.4.5
+      - scikit-learn==1.5.2
+      - scipy==1.14.1
+      - selenium==4.25.0
+      - sentence-transformers==3.0.1
+      - shellingham==1.5.4
+      - six==1.16.0
+      - smart-open==6.4.0
+      - sniffio==1.3.1
+      - sortedcontainers==2.4.0
+      - soupsieve==2.6
+      - spacy==3.7.4
+      - spacy-legacy==3.0.12
+      - spacy-loggers==1.0.5
+      - srsly==2.4.8
+      - textstat==0.7.4
+      - thinc==8.2.5
+      - threadpoolctl==3.5.0
+      - tokenizers==0.19.1
+      - transformers==4.44.0
+      - trio==0.27.0
+      - trio-websocket==0.11.1
+      - typer==0.9.4
+      - types-python-dateutil==2.9.0.20241003
+      - tzdata==2024.2
+      - wasabi==1.1.3
+      - weasel==0.3.4
+      - websocket-client==1.8.0
+      - wrapt==1.16.0
+      - wsproto==1.2.0
+      - zhipuai==2.1.5.20230904
+      - streamlit==1.39.0
+      - huggingface-hub==0.26.2

scripts/crawling.sh

@@ -0,0 +1 @@
+python src/paper_manager.py update --year all --venue-name iccv

scripts/embedding.sh

@@ -0,0 +1 @@
+python data/paper_manager.py embedding -c .configs/datasets.yaml

scripts/env.sh

@@ -0,0 +1,16 @@
+export NEO4J_URL="bolt://127.0.0.1:7687"
+export NEO4J_USERNAME="neo4j" # default neo4j
+export NEO4J_PASSWD="****" # your passwd
+## Use Qwen
+export MODEL_NAME="qwen-turbo"
+export MODEL_TYPE="OpenAI"
+export MODEL_API_KEY="sk-****"
+export BASE_URL="https://dashscope.aliyuncs.com/compatible-mode/v1"
+
+## Use GPT-4o
+# export MODEL_NAME="gpt-4o"
+# export MODEL_TYPE="OpenAI"
+# export MODEL_API_KEY="sk-***"
+# export BASE_URL="https://openai.***"
+
+

scripts/generator.sh

@@ -0,0 +1,5 @@
+python src/generator.py backtracking -m with_cited_paper --use-cue-words True --sum-api Qwen --gen-api Qwen
+# python generator.py backtracking -m with_cited_paper --use-cue-words False &&
+# python generator.py backtracking -m with_retrieved_paper --use-cue-words True &&
+# python generator.py backtracking -m with_retrieved_paper --use-cue-words False &&
+# python src/generator.py new-idea

scripts/retriever_eval.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+python src/retriever.py retrieve \
+  -c configs/datasets.yaml \
+  --ids-path assets/data/test_acl_2024.json
+
+wait
+echo "Retriever Eval Finish..."