Merge pull request #9 from CCCBora/dev

api_wrapper.py

@@ -12,18 +12,26 @@ todo:
         If `generator_wrapper` returns nothing or Timeout, or raise any error:
             Change Task status from Running to Failed.
 '''
+import os.path
 
 from auto_backgrounds import generate_draft
-import json
+import json, time
+from utils.file_operations import make_archive
 
 
-GENERATOR_MAPPING = {"draft": generate_draft}
+# GENERATOR_MAPPING = {"draft": generate_draft}
+GENERATOR_MAPPING = {"draft": None}
 
 def generator_wrapper(path_to_config_json):
     # Read configuration file and call corresponding function
     with open(path_to_config_json, "r", encoding='utf-8') as f:
         config = json.load(f)
-
-    generator = GENERATOR_MAPPING.get(config["generator"])
+    print("Configuration:", config)
+    # generator = GENERATOR_MAPPING.get(config["generator"])
+    generator = None
     if generator is None:
-        pass
+        # generate a fake ZIP file and upload
+        time.sleep(150)
+        zip_path = os.path.splitext(path_to_config_json)[0]+".zip"
+        return make_archive(path_to_config_json, zip_path)
+

app.py

@@ -5,8 +5,8 @@ from auto_backgrounds import generate_backgrounds, generate_draft
 from utils.file_operations import hash_name
 from references_generator import generate_top_k_references
 
-# note: App白屏bug：允许第三方cookie
 # todo:
+#   generation.log sometimes disappears
 #   6. get logs when the procedure is not completed. *
 #   7. 自己的文件库； 更多的prompts
 #   8. Decide on how to generate the main part of a paper * (Langchain/AutoGPT
@@ -54,7 +54,7 @@ def clear_inputs_refs(*args):
 
 
 def wrapped_generator(paper_title, paper_description, openai_api_key=None,
-                      paper_template="ICLR2022", tldr=True, max_num_refs=50, selected_sections=None, bib_refs=None, model="gpt-4",
+                      paper_template="ICLR2022", tldr=True, selected_sections=None, bib_refs=None, model="gpt-4",
                       cache_mode=IS_CACHE_AVAILABLE):
     # if `cache_mode` is True, then follow the following steps:
     #        check if "title"+"description" have been generated before
@@ -82,16 +82,14 @@ def wrapped_generator(paper_title, paper_description, openai_api_key=None,
             # generate the result.
             # output = fake_generate_backgrounds(title, description, openai_key)
             output = generate_draft(paper_title, paper_description, template=paper_template,
-                                    tldr=tldr, max_num_refs=max_num_refs,
-                                    sections=selected_sections, bib_refs=bib_refs, model=model)
+                                    tldr=tldr, sections=selected_sections, bib_refs=bib_refs, model=model)
             # output = generate_draft(paper_title, paper_description, template, "gpt-4")
             upload_file(output)
             return output
     else:
         # output = fake_generate_backgrounds(title, description, openai_key)
         output = generate_draft(paper_title, paper_description, template=paper_template,
-                                tldr=tldr, max_num_refs=max_num_refs,
-                                sections=selected_sections, bib_refs=bib_refs, model=model)
+                                tldr=tldr, sections=selected_sections, bib_refs=bib_refs, model=model)
         return output
 
 
@@ -170,9 +168,6 @@ with gr.Blocks(theme=theme) as demo:
 
                 title = gr.Textbox(value="Playing Atari with Deep Reinforcement Learning", lines=1, max_lines=1,
                                    label="Title", info="论文标题")
-
-                slider = gr.Slider(minimum=1, maximum=100, value=20, step=1,
-                                   interactive=True, visible=False, label="最大参考文献数目")
                 with gr.Accordion("高级设置", open=False):
                     with gr.Row():
                         description_pp = gr.Textbox(lines=5, label="Description (Optional)", visible=True,
@@ -264,7 +259,7 @@ with gr.Blocks(theme=theme) as demo:
 
     clear_button_pp.click(fn=clear_inputs, inputs=[title, description_pp], outputs=[title, description_pp])
     submit_button_pp.click(fn=wrapped_generator,
-                           inputs=[title, description_pp, key, template, tldr_checkbox, slider, sections, bibtex_file,
+                           inputs=[title, description_pp, key, template, tldr_checkbox, sections, bibtex_file,
                                    model_selection], outputs=file_output)
 
     clear_button_refs.click(fn=clear_inputs_refs, inputs=[title_refs, slider_refs], outputs=[title_refs, slider_refs])

auto_backgrounds.py

@@ -2,8 +2,7 @@ import os.path
 from utils.references import References
 from utils.file_operations import hash_name, make_archive, copy_templates
 from utils.tex_processing import create_copies
-from section_generator import section_generation_bg, keywords_generation, figures_generation, section_generation
-from references_generator import generate_top_k_references
+from section_generator import keywords_generation, section_generation # figures_generation, section_generation_bg,
 import logging
 import time
 
@@ -26,14 +25,16 @@ def log_usage(usage, generating_target, print_out=True):
     TOTAL_PROMPTS_TOKENS += prompts_tokens
     TOTAL_COMPLETION_TOKENS += completion_tokens
 
-    message = f"For generating {generating_target}, {total_tokens} tokens have been used ({prompts_tokens} for prompts; {completion_tokens} for completion). " \
+    message = f"For generating {generating_target}, {total_tokens} tokens have been used " \
+              f"({prompts_tokens} for prompts; {completion_tokens} for completion). " \
               f"{TOTAL_TOKENS} tokens have been used in total.\n\n"
     if print_out:
         print(message)
     logging.info(message)
 
+
 def _generation_setup(title, description="", template="ICLR2022", tldr=False,
-                      max_kw_refs=10, max_num_refs=50, bib_refs=None, max_tokens=2048):
+                      max_kw_refs=10, bib_refs=None, max_tokens=2048):
     """
     This function handles the setup process for paper generation; it contains three folds
         1. Copy the template to the outputs folder. Create the log file `generation.log`
@@ -44,9 +45,12 @@ def _generation_setup(title, description="", template="ICLR2022", tldr=False,
         title (str): The title of the paper.
         description (str, optional): A short description or abstract for the paper. Defaults to an empty string.
         template (str, optional): The template to be used for paper generation. Defaults to "ICLR2022".
-        tldr (bool, optional): A flag indicating whether a TL;DR (Too Long; Didn't Read) summary should be generated for the collected papers. Defaults to False.
-        max_kw_refs (int, optional): The maximum number of references that can be associated with each keyword. Defaults to 10.
-        max_num_refs (int, optional): The maximum number of references that can be included in the paper. Defaults to 50.
+        tldr (bool, optional): A flag indicating whether a TL;DR (Too Long; Didn't Read) summary should be used
+                               for the collected papers. Defaults to False.
+        max_kw_refs (int, optional): The maximum number of references that can be associated with each keyword.
+                                     Defaults to 10.
+        max_num_refs (int, optional): The maximum number of references that can be included in the paper.
+                                      Defaults to 50.
         bib_refs (list, optional): A list of pre-existing references in BibTeX format. Defaults to None.
 
     Returns:
@@ -69,9 +73,8 @@ def _generation_setup(title, description="", template="ICLR2022", tldr=False,
     keywords, usage = keywords_generation(input_dict)
     log_usage(usage, "keywords")
 
-    # generate keywords dictionary
+    # generate keywords dictionary # todo: in some rare situations, collected papers will be an empty list.
     keywords = {keyword:max_kw_refs for keyword in keywords}
-    print(f"keywords: {keywords}\n\n")
 
     ref = References(title, bib_refs)
     ref.collect_papers(keywords, tldr=tldr)
@@ -109,23 +112,38 @@ def generate_backgrounds(title, description="", template="ICLR2022", model="gpt-
 
 
 def generate_draft(title, description="", template="ICLR2022",
-                   tldr=True, max_kw_refs=10, max_num_refs=30, sections=None, bib_refs=None, model="gpt-4"):
+                   tldr=True, max_kw_refs=10, sections=None, bib_refs=None, model="gpt-4"):
+
+    def _filter_sections(sections):
+        ordered_sections = ["introduction", "related works", "backgrounds", "methodology", "experiments", "conclusion",
+                            "abstract"]
+        return [section for section in ordered_sections if section in sections]
     # pre-processing `sections` parameter;
+    print("================START================")
+    print(f"Generating the paper '{title}'.")
+    print("\n") # todo: use a configuration file to define parameters
     print("================PRE-PROCESSING================")
     if sections is None:
         sections = ["introduction", "related works", "backgrounds", "methodology", "experiments", "conclusion", "abstract"]
+    else:
+        sections = _filter_sections(sections)
 
-    # todo: add more parameters; select which section to generate; select maximum refs.
-    paper, destination_folder, _ = _generation_setup(title, description, template, tldr, max_kw_refs, max_num_refs, bib_refs)
+    if model == "gpt-4":
+        max_tokens = 4096
+    else:
+        max_tokens = 2048
+    paper, destination_folder, _ = _generation_setup(title, description, template, tldr, max_kw_refs, bib_refs, max_tokens=max_tokens)
 
     # main components
+    print(f"================PROCESSING================")
     for section in sections:
-        print(f"================Generate {section}================")
+        print(f"Generate {section} part...")
         max_attempts = 4
         attempts_count = 0
         while attempts_count < max_attempts:
             try:
                 usage = section_generation(paper, section, destination_folder, model=model)
+                print(f"{section} part has been generated. ")
                 log_usage(usage, section)
                 break
             except Exception as e:
@@ -153,7 +171,7 @@ if __name__ == "__main__":
     import openai
     openai.api_key = os.getenv("OPENAI_API_KEY")
 
-    title = "Using interpretable boosting algorithms for modeling environmental and agricultural data"
-    description = ""
-    output = generate_draft(title, description, tldr=True, max_kw_refs=10)
+    target_title = "Using interpretable boosting algorithms for modeling environmental and agricultural data"
+    target_description = ""
+    output = generate_draft(target_title, target_description, tldr=True, max_kw_refs=10)
     print(output)

references_generator.py

@@ -1,7 +1,16 @@
+'''
+This script is used to generate the most relevant papers of a given title.
+    - Search for as many as possible references. For 10~15 keywords, 10 references each.
+    - Sort the results from most relevant to least relevant.
+    - Return the most relevant using token size.
+
+Note: we do not use this function in auto-draft function. It has been integrated in that.
+'''
+
 import os.path
 import json
 from utils.references import References
-from section_generator import section_generation_bg, keywords_generation, figures_generation, section_generation
+from section_generator import keywords_generation # section_generation_bg,  #, figures_generation, section_generation
 import itertools
 from gradio_client import Client
 

section_generator.py

@@ -1,5 +1,6 @@
-from utils.prompts import generate_paper_prompts, generate_keywords_prompts, generate_experiments_prompts, generate_bg_summary_prompts
-from utils.gpt_interaction import get_responses, extract_responses, extract_keywords, extract_json
+# from utils.prompts import generate_paper_prompts, generate_keywords_prompts, generate_experiments_prompts, generate_bg_summary_prompts
+from utils.prompts import generate_paper_prompts, generate_bg_summary_prompts
+# from utils.gpt_interaction import get_responses #, extract_responses, extract_keywords, extract_json
 from utils.figures import generate_random_figures
 import time
 import os
@@ -15,8 +16,10 @@ import json
 
 MAX_ATTEMPTS = 6
 
+
 def section_generation_bg(paper, section, save_to_path, model):
     """
+    todo: this part should be revised
     The main pipeline of generating a section.
         1. Generate prompts.
         2. Get responses from AI assistant.
@@ -26,8 +29,9 @@ def section_generation_bg(paper, section, save_to_path, model):
     """
     print(f"Generating {section}...")
     prompts = generate_bg_summary_prompts(paper, section)
-    gpt_response, usage = get_responses(prompts, model)
-    output = gpt_response # extract_responses(gpt_response)
+    # gpt_response, usage = get_responses(prompts, model)
+    gpt_response, usage = get_gpt_responses(prompts, model)
+    output = gpt_response  # extract_responses(gpt_response)
     paper["body"][section] = output
     tex_file = os.path.join(save_to_path, f"{section}.tex")
     # tex_file = save_to_path + f"/{section}.tex"
@@ -58,8 +62,8 @@ def section_generation(paper, section, save_to_path, model, research_field="mach
     :return usage
     """
     prompts = generate_paper_prompts(paper, section)
-    output, usage= get_gpt_responses(SECTION_GENERATION_SYSTEM.format(research_field=research_field), prompts,
-                             model=model, temperature=0.4)
+    output, usage = get_gpt_responses(SECTION_GENERATION_SYSTEM.format(research_field=research_field), prompts,
+                                      model=model, temperature=0.4)
     paper["body"][section] = output
     tex_file = os.path.join(save_to_path, f"{section}.tex")
     with open(tex_file, "w") as f:
@@ -69,7 +73,7 @@ def section_generation(paper, section, save_to_path, model, research_field="mach
 
 
 def keywords_generation(input_dict, default_keywords=None):
-    '''
+    """
     Input:
         input_dict: a dictionary containing the title of a paper.
         default_keywords: if anything went wrong, return this keywords.
@@ -79,13 +83,13 @@ def keywords_generation(input_dict, default_keywords=None):
 
     Input example: {"title": "The title of a Machine Learning Paper"}
     Output Example: {"machine learning": 5, "reinforcement learning": 2}
-    '''
+    """
     title = input_dict.get("title")
     attempts_count = 0
     while (attempts_count < MAX_ATTEMPTS) and (title is not None):
         try:
-            keywords, usage= get_gpt_responses(KEYWORDS_SYSTEM.format(min_refs_num=1, max_refs_num=10), title,
-                                     model="gpt-3.5-turbo", temperature=0.4)
+            keywords, usage = get_gpt_responses(KEYWORDS_SYSTEM.format(min_refs_num=1, max_refs_num=10), title,
+                                                model="gpt-3.5-turbo", temperature=0.4)
             print(keywords)
             output = json.loads(keywords)
             return output.keys(), usage
@@ -99,10 +103,10 @@ def keywords_generation(input_dict, default_keywords=None):
     else:
         return default_keywords
 
-def figures_generation(paper, save_to_path, model):
-    # todo: this function is not complete.
-    prompts = generate_experiments_prompts(paper)
-    gpt_response, usage = get_responses(prompts, model)
-    list_of_methods = list(extract_json(gpt_response))
-    generate_random_figures(list_of_methods, os.path.join(save_to_path, "comparison.png"))
-    return usage
+# def figures_generation(paper, save_to_path, model):
+#     # todo: this function is not complete.
+#     prompts = generate_experiments_prompts(paper)
+#     gpt_response, usage = get_responses(prompts, model)
+#     list_of_methods = list(extract_json(gpt_response))
+#     generate_random_figures(list_of_methods, os.path.join(save_to_path, "comparison.png"))
+#     return usage

utils/gpt_interaction.py

@@ -1,80 +1,11 @@
+import os
 import openai
-import re
-import json
 import logging
+import requests
 
-log = logging.getLogger(__name__)
-
-
-def extract_responses(assistant_message):
-    # pattern = re.compile(r"f\.write\(r'{1,3}(.*?)'{0,3}\){0,1}$", re.DOTALL)
-    pattern = re.compile(r"f\.write\(r['\"]{1,3}(.*?)['\"]{0,3}\){0,1}$", re.DOTALL)
-    match = re.search(pattern, assistant_message)
-    if match:
-        return match.group(1)
-    else:
-        log.info("Responses are not put in Python codes. Directly return assistant_message.\n")
-        log.info(f"assistant_message: {assistant_message}")
-        return assistant_message
-
-
-def extract_keywords(assistant_message, default_keywords=None):
-    if default_keywords is None:
-        default_keywords = {"machine learning": 5}
-
-    try:
-        keywords = json.loads(assistant_message)
-    except ValueError:
-        log.info("Responses are not in json format. Return the default dictionary.\n ")
-        log.info(f"assistant_message: {assistant_message}")
-        return default_keywords
-    return keywords
-
-
-def extract_section_name(assistant_message, default_section_name=""):
-    try:
-        keywords = json.loads(assistant_message)
-    except ValueError:
-        log.info("Responses are not in json format. Return None.\n ")
-        log.info(f"assistant_message: {assistant_message}")
-        return default_section_name
-    return keywords
 
+log = logging.getLogger(__name__)
 
-def extract_json(assistant_message, default_output=None):
-    if default_output is None:
-        default_keys = ["Method 1", "Method 2"]
-    else:
-        default_keys = default_output
-    try:
-        dict = json.loads(assistant_message)
-    except:
-        log.info("Responses are not in json format. Return the default keys.\n ")
-        log.info(f"assistant_message: {assistant_message}")
-        return default_keys
-    return dict.keys()
-
-
-def get_responses(user_message, model="gpt-4", temperature=0.4, openai_key=None):
-    if openai.api_key is None and openai_key is None:
-        raise ValueError("OpenAI API key must be provided.")
-    if openai_key is not None:
-        openai.api_key = openai_key
-
-    conversation_history = [
-        {"role": "system", "content": "You are an assistant in writing machine learning papers."}
-    ]
-    conversation_history.append({"role": "user", "content": user_message})
-    response = openai.ChatCompletion.create(
-        model=model,
-        messages=conversation_history,
-        n=1,  # Number of responses you want to generate
-        temperature=temperature,  # Controls the creativity of the generated response
-    )
-    assistant_message = response['choices'][0]["message"]["content"]
-    usage = response['usage']
-    log.info(assistant_message)
-    return assistant_message, usage
 
 def get_gpt_responses(systems, prompts, model="gpt-4", temperature=0.4):
     conversation_history = [
@@ -93,17 +24,31 @@ def get_gpt_responses(systems, prompts, model="gpt-4", temperature=0.4):
     return assistant_message, usage
 
 
-if __name__ == "__main__":
-    test_strings = [r"f.write(r'hello world')", r"f.write(r'''hello world''')", r"f.write(r'''hello world",
-                    r"f.write(r'''hello world'", r'f.write(r"hello world")', r'f.write(r"""hello world""")',
-                    r'f.write(r"""hello world"', r'f.write(r"""hello world']
-    for input_string in test_strings:
-        print("input_string: ", input_string)
-        pattern = re.compile(r"f\.write\(r['\"]{1,3}(.*?)['\"]{0,3}\){0,1}$", re.DOTALL)
+def get_gpt_responses_test(systems, prompts, model="gpt-4", temperature=0.4, base_url=None, key=None):
+    end_point = r"/v1/chat/completions"
+    if base_url is None:
+        base_url =  r"https://api.openai.com" + end_point
+    if key is None:
+        key = os.getenv("OPENAI_API_KEY")
+
+    url = base_url + end_point
+
+    headers = {
+        'Content-Type': 'application/json',
+        'Authorization': f'Bearer {key}'  # <-- 把 fkxxxxx 替换成你自己的 Forward Key，注意前面的 Bearer 要保留，并且和 Key 中间有一个空格。
+    }
 
-        match = re.search(pattern, input_string)
-        if match:
-            extracted_string = match.group(1)
-            print("Extracted string:", extracted_string)
-        else:
-            print("No match found")
+    message = [{"role": "system", "content": systems},
+                {"role": "user", "content": prompts}]
+    data = {
+        "model": model,
+        "message": message,
+        "temperature": temperature
+    }
+    response = requests.post(url, headers=headers, json=data)
+    response = response.json()
+    return response['choices'][0]["message"]["content"]
+
+
+if __name__ == "__main__":
+    pass

utils/prompts.py

@@ -9,23 +9,23 @@ log = logging.getLogger(__name__)
 ######################################################################################################################
 # Some basic functions
 ######################################################################################################################
-def generate_keywords_prompts(title, description="", num_refs=5):
-    prompts = f"I am writing a machine learning paper with the title '{title}'. {description}\n" \
-                f"Generate three to five keywords. For each keyword, rate it from 1 to {num_refs}; the larger number means more important." \
-                r"Your response must be in JSON format like {\"keyword1\":1, \"keyword2\":3}."
-    return prompts
-
-def generate_rename_prompts(paper_info, section):
-    prompts = f"Please read the {section} section of the paper {paper_info['title']}: {paper_info['body'][section]}. \n" \
-              f"You need to rename this section to make it more specific to the context. " \
-              r"Response in a dictionary format like {\"option_1\": \"new_section_name_1\", \"option_2\": \"new_section_name_2\", ...}."
-    return prompts
-
-def generate_experiments_prompts(paper_info):
-    prompts = f"I am writing a machine learning paper with the title {paper_info['title']}\n" \
-              f"Please list two to four methods that I should compare my methods with and assign them with scores (5 means most related, 1 means least related). " \
-              r"Response in a dictionary format like {\"method_name_1\": 2, \"method_name_2\": 5, ...}. Use abbreviation to make their names have 5 characters or less."
-    return prompts
+# def generate_keywords_prompts(title, description="", num_refs=5):
+#     prompts = f"I am writing a machine learning paper with the title '{title}'. {description}\n" \
+#                 f"Generate three to five keywords. For each keyword, rate it from 1 to {num_refs}; the larger number means more important." \
+#                 r"Your response must be in JSON format like {\"keyword1\":1, \"keyword2\":3}."
+#     return prompts
+#
+# def generate_rename_prompts(paper_info, section):
+#     prompts = f"Please read the {section} section of the paper {paper_info['title']}: {paper_info['body'][section]}. \n" \
+#               f"You need to rename this section to make it more specific to the context. " \
+#               r"Response in a dictionary format like {\"option_1\": \"new_section_name_1\", \"option_2\": \"new_section_name_2\", ...}."
+#     return prompts
+#
+# def generate_experiments_prompts(paper_info):
+#     prompts = f"I am writing a machine learning paper with the title {paper_info['title']}\n" \
+#               f"Please list two to four methods that I should compare my methods with and assign them with scores (5 means most related, 1 means least related). " \
+#               r"Response in a dictionary format like {\"method_name_1\": 2, \"method_name_2\": 5, ...}. Use abbreviation to make their names have 5 characters or less."
+#     return prompts
 
 
 
@@ -68,6 +68,10 @@ SECTION_GENERATION_SYSTEM = PromptTemplate(input_variables=["research_field"],
 # Academic Paper
 ######################################################################################################################
 
+# When generating Academic Paper. Load instructions.
+# with open("../prompts/instructions.json", "r") as f:
+#     INSTRUCTIONS = json.load(f)
+#
 INSTRUCTIONS = {"introduction":
                     "- Include five paragraph: Establishing the motivation for the research. Explaining its importance and relevance to the AI community. Clearly state the problem you're addressing, your proposed solution, and the specific research questions or objectives. Briefly mention key related works for context and explain the main differences from this work. List three novel contributions of this paper.",
                "results":
@@ -213,4 +217,13 @@ def generate_bg_summary_prompts(paper_info, section):
         raise NotImplementedError
 
     log.info(f"Generated prompts for {section}: {prompts}")
-    return prompts
+    return prompts
+
+if __name__ == "__main__":
+    # import json
+    # with open("../prompts/instructions.json", "w") as f:
+    #     json.dump(INSTRUCTIONS, f)
+    import json
+    with open("../prompts/instructions.json", "r") as f:
+        ins = json.load(f)
+    print(ins == INSTRUCTIONS)

utils/references.py

@@ -27,6 +27,7 @@ from scholarly import ProxyGenerator
 import tiktoken
 import itertools, uuid, json
 from gradio_client import Client
+import time
 
 
 ######################################################################################################################
@@ -251,8 +252,10 @@ class References:
         comb_keywords = list(itertools.combinations(keywords, 2))
         for comb_keyword in comb_keywords:
             keywords.append(" ".join(comb_keyword))
+        print("Keywords: ", keywords)
         for key in keywords:
             self.papers[key] = _collect_papers_ss(key, 10, tldr)
+        print("Collected papers: ", papers)
         # for key, counts in keywords_dict.items():
         #     self.papers[key] = _collect_papers_ss(key, counts, tldr)
 
@@ -334,8 +337,12 @@ class References:
         prompts = {}
         tokens = 0
         for paper in result:
-            prompts[paper["paper_id"]] = paper["abstract"]
-            tokens += tiktoken_len(paper["abstract"])
+            abstract = paper.get("abstract")
+            if abstract is not None and isinstance(abstract, str):
+                prompts[paper["paper_id"]] = paper["abstract"]
+                tokens += tiktoken_len(paper["abstract"])
+            else:
+                prompts[paper["paper_id"]] = " "
             if tokens >= max_tokens:
                 break
         return prompts

worker.py

@@ -0,0 +1,172 @@
+'''
+This script is only used for service-side host.
+'''
+import boto3
+import os, time
+from api_wrapper import generator_wrapper
+from sqlalchemy import create_engine, Table, MetaData, update, select
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import inspect
+
+QUEUE_URL = os.getenv('QUEUE_URL')
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+BUCKET_NAME = os.getenv('BUCKET_NAME')
+DB_STRING = os.getenv('DATABASE_STRING')
+
+# Create engine
+ENGINE = create_engine(DB_STRING)
+SESSION = sessionmaker(bind=ENGINE)
+
+
+#######################################################################################################################
+# Amazon SQS Handler
+#######################################################################################################################
+def get_sqs_client():
+    sqs = boto3.client('sqs', region_name="us-east-2",
+                       aws_access_key_id=AWS_ACCESS_KEY_ID,
+                       aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
+    return sqs
+
+
+def receive_message():
+    sqs = get_sqs_client()
+    message = sqs.receive_message(QueueUrl=QUEUE_URL)
+    if message.get('Messages') is not None:
+        receipt_handle = message['Messages'][0]['ReceiptHandle']
+    else:
+        receipt_handle = None
+    return message, receipt_handle
+
+
+def delete_message(receipt_handle):
+    sqs = get_sqs_client()
+    response = sqs.delete_message(QueueUrl=QUEUE_URL, ReceiptHandle=receipt_handle)
+    return response
+
+
+#######################################################################################################################
+# AWS S3 Handler
+#######################################################################################################################
+def get_s3_client():
+    access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
+    secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+    session = boto3.Session(
+        aws_access_key_id=access_key_id,
+        aws_secret_access_key=secret_access_key,
+    )
+    s3 = session.resource('s3')
+    bucket = s3.Bucket(BUCKET_NAME)
+    return s3, bucket
+
+
+def upload_file(file_name, target_name=None):
+    s3, _ = get_s3_client()
+
+    if target_name is None:
+        target_name = file_name
+    s3.meta.client.upload_file(Filename=file_name, Bucket=BUCKET_NAME, Key=target_name)
+    print(f"The file {file_name} has been uploaded!")
+
+
+def download_file(file_name):
+    """ Download `file_name` from the bucket.
+    Bucket (str) – The name of the bucket to download from.
+    Key (str) – The name of the key to download from.
+    Filename (str) – The path to the file to download to.
+    """
+    s3, _ = get_s3_client()
+    s3.meta.client.download_file(Bucket=BUCKET_NAME, Key=file_name, Filename=os.path.basename(file_name))
+    print(f"The file {file_name} has been downloaded!")
+
+
+#######################################################################################################################
+# AWS SQL Handler
+#######################################################################################################################
+def modify_status(task_id, new_status):
+    session = SESSION()
+    metadata = MetaData()
+    task_to_update = task_id
+    task_table = Table('task', metadata, autoload_with=ENGINE)
+    stmt = select(task_table).where(task_table.c.task_id == task_to_update)
+    # Execute the statement
+    with ENGINE.connect() as connection:
+        result = connection.execute(stmt)
+
+        # Fetch the first result (if exists)
+        task_data = result.fetchone()
+
+        # If user_data is not None, the user exists and we can update the password
+        if task_data:
+            # Update statement
+            stmt = (
+                update(task_table).
+                    where(task_table.c.task_id == task_to_update).
+                    values(status=new_status)
+            )
+            # Execute the statement and commit
+            result = connection.execute(stmt)
+            connection.commit()
+    # Close the session
+    session.close()
+
+#######################################################################################################################
+# Pipline
+#######################################################################################################################
+def pipeline(message_count=0, query_interval=10):
+    # status: 0 - pending (default), 1 - running, 2 - completed, 3 - failed
+
+    # Query a message from SQS
+    msg, handle = receive_message()
+    if handle is None:
+        print("No message in SQS. ")
+        time.sleep(query_interval)
+    else:
+        print("===============================================================================================")
+        print(f"MESSAGE COUNT: {message_count}")
+        print("===============================================================================================")
+        config_s3_path = msg['Messages'][0]['Body']
+        config_s3_dir = os.path.dirname(config_s3_path)
+        config_local_path = os.path.basename(config_s3_path)
+        task_id, _ = os.path.splitext(config_local_path)
+
+        print("Initializing ...")
+        print("Configuration file on S3: ", config_s3_path)
+        print("Configuration file on S3 (Directory): ", config_s3_dir)
+        print("Local file path: ", config_local_path)
+        print("Task id: ", task_id)
+
+        print(f"Success in receiving message: {msg}")
+        print(f"Configuration file path: {config_s3_path}")
+
+        # Process the downloaded configuration file
+        download_file(config_s3_path)
+        modify_status(task_id, 1)  # status: 0 - pending (default), 1 - running, 2 - completed, 3 - failed
+        delete_message(handle)
+        print(f"Success in the initialization. Message deleted.")
+
+        print("Running ...")
+        # try:
+        zip_path = generator_wrapper(config_local_path)
+        # Upload the generated file to S3
+        upload_to = os.path.join(config_s3_dir, zip_path).replace("\\", "/")
+
+        print("Local file path (ZIP): ", zip_path)
+        print("Upload to S3: ", upload_to)
+        upload_file(zip_path, upload_to)
+        modify_status(task_id, 2) # status: 0 - pending (default), 1 - running, 2 - completed, 3 - failed, 4 - deleted
+        print(f"Success in generating the paper.")
+
+        # Complete.
+        print("Task completed.")
+
+
+def initialize_everything():
+    # Clear S3
+
+    # Clear SQS
+    pass
+
+
+if __name__ == "__main__":
+    pipeline()