major release - add pipeline & batch for difference use cases

.gitignore

@@ -161,7 +161,9 @@ data/*
 .env
 *.env
 
-clients/
+clients/*
 !clients/.gitkeep
 
-creds/
+creds/*
+logs/*
+!logs/.gitkeep

app.py

@@ -1,5 +1,4 @@
 import os
-import logging
 
 import gradio as gr
 import pandas as pd
@@ -9,13 +8,14 @@ jieba.cut('你好')
 from wordcloud import WordCloud
 from PIL import Image
 import matplotlib.pyplot as plt
+from loguru import logger
 
 from sheet import compose_query, get_serp, get_condensed_result, extract_results, postprocess_result, format_output, category2supercategory
 
 load_dotenv()
 
-logger = logging.getLogger(__name__) 
-logger.setLevel(logging.DEBUG)
+# logger = logging.getLogger(__name__) 
+# logger.setLevel(logging.DEBUG)
 
 classes = list([ x for x in category2supercategory.keys() if len(x)>0])
 
@@ -38,6 +38,7 @@ def format_category( formatted_results):
     return "\n\n".join([
         f"> 大類別：{formatted_results['supercategory'].values[0]}",
         f"> 小類別：{formatted_results['category'].values[0]}",
+        f"> 推測提供酒品：{ '是' if formatted_results['provide_alcohol'].values[0] else '否' }",
         f"> 商家名稱：{formatted_results['store_name'].values[0]}",
         f"> 電話：{formatted_results['phone_number'].values[0]}",
         f"> 描述：{formatted_results['description'].values[0]}"
@@ -48,8 +49,8 @@ def do( business_name: str, address: str):
     """
 
     crawled_results = []
-    provider = 'openai'
-    model = 'gpt-4-0125-preview'
+    provider = os.environ.get("DEFAULT_PROVIDER", "openai")
+    model = os.environ.get("DEFAULT_MODEL", "'gpt-4o'") 
 
     google_domain = "google.com.tw"
     gl = 'tw'
@@ -77,19 +78,30 @@ def do( business_name: str, address: str):
     # logger.debug(crawled_results)
     extracted_results = extract_results( crawled_results, classes=classes, provider = provider, model = model)
     # logger.error(extracted_results['extracted_results'].columns)
-    extracted_results = extracted_results['extracted_results'][ [ 'business_id', 'business_name', 'address', 'category', 'evidence', 'phone_number', 'description', 'store_name'] ]
+    extracted_results = extracted_results['extracted_results'][ [ 'business_id', 'business_name', 'address', 'category', 'evidence', 'phone_number', 'description', 'store_name', 'provide_alcohol'] ]
     logger.debug( extracted_results['category'])
-    print(extracted_results['category'])
+    
     postprocessed_results = postprocess_result( extracted_results, postprocessed_results_path="/tmp/postprocessed_results.joblib", category_hierarchy=category2supercategory)
     os.remove("/tmp/postprocessed_results.joblib")
 
     formatted_results = format_output( postprocessed_results)
     logger.debug( formatted_results)
-    print(formatted_results)
+    
     formatted_output = format_category( formatted_results)
 
     img = plot_wordcloud(formatted_results['formatted_evidence'].values[0])
-    return f"【搜尋結果】\n{formatted_results['formatted_evidence'].values[0]}", img, f"【判斷結果】\n{formatted_output}"
+    return f"【搜尋結果】\n{formatted_results['formatted_evidence'].values[0][6:]}", img, f"【判斷結果】\n{formatted_output}"
+
+def load( blob, progress=gr.Progress()):
+    """
+    """
+    if isinstance(blob, str):
+    #   df = pd.read_csv(StringIO(temp_file), parse_dates=[ "Start", "Finish"])
+        df = pd.read_csv(blob, names=COLUMNS, header=None) # parse_dates=[ "Start", "Finish"]
+    else:
+      df = pd.read_csv(blob.name, names=COLUMNS, header=None) # parse_dates=[ "Start", "Finish"]
+    print( df.head() )
+    return df
 
 ## --- interface --- ##
 # outputs = [gr.Dataframe(row_count = (1, "dynamic"), col_count=(6,"dynamic"), label="output data", interactive=1)]
@@ -99,20 +111,43 @@ def do( business_name: str, address: str):
 #         outputs=outputs,
 #     )
 
+COLUMNS = ['營業地址', '統一編號', '總機構統一編號', '營業人名稱', '資本額', '設立日期', '組織別名稱', '使用統一發票',
+        '行業代號', '名稱', '行業代號1', '名稱1', '行業代號2', '名稱2', '行業代號3', '名稱3']
+CSS = """
+h1 {
+    text-align: center;
+    display:block;
+}
+"""
 ## --- block --- ##
-with gr.Blocks() as demo:
-    gr.Markdown("🌟 自動分類餐廳型態 🌟")
-    with gr.Row():
-        # gr.Textbox( label="統一編號", placeholder="輸入八碼數字（optional）"), 
-        inputs = [ gr.Textbox( label="商家名稱", placeholder="輸入商家或公司名稱"), gr.Textbox(label="地址", placeholder="至少輸入縣市，完整地址更好")]
-    with gr.Row():
-        # outputs = [gr.Dataframe(row_count = (1, "dynamic"), col_count=(6,"dynamic"), label="output data", interactive=1)]
-        outputs = [ gr.Markdown( label="參考資料（google search）"), gr.Image( label="文字雲"), gr.Markdown( label="類別", )]
-    btn = gr.Button("Submit")
-    btn.click(fn=do, inputs=inputs, outputs=outputs)
+with gr.Blocks(css=CSS) as demo:
+    gr.Markdown("# 🌟 自動分類餐廳型態 🌟")
+    with gr.Tab('單筆'):
+        with gr.Row():
+            inputs = [ gr.Textbox( label="商家名稱", placeholder="輸入商家或公司名稱"), gr.Textbox(label="地址", placeholder="至少輸入縣���，完整地址更好")]
+        with gr.Row():
+            btn = gr.Button("Submit")
+        with gr.Row():
+            outputs = [ gr.Markdown( label="參考資料（google search）"), gr.Image( label="文字雲"), gr.Markdown( label="類別", )]
+        btn.click(fn=do, inputs=inputs, outputs=outputs)
+    with gr.Tab('批次'):
+        with gr.Row():
+            batch_inputs = [ gr.UploadButton("上傳檔案", file_count="single")]
+        with gr.Row():
+            batch_btn = gr.Button("批量處理")
+        with gr.Row():
+            batch_outputs = [ gr.Dataframe(
+                headers=COLUMNS,
+                datatype=["str"] * 16
+            )]
+        batch_btn.click(fn=load, inputs=batch_inputs, outputs=batch_outputs)
+        
 
 
 if __name__ == "__main__":
     
-    demo.launch(share=True, auth=("kota", "kota"))
+    demo.launch(
+        # share=True, 
+        server_name = '0.0.0.0', auth=( os.environ.get('USERNAME'), os.environ.get('PASSWORD'))
+    )
 

batch.py

@@ -0,0 +1,498 @@
+import os
+import sys
+import json
+import glob
+import time
+import yaml
+import joblib
+import argparse
+
+import jinja2
+import anthropic
+import pandas as pd
+from tqdm import tqdm
+from pathlib import Path
+from loguru import logger
+from openai import OpenAI
+from dotenv import load_dotenv
+import google.generativeai as genai
+from google.generativeai.types import HarmCategory, HarmBlockThreshold
+
+from data import get_leads
+from utils import parse_json_garbage, compose_query
+
+tqdm.pandas()
+
+try:
+    logger.remove(0)
+    logger.add(sys.stderr, level="INFO")
+except ValueError:
+    pass
+
+load_dotenv()
+
+
+def prepare_batch( crawled_result_path: str, config: dict, output_path: str, topn: int = None):
+    """
+    Argument
+    --------
+        crawled_result_path: str
+            Path to the crawled result file (result from the crawl task)
+        config: dict
+            Configuration for the batch job
+        output_path: str
+            Path to the output file
+    Return
+    ------
+        items: list
+    
+    Example
+        {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
+        {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
+        model = model, 
+        response_format = {"type": "json_object"},
+        temperature = 0,
+        max_tokens = 4096,
+    """
+    assert os.path.exists(crawled_result_path), f"File not found: {crawled_result_path}"
+    crawled_results = joblib.load(open(crawled_result_path, "rb"))['crawled_results']
+    if topn:
+        crawled_results = crawled_results.head(topn)
+
+    jenv = jinja2.Environment()
+
+    template = jenv.from_string(config['extraction_prompt'])
+    system_prompt = template.render( classes = config['classes'], traits = config['traits'])
+
+    template = jenv.from_string(config['user_content'])
+    
+    items = []
+    for i, d in tqdm(enumerate(crawled_results.itertuples())):
+        idx = d.index # d[1]
+        evidence = d.googlemap_results +"\n" + d.search_results
+        business_id = d.business_id # d[2]
+        business_name = d.business_name # d[3]
+        address = d.address # d[7]
+        ana_res = None
+        query = compose_query( address, business_name, use_exclude=False)
+
+        user_content = template.render( query = query, search_results = evidence)
+        item = {
+            "custom_id": str(business_id),
+            "method": "POST",
+            "url": "/v1/chat/completions",
+            "body": {
+                "model": config['model'],
+                "messages": [
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_content}
+                ],
+                "max_tokens": config['max_tokens'],
+                "temperature": config['temperature'],
+                "response_format": {"type": "json_object"},
+            }
+        }
+        items.append( json.dumps(item, ensure_ascii=False))
+
+    with open(output_path, "w") as f:
+        for item in items:
+            f.write(item + "\n")
+
+def prepare_regularization( extracted_result_path: str, config: dict, output_path: str, topn: int = None):
+    """
+    Argument
+    --------
+        extracted_file_path: str
+            Path to the extracted result file (result from the extraction task)
+        config: dict
+            Configuration for the batch job
+        output_path: str
+            Path to the output file
+        topn: int
+            Number of records to be processed
+    Return
+    ------
+        items: list
+    
+    Example
+        {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
+        {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
+        model = model, 
+        response_format = {"type": "json_object"},
+        temperature = 0,
+        max_tokens = 4096,
+    """
+    assert os.path.exists(extracted_result_path), f"File not found: {extracted_result_path}"
+    extracted_results = joblib.load(open(extracted_result_path, "rb"))['extracted_results']
+    if topn:
+        extracted_results = extracted_results.head(topn)
+
+    jenv = jinja2.Environment()
+
+    template = jenv.from_string(config['regularization_prompt'])
+    system_prompt = template.render()
+
+    template = jenv.from_string(config['regularization_user_content'])
+
+    items = []
+    for i, d in tqdm(enumerate(extracted_results.itertuples())):
+        idx = d.index # d[1]
+        category = d.category
+        business_id = d.business_id 
+        if pd.isna(category) or len(category)==0:
+            category = ""
+        user_content = template.render( category = category)
+        item = {
+            "custom_id": str(business_id),
+            "method": "POST",
+            "url": "/v1/chat/completions",
+            "body": {
+                "model": config['model'],
+                "messages": [
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_content}
+                ],
+                "max_tokens": config['max_tokens'],
+                "temperature": config['temperature'],
+                "response_format": {"type": "json_object"},
+            }
+        }
+        items.append( json.dumps(item, ensure_ascii=False))
+    with open(output_path, "w") as f:
+        for item in items:
+            f.write(item + "\n")
+
+def run_batch( input_path: str, job_path: str, jsonl_path: str):
+    """
+    Argument
+    --------
+        input_path: str
+            Path to the prepared batch input file (result from prepare_batch)
+        job_path: str
+            Path to the job file (response from creating a batch job)
+        jsonl_path: str
+            Path to the output file
+        extracted_result_path: str
+            Path to the extracted result file
+    
+    """
+    assert os.path.exists(input_path), f"File not found: {input_path}"
+    st = time.time()
+
+    client = OpenAI( organization = os.getenv('ORGANIZATION_ID'))
+    batch_input_file = client.files.create(
+        file=open( input_path, "rb"),
+        purpose="batch"
+    )
+    batch_input_file_id = batch_input_file.id
+    logger.info(f"batch_input_file_id -> {batch_input_file_id}")
+    batch_resp = client.batches.create(
+        input_file_id=batch_input_file_id,
+        endpoint="/v1/chat/completions",
+        completion_window="24h",
+        metadata={
+            "description": "batch job"
+        }
+    )
+    logger.info(f"batch resp -> {batch_resp}")
+    try:
+        with open( job_path, "wb") as f:
+            joblib.dump(batch_resp, f)
+    except Exception as e:
+        logger.error(f"Error -> {e}")
+        with open("./job.joblib", "wb") as f:
+            joblib.dump(batch_resp, f)
+    
+    is_ready = False
+    while 1:
+        batch_resp = client.batches.retrieve(batch_resp.id)
+
+        if batch_resp.status == 'validating':
+            logger.info("the input file is being validated before the batch can begin")
+
+        elif batch_resp.status == 'failed':
+            logger.info("the input file has failed the validation process")
+            break
+        elif batch_resp.status == 'in_progress':
+            logger.info("the input file was successfully validated and the batch is currently being ru")
+
+        elif batch_resp.status == 'finalizing':
+            logger.info("the batch has completed and the results are being prepared")
+        
+        elif batch_resp.status == 'completed':
+            logger.info("the batch has been completed and the results are ready")
+            is_ready = True
+            break
+        elif batch_resp.status == 'expired':
+            logger.info("the batch was not able to be completed within the 24-hour time window")
+            break
+        elif batch_resp.status == 'cancelling':
+            logger.info("the batch is being cancelled (may take up to 10 minutes)")
+
+        elif batch_resp.status == 'cancelled':
+            logger.info("the batch was cancelled")
+            break
+        else:
+            raise logger.error("Invalid status")
+        
+        time.sleep(10)
+
+    if is_ready:
+        output_resp = client.files.content(batch_resp.output_file_id)
+        llm_results = []
+        try:
+            with open(jsonl_path, "w") as f:
+                for line in output_resp.content.decode('utf-8').split("\n"):
+                    line = line.strip()
+                    if len(line)==0:
+                        break
+                    llm_results.append(line)
+                    f.write(f"{line}\n")
+            
+        except Exception as e:
+            logger.error(f"Error -> {e}")
+            with open("./output.jsonl", "w") as f:
+                for line in output_resp.content.decode('utf-8').split("\n"):
+                    line = line.strip()
+                    if len(line)==0:
+                        break
+                    llm_results.append(line)
+                    f.write(f"{line}\n")
+    print( f"Time elapsed: {time.time()-st:.2f} seconds")
+
+def batch2extract( jsonl_path: str, crawled_result_path: str, extracted_result_path: str):
+    """
+    Argument
+    --------
+        jsonl_path: str
+            Path to the batch output file
+        crawled_result_path: str    
+            Path to the crawled result file (result from the crawl task)    
+        extracted_result_path: str
+            Path to the extracted result file
+    
+    """
+    assert os.path.exists(jsonl_path), f"File not found: {jsonl_path}"
+    assert os.path.exists(crawled_result_path), f"File not found: {crawled_result_path}"
+    crawled_results = joblib.load(open(crawled_result_path, "rb"))
+    extracted_results = []
+    empty_indices = []
+    llm_results = []
+
+    for line in open(jsonl_path, "r"):
+        line = line.strip()
+        if len(line)==0:
+            break
+        llm_results.append(line)
+    
+    for i,llm_result in enumerate(llm_results):
+        try:
+            llm_result = json.loads(llm_result)
+            business_id = llm_result['custom_id']
+            llm_result = llm_result['response']['body']['choices'][0]['message']['content']
+            llm_result = parse_json_garbage(llm_result)
+            llm_result['business_id'] = business_id
+            extracted_results.append(llm_result)
+        except Exception as e:
+            logger.error(f"Error -> {e}, llm_result -> {llm_result}")
+            empty_indices.append(i)
+    extracted_results = pd.DataFrame(extracted_results)
+
+    basic_info = []
+    for i, d in tqdm(enumerate(crawled_results['crawled_results'].itertuples())):
+        idx = d.index # d[1]
+        evidence = d.googlemap_results +"\n" + d.search_results
+        business_id = d.business_id # d[2]
+        business_name = d.business_name # d[3]
+        address = d.address # d[7]
+        # ana_res = None
+        # query = compose_query( address, business_name, use_exclude=False)
+        basic_info.append( { 
+            "index": idx, 
+            "business_id": business_id, 
+            "business_name": business_name, 
+            "evidence": evidence, 
+            # ** ext_res
+        } )
+    basic_info = pd.DataFrame(basic_info)
+    
+    extracted_results = basic_info.astype({"business_id": str}).merge(extracted_results, on="business_id", how="inner")
+    print( f"{ extracted_results.shape[0]} records merged.")
+    extracted_results = {"extracted_results": extracted_results, "empty_indices": empty_indices}
+    with open(extracted_result_path, "wb") as f:
+        joblib.dump(extracted_results, f)
+
+def batch2reg( jsonl_path: str, extracted_result_path: str, regularized_result_path: str):
+    """
+    Argument
+    --------
+        jsonl_path: str
+            Path to the batch output file
+        extracted_result_path: str
+            Path to the extracted result file
+        regularized_result_path: str
+            Path to the regularization result file
+
+    """
+    assert os.path.exists(jsonl_path), f"File not found: {jsonl_path}"
+    assert os.path.exists(extracted_result_path), f"File not found: {extracted_result_path}"
+    extracted_results = joblib.load(open(extracted_result_path, "rb"))['extracted_results']
+
+    llm_results, regularized_results, empty_indices = [], [], []
+    for line in open(jsonl_path, "r"):
+        line = line.strip()
+        if len(line)==0:
+            break
+        llm_results.append(line)
+    
+    for i,llm_result in enumerate(llm_results):
+        try:
+            llm_result = json.loads(llm_result)
+            business_id = llm_result['custom_id']
+            llm_result = llm_result['response']['body']['choices'][0]['message']['content']
+            llm_result = parse_json_garbage(llm_result)
+            llm_result['business_id'] = business_id
+            regularized_results.append(llm_result)
+        except Exception as e:
+            logger.error(f"Error -> {e}, llm_result -> {llm_result}")
+            empty_indices.append(i)
+    regularized_results = pd.DataFrame(regularized_results)
+
+    basic_info = []
+    for i, d in tqdm(enumerate(extracted_results.itertuples())):
+        idx = d.index # d[1]
+        # evidence = d.googlemap_results +"\n" + d.search_results
+        evidence = d.evidence
+        business_id = d.business_id # d[2]
+        business_name = d.business_name # d[3]
+        # address = d.address # d[7]
+        # ana_res = None
+        # query = compose_query( address, business_name, use_exclude=False)
+        basic_info.append( { 
+            "index": idx, 
+            "business_id": business_id, 
+            "business_name": business_name, 
+            "evidence": evidence, 
+            # ** ext_res
+        } )
+    basic_info = pd.DataFrame(basic_info)
+
+    regularized_results = basic_info.astype({"business_id": str}).merge(regularized_results, on="business_id", how="inner")
+    print( f"{ regularized_results.shape[0]} records merged.")
+    regularized_results = {"regularized_results": regularized_results, "empty_indices": empty_indices}
+    with open(regularized_result_path, "wb") as f:
+        joblib.dump(regularized_results, f)
+
+
+def postprocess_result( config: dict, regularized_result_path: str, postprocessed_result_path, category_hierarchy: dict, column_name: str = 'category') -> pd.DataFrame:
+    """
+    Argument
+        config: dict
+        regularized_results_path: str
+        analysis_result: `evidence`, `result`
+        postprocessed_results_path
+    Return
+    """
+    assert os.path.exists(regularized_result_path), f"File not found: {regularized_result_path}"
+    regularized_results = joblib.load(open(regularized_result_path, "rb"))['regularized_results']
+
+    if True:
+    # if not os.path.exists(postprocessed_result_path):
+        postprocessed_results = regularized_results.copy()
+        postprocessed_results.loc[ :, "category"] = postprocessed_results[column_name].progress_apply(lambda x: "" if x not in category_hierarchy else x)
+        postprocessed_results['supercategory'] = postprocessed_results[column_name].progress_apply(lambda x: category_hierarchy.get(x, ''))
+        # with open( postprocessed_results_path, "wb") as f:
+        #     joblib.dump( postprocessed_results, f)
+        postprocessed_results.to_csv( postprocessed_result_path, index=False)
+    else:
+        # with open( postprocessed_results_path, "rb") as f:
+        #     postprocessed_results = joblib.load(f)
+        postprocessed_results = pd.read_csv( postprocessed_result_path)
+    return postprocessed_results
+
+
+def combine_postprocessed_results( config: dict, input_path: str, postprocessed_result_path: str, reference_path: str, output_path: str):
+    """
+    Argument
+        config: dict
+        input_path: str
+        postprocessed_result_path: str
+        reference_path: str
+        output_path: str
+    """
+    file_pattern = str(Path(input_path).joinpath( postprocessed_result_path, "postprocessed_results.csv"))
+    logger.info(f"file_pattern -> {file_pattern}")
+    file_paths = list(glob.glob(file_pattern))
+    assert len(file_paths)>0, f"File not found: {postprocessed_result_path}"
+    postprocessed_results = pd.concat([pd.read_csv(file_path, dtype={"business_id": str}) for file_path in file_paths], axis=0)
+    reference_results = get_leads( reference_path)
+    # reference_results = reference_results.rename(config['column_mapping'], axis=1)
+    postprocessed_results = reference_results.merge( postprocessed_results, left_on = "統一編號", right_on="business_id", how="left")
+    postprocessed_results.to_csv( output_path, index=False)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument( "-c", "--config", type=str, default='config/config.yml', help="Path to the configuration file")
+    parser.add_argument( "-t", "--task", type=str, default='prepare_batch', choices=['prepare_batch', 'prepare_regularization', 'run_batch', 'batch2extract', 'batch2reg', 'postprocess', 'combine'])
+    parser.add_argument( "-i", "--input_path", type=str, default='', )
+    parser.add_argument( "-o", "--output_path", type=str, default='', )
+    parser.add_argument( "-b", "--batch_path", type=str, default='', )
+    parser.add_argument( "-j", "--job_path", type=str, default='', )
+    parser.add_argument( "-jp", "--jsonl_path", type=str, default='', )
+    parser.add_argument( "-crp", "--crawled_result_path", type=str, default='', )
+    parser.add_argument( "-erp", "--extracted_result_path", type=str, default='', )
+    parser.add_argument( "-rrp", "--regularized_result_path", type=str, default='', )
+    parser.add_argument( "-prp", "--postprocessed_result_path", type=str, default='', )
+    parser.add_argument( "-rp", "--reference_path", type=str, default='', )
+    parser.add_argument( "-topn", "--topn", type=int, default=None )
+    args = parser.parse_args()
+    # classes = ['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)', ]
+    # backup_classes = [ '中式', '西式']
+
+    assert os.path.exists(args.config), f"File not found: {args.config}"
+    config = yaml.safe_load(open(args.config, "r").read())
+
+    if args.task == 'prepare_batch':
+        prepare_batch( crawled_result_path = args.crawled_result_path, config = config, output_path = args.output_path, topn = args.topn)
+
+    elif args.task == 'run_batch':
+        run_batch( input_path = args.input_path, job_path = args.job_path, jsonl_path = args.jsonl_path)
+
+    elif args.task == 'prepare_regularization':
+        prepare_regularization( extracted_result_path = args.extracted_result_path, config = config, output_path = args.output_path, topn = args.topn)
+
+    elif args.task == 'batch2extract':
+        batch2extract( 
+            jsonl_path = args.jsonl_path, 
+            crawled_result_path = args.crawled_result_path, 
+            extracted_result_path = args.extracted_result_path
+        )
+
+    elif args.task == 'batch2reg':
+        batch2reg( 
+            jsonl_path = args.jsonl_path, 
+            extracted_result_path = args.extracted_result_path, 
+            regularized_result_path = args.regularized_result_path
+        )
+    elif args.task == 'postprocess':
+        postprocess_result( 
+            config = config,
+            regularized_result_path = args.regularized_result_path, 
+            postprocessed_result_path = args.postprocessed_result_path, 
+            category_hierarchy = config['category2supercategory'], 
+            column_name = 'category'
+        )
+    elif args.task == 'combine':
+        combine_postprocessed_results( 
+            config, 
+            args.input_path, 
+            args.postprocessed_result_path, 
+            args.reference_path, 
+            args.output_path
+        )
+
+    else:
+        raise Exception("Invalid task")
+    
+    

config/config.yml

@@ -0,0 +1,115 @@
+model: "gpt-4o-mini"
+provider: openai
+column_mapping:
+  "營業地址": "address"
+  "統一編號": "business_id"
+  "總機構統一編號": "main_business_id"
+  "營業人名稱": "store_name"
+  "資本額": "capital"
+  "設立日期": "date"
+  "組織別名稱": "business_name"
+  "使用統一發票": "use_receipt"
+  "行業代號": "business_code"
+  "名稱": "business_name"
+  "行業代號1": "business_code_1"
+  "名稱1": "business_code_name_1"
+  "行業代號2": "business_code_2"
+  "名稱2": "business_code_name_2"
+  "行業代號3": "business_code_3"
+  "名稱3": "business_code_name_3"
+classes:
+  - 小吃店
+  - 日式料理(含居酒屋，串燒)
+  - 火(鍋／爐)
+  - 東南亞料理(不含日韓)
+  - 海鮮熱炒
+  - 特色餐廳(含雞、鵝、牛、羊肉)
+  - 釣蝦場
+  - 傳統餐廳
+  - 燒烤
+  - 韓式料理(含火鍋，烤肉)
+  - PUB(Live Band)
+  - PUB(一般，含Lounge)
+  - PUB(電音＼舞場)
+  - 五星級飯店
+  - 自助KTV(含連鎖，庭園自助)
+  - 西餐廳(含美式，義式，墨式)
+  - 咖啡廳(泡沫紅茶)
+  - 飯店(星級／旅館，不含五星級)
+  - 運動休閒館(含球類練習場，飛鏢等)
+  - 西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)
+  - 西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)
+  - 早餐
+category2supercategory:
+  "小吃店": "中式"
+  "日式料理(含居酒屋，串燒)": "中式"
+  "火(鍋／爐)": "中式"
+  "東南亞料理(不含日韓)": "中式"
+  "海鮮熱炒": "中式"
+  "特色餐廳(含雞、鵝、牛、羊肉)": "中式"
+  "釣蝦場": "中式"
+  "傳統餐廳": "中式"
+  "燒烤": "中式"
+  "韓式料理(含火鍋，烤肉)": "中式"
+  'PUB(Live Band)': "西式"
+  'PUB(一般，含Lounge)': "西式"
+  'PUB(電音＼舞場)': "西式"
+  "五星級飯店": "西式"
+  '自助KTV(含連鎖，庭園自助)': "西式"
+  "西餐廳(含美式，義式，墨式)": "西式"
+  '咖啡廳(泡沫紅茶)': "西式"
+  '飯店(星級／旅館，不含五星級)': "西式"
+  '運動休閒館(含球類練習場，飛鏢等)': "西式"
+  "中式": "中式"
+  "西式": "西式"
+  "西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)": "西式"
+  "西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)": "西式"
+  "早餐": ""
+traits: "Gathering, Chill, Enjoying Together, Drinking Freely, Winery, Wine Cellar, Wine Storage, Relaxing, Unwinding, Lyrical, Romantic, Pleasant, Stress Relief, Wine and Dine, Light Drinking Gatherings, Birthday Celebrations, Socializing, Parties, Networking, After Work Relaxation with a Drink, Relaxing Places Suitable for Drinking, Every Dish Goes Well with Beer, Shared Dishes, Dining Together, Atmosphere Suitable for Celebratory Drinking, Places Suitable for Light Drinking Gatherings with Friends, Small Shops Suitable for Relaxing and Light Drinking"
+extraction_prompt: |
+          As a helpful and rigorous retail analyst, given the provided query and a list of search results for the query,  your task is to first use store name and address to identify relevant information. 
+            After that, from the relevant information, extract `store_name`, `address`, `description`, `category`, `provide_alcohol` and `phone_number` from the found relevant information. 
+            Note that `category` can only be {{classes}}. 
+            According to our experience,`provide_alcohol` can be inferred based on whether a store is suitable for scenarios such as {{traits}}. 
+            `description` is a summary of key piece of evidence and reasons that lead you decide `category` and `provide_alcohol` .
+
+            It's very important to omit unrelated results. Do not make up any assumption.
+            Please think step by step, and output a single json that starts with `{` and ends with `}`. An example output json is like {"store_name": "...", "address": "...", "description": "... products, service or highlights ...", "category": "...", "phone_number": "...", "provide_alcohol": true or false}
+            If no relevant information has been found, simply output json with empty values.
+user_content: "`query`: `{{query}}`\n`search_results`: {{search_results}}"
+max_tokens: 4096
+temperature: 0.0
+classification_prompt: |
+          As a helpful and rigorous retail analyst, given the provided information about a store, 
+            your task is two-fold. First, classify provided evidence below into the mostly relevant category from the following: {classes}. 
+            Second, if no relevant information has been found, classify the evidence into the mostly relevant supercategory from the following: {backup_classes}.
+            It's very important to omit unrelated piece of evidence and don't make up any assumption.
+            Please think step by step, and must output in json format. An example output json is like {"category": "..."}
+            If no relevant piece of information can ever be found at all, simply output json with empty string "".
+            I'll tip you and guarantee a place in heaven you do a great job completely according to my instruction.
+
+regularization_prompt: |
+    As a helpful and factual assistant, your task is to classify the provided raw cuisine category into a conformed category. The definition of each conformed category is show below (in the format of `category`: `... definition ...`):
+    - `小吃店`:小吃、擔仔麵、小吃攤、街邊小店、傳統小吃、麵食、麵攤、炒飯、餃子館、鯊魚煙、黑白切、牛肉麵、銅板美食、小點心、簡餐、色小菜、開放空間攤販
+    - `日式料理(含居酒屋，串燒)`:居酒屋、酒場、水產、清酒、生魚片、壽司、日式啤酒、日式料理、代烤服務、日本餐飲場所、日本傳統食物、日式定食
+    - `火(鍋／爐)`:麻辣鍋、薑母鴨、鴨味仔、鍋物、湯底、滋補、冬令補、涮涮鍋、個人鍋、冬天圍爐、羊肉爐、鴛鴦鍋、炭火爐、氣火爐、燒酒雞、蒸氣海鮮鍋
+    - `東南亞料理(不含日韓)`:印尼、越式、泰式、沙嗲、海南雞、河粉、馬來西亞料理、新加坡料理、寮國料理、緬甸料理、南洋風味、印度料理、越南春捲、泰式綠咖哩、異國風情裝潢、滇緬料理
+    - `海鮮熱炒`:海鮮、現撈、活海鮮、生猛、大排檔、活魚活蝦、生猛海鮮、快炒、海產、台式海鮮、下酒菜
+    - `特色餐廳(含雞、鵝、牛、羊肉)`:烤鴨、燒鵝、甕仔雞、甕缸雞、桶仔雞、牛雜、蒙古烤肉、鵝肉城、金山鴨肉、生牛肉、全羊宴、活鱉、烤雞店、鵝肉餐廳、溫體牛、現宰羊肉、鹹水鵝、土羊肉
+    - `傳統餐廳`:江浙、台菜、合菜、桌菜、粵菜、中式、川菜、港式、上海菜、砂鍋魚頭、東北菜、北京烤鴨、一鴨三吃、婚宴、辦桌、老字號、宴會廳、台灣料理
+    - `燒烤`:燒烤、串燒、串串、烤魚、鮮蚵、炭烤、直火、碳火、和牛、戶外生火、烤肉、路邊燒烤
+    - `韓式料理(含火鍋，烤肉)`:韓國泡菜、韓式年糕、首爾、燒酒、韓式炸雞、春川辣炒雞、韓式炸醬麵、海鮮煎餅、烤三層肉、烤五花、烤韓牛、醬料和飯、石鍋拌飯、韓式風格、韓式清酒、啤酒、銅盤烤肉、韓流
+    - `PUB(Live Band)`:音樂餐廳、樂團表演、現場表演、LIVE表演、樂團駐唱、定期表演、有舞台場地、樂隊、專人駐唱
+    - `PUB(一般，含Lounge)`:酒吧、bar、lounge、飛鏢、調酒、運動酒吧、音樂酒吧、沙發聊天、女公關、互動調酒師、公關服務
+    - `PUB(電音＼舞場)`:夜店、舞池電音、藝人、包廂低消制、電子音樂表演、DJ、派對狂歡
+    - `五星級飯店`:高級飯店、奢華酒店、連鎖五星級飯店、國際集團飯店、米其林飯店、高檔住宿
+    - `自助KTV(含連鎖，庭園自助)`:卡拉OK、唱歌、歌坊、歡唱吧、自行點歌、自助唱歌、唱歌包廂、慶生聯誼包廂
+    - `西餐廳(含美式，義式，墨式)`:牛排、餐酒、歐式、義式、西餐、義大利麵、凱薩沙拉、紅酒、白酒、調酒、墨西哥式料理、阿根廷式料理、漢堡、比薩
+    - `咖啡廳(泡沫紅茶)`:泡沫紅茶店、咖啡店、café、coffee、輕食、軟性飲料、簡餐、茶街
+    - `飯店(星級／旅館，不含五星級)`:飯店、酒店、商務旅館、平價住宿
+    - `運動休閒館(含球類練習場，飛鏢等)`:撞球、高爾夫、運動、保齡球、娛樂、高爾夫練習場、大魯閣棒球場、籃球、羽毛球、PHOENIX鳳凰、羽球館、看球賽
+    - `釣蝦場`:釣蝦、蝦寶、投幣卡拉OK、釣竿和餌料、蝦子現場烹煮食用、泰國蝦、現烤蝦子、包廂唱歌、現釣現烤、自備或租用釣竿。
+    Note that you must choose from the above categories. Other ones are strongly prohibited. 
+    Output in json format such as `{"category": "..."}`.
+
+regularization_user_content: "{{ category }}"

data.py

@@ -0,0 +1,56 @@
+import os
+import json
+import math
+
+import pandas as pd
+from loguru import logger
+
+
+def get_leads( file_path: str, names: list = ['營業地址', '統一編號', '總機構統一編號', '營業人名稱', '資本額', '設立日期', '組織別名稱', '使用統一發票',
+       '行業代號', '名稱', '行業代號1', '名稱1', '行業代號2', '名稱2', '行業代號3', '名稱3']):
+    """
+    """
+    assert os.path.exists(file_path), f"File not found: {file_path}"
+    data = pd.read_csv( file_path, names=names, dtype={"統一編號": str})
+    return data
+
+def format_search_results(evidence):
+    """Format evidence (serp_results)
+    Argument
+        evidence: str. The format is a list of dictionaries
+    Return
+        formatted_evidence: str
+    """
+    search_results = []
+    gmap_store_mentions = []
+    evidence = json.loads(evidence)
+    # print( len(evidence) )
+    for i in range(len(evidence)):
+        if 'title' in evidence[i] and '顧客評價' in evidence[i]:
+            f = f"\n> 顧客評價： {evidence[i]['顧客評價']}"
+            gmap_store_mentions.append(f)
+        elif 'title' in evidence[i] and evidence[i]['title']=='類似的店':
+            f = f"\n> 類似的店： {evidence[i]['snippet']}"
+            gmap_store_mentions.append(f)
+        elif 'status' in evidence[i]:
+            f = f"\n> 經營狀態： {evidence[i]['status']}"
+            gmap_store_mentions.append(f)
+        elif 'telephone_number' in evidence[i]:
+            f = f"\n> 電話號碼： {evidence[i]['telephone_number']}"
+            gmap_store_mentions.append(f)
+        else:
+            try:
+                f = f"{i+1}. {evidence[i]['title']} ({evidence[i].get('snippet','')})"
+            except KeyError:
+                logger.error( evidence[i] )
+                raise KeyError
+            search_results.append(f)
+    return "## 店面資訊: " + "\n".join(gmap_store_mentions) + "\n" + "\n## 網路搜尋結果: " + "\n".join(search_results)
+
+def split_data( data: pd.DataFrame, samples: int = 4000):
+    """
+    """
+    data_len = len(data)
+    n = math.ceil(data_len/samples)
+    data_list = [data[ i*samples: (i+1)*samples] for i in range(n)]
+    return data_list

model.py

@@ -1,12 +1,28 @@
 import os
-import argparse
+import sys
+import json
 import time
+import yaml
+import joblib
+import argparse
 
-from dotenv import load_dotenv
+import jinja2
 import anthropic
+import pandas as pd
+from tqdm import tqdm
+from loguru import logger
 from openai import OpenAI
+from dotenv import load_dotenv
+import google.generativeai as genai
+from google.generativeai.types import HarmCategory, HarmBlockThreshold
 
-from utils import parse_json_garbage
+from utils import parse_json_garbage, compose_query
+
+try:
+    logger.remove(0)
+    logger.add(sys.stderr, level="INFO")
+except ValueError:
+    pass
 
 load_dotenv()
 
@@ -45,92 +61,105 @@ def llm( provider, model, system_prompt, user_content, delay:int = 0):
             model = model, 
             response_format = {"type": "json_object"},
             temperature = 0,
+            max_tokens = 4096,
             # stream = True
         )
         response = chat_completion.choices[0].message.content
 
     elif provider=='anthropic':
-        client = anthropic.Client(api_key=os.getenv('ANTHROPIC_APIKEY'))
+        client = anthropic.Client(api_key=os.getenv('ANTHROPIC_API_KEY'))
         response = client.messages.create(
             model= model,
             system= system_prompt,
             messages=[
                 {"role": "user", "content": user_content} # <-- user prompt
             ],
-            max_tokens = 1024
+            max_tokens = 4000
         )
         response = response.content[0].text
+
+    elif provider=='google':
+        genai.configure(api_key=os.getenv('GOOGLE_API_KEY'))
+        model = genai.GenerativeModel(
+            model_name = model,  
+            system_instruction = system_prompt,
+            generation_config={
+            "temperature": 0,
+            "max_output_tokens": 8192,
+            "response_mime_type": "application/json"
+        })
+        safety_settings = {
+            HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_ONLY_HIGH,
+            HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_ONLY_HIGH,
+            HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_ONLY_HIGH,
+            HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_ONLY_HIGH,
+        }
+        messages = []
+        # messages.append({
+        #     'role':'user',
+        #     'parts': [f"System instruction: {system_prompt}"]
+        # })
+        # response = model.generate_content(messages, safety_settings=safety_settings)
+        # try:
+        #     messages.append({
+        #         'role': 'model',
+        #         'parts': [response.text]
+        #     })
+        # except Exception as e:
+        #     logger.error(f"response.candidates -> {response.candidates}")
+        #     logger.error(f"error -> {e}")
+        #     messages.append({
+        #         'role': 'model',
+        #         'parts': ["OK. I'm ready to help you."]
+        #     })
+        messages.append({
+            'role': 'user',
+            'parts': [user_content]
+        })
+        try:
+            response = model.generate_content(messages, safety_settings=safety_settings, )
+            response = response.text
+        except Exception as e:
+            logger.error(f"Error (will still return response) -> {e}")
+            logger.error(f"response.candidates -> {response.candidates}")
+            return response
     else:
         raise Exception("Invalid provider")
     
     return response
 
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--provider", type=str, default='anthropic', help="openai or anthropic")
-    parser.add_argument("--model", type=str, default='gpt-4o', help="Model name for the API", 
-                        choices = ["claude-3-sonnet-20240229", "claude-3-haiku-20240307", "gpt-3.5-turbo-0125", "gpt-4-0125-preview"])
-    parser.add_argument("--classes", type=list, default=['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)', '西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)', '西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)', '早餐'])
-    parser.add_argument("--task", type=list, default='extract', choices=['extract', 'classify'])
+    parser.add_argument( "-c", "--config", type=str, default='config/config.yml', help="Path to the configuration file")
+    parser.add_argument( "-t", "--task", type=str, default='prepare_batch', choices=['extract', 'classify'])
+    parser.add_argument( "-i", "--input_path", type=str, default='', )
+    parser.add_argument( "-o", "--output_path", type=str, default='', )
+    parser.add_argument( "-topn", "--topn", type=int, default=None )
     args = parser.parse_args()
+    # classes = ['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)', ]
+    # backup_classes = [ '中式', '西式']
 
+    assert os.path.exists(args.config), f"File not found: {args.config}"
+    config = yaml.safe_load(open(args.config, "r").read())
 
-
-    classes = ['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)', ]
-    backup_classes = [ '中式', '西式']
-    
-    extraction_prompt = '''
-            As a helpful and rigorous retail analyst, given the provided query and a list of search results for the query, 
-            your task is to first identify relevant information of the identical store based on store name and proxmity of address if known. After that, extract `store_name`, `address`, `description`, `category` and `phone_number` from the found relevant information, where `category` can only be `小吃店`, `日式料理(含居酒屋，串燒)`, `火(鍋／爐)`, `東南亞料理(不含日韓)`, `海鮮熱炒`,  `特色餐廳(含雞、鵝、牛、羊肉)`, `傳統餐廳`, `燒烤`, `韓式料理(含火鍋，烤肉)`, `西餐廳(含美式，義式，墨式)`, `西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)`, `西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)` or `早餐`. 
-            It's very important to omit unrelated results. Do not make up any assumption.
-            Please think step by step, and output in json format. An example output json is like {"store_name": "...", "address": "...", "description": "... products, service or highlights ...", "category": "...", "phone_number": "..."}
-            If no relevant information has been found, simply output json with empty values.
-            I'll tip you and guarantee a place in heaven you do a great job completely according to my instruction.
-        '''
-    classification_prompt = f"""
-            As a helpful and rigorous retail analyst, given the provided information about a store, 
-            your task is two-fold. First, classify provided evidence below into the mostly relevant category from the following: {classes}. 
-            Second, if no relevant information has been found, classify the evidence into the mostly relevant supercategory from the following: {backup_classes}.
-            It's very important to omit unrelated piece of evidence and don't make up any assumption.
-            Please think step by step, and must output in json format. An example output json is like {{"category": "..."}}
-            If no relevant piece of information can ever be found at all, simply output json with empty string "".
-            I'll tip you and guarantee a place in heaven you do a great job completely according to my instruction.
-    """
-    
     if args.task == 'extract':
-        system_prompt = extraction_prompt
-    elif args.task == 'classify':
-        system_prompt = classification_prompt
-    else:
-        raise Exception("Invalid task")
-    
-    query = "山の迴饗"
-    search_results = str([{"title": "山の迴饗", "snippet": "謝謝大家這麼支持山の迴饗 我們會繼續努力用心做出美味的料理 ————————— ⛰️ 山の迴饗地址：台東縣關山鎮中華路56號訂位專線：0975-957-056 · #山的迴饗 · #夢想起飛"}, {"title": "山的迴饗餐館- 店家介紹", "snippet": "營業登記資料 · 統一編號. 92433454 · 公司狀況. 營業中 · 公司名稱. 山的迴饗餐館 · 公司類型. 獨資 · 資本總額. 30000 · 所在地. 臺東縣關山鎮中福里中華路56號 · 使用發票."}, {"title": "關山漫遊| 💥山の迴饗x night bar", "snippet": "山の迴饗x night bar 即將在12/1號台東關山開幕! 別再煩惱池上、鹿野找不到宵夜餐酒館 各位敬請期待並關注我們✨ night bar❌山的迴饗 12/1 ..."}, {"title": "山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵 - 台灣美食網", "snippet": "山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵｜台式三杯雞｜滷肉飯｜便當｜CP美食營業時間 ; 星期一, 休息 ; 星期二, 10:00–14:00 16:00–21:00 ; 星期三, 10:00–14:00 16:00– ..."}, {"title": "便當｜CP美食- 山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵", "snippet": "餐廳山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵｜台式三杯雞｜滷肉飯｜便當｜CP美食google map 導航. 臺東縣關山鎮中華路56號 +886 975 957 056 ..."}, {"title": "山的迴饗餐館", "snippet": "山的迴饗餐館,統編:92433454,地址:臺東縣關山鎮中福里中華路56號,負責人姓名:周偉慈,設立日期:112年11月15日."}, {"title": "山的迴饗餐館", "snippet": "山的迴饗餐館. 資本總額(元), 30,000. 負責人, 周偉慈. 登記地址, 看地圖 臺東縣關山鎮中福里中華路56號 郵遞區號查詢. 設立日期, 2023-11-15. 資料管理 ..."}, {"title": "山的迴饗餐館, 公司統一編號92433454 - 食品業者登錄資料集", "snippet": "公司或商業登記名稱山的迴饗餐館的公司統一編號是92433454, 登錄項目是餐飲場所, 業者地址是台東縣關山鎮中福里中華路56號, 食品業者登錄字號是V-202257990-00001-5."}, {"title": "山的迴饗餐館, 公司統一編號92433454 - 食品業者登錄資料集", "snippet": "公司或商業登記名稱山的迴饗餐館的公司統一編號是92433454, 登錄項目是公司/商業登記, 業者地址是台東縣關山鎮中福里中華路56號, 食品業者登錄字號是V-202257990-00000-4 ..."}, {"title": "山的迴饗餐館", "snippet": "負責人, 周偉慈 ; 登記地址, 台東縣關山鎮中福里中華路56號 ; 公司狀態, 核准設立 「查詢最新營業狀況請至財政部稅務入口網 」 ; 資本額, 30,000元 ; 所在縣市 ..."}, {"title": "山的迴饗 | 關山美食｜焗烤飯｜酒吧｜義大利麵｜台式三杯雞｜滷肉飯｜便當｜CP美食", "顧客評價": "324晚餐餐點豬排簡餐加白醬焗烤等等餐點。\t店家也提供免費的紅茶 綠茶 白開水 多種的調味料自取 總而言之 CP值真的很讚\t空間舒適涼爽，店員服務周到"}, {"title": "類似的店", "snippet": "['中國菜']\t['客家料理']\t['餐廳']\t['熟食店']\t['餐廳']"}, {"telephone_number": "0975 957 056"}])
-
-    # query = "大吃一斤泰國蝦麻辣牛肉爐"
-    # search_results = str([{"title": "大吃一斤泰國蝦麻辣牛肉爐", "snippet": "... 一支、本店特賣價600元免費代料理、 保證、活的!歡迎來電預定0975-147-848大吃一斤活蝦料理店新北市三重區自強路一段222號泰國蝦活蝦現場料理不漲價一斤維持一斤480元."}, {"title": "大吃一斤泰國蝦麻辣牛肉爐", "snippet": "... 一支、本店特賣價600元免費代料理、 保證、活的!歡迎來電預定0975-147-848大吃一斤活蝦料理店新北市三重區自強路一段222號泰國蝦活蝦現場料理不漲價一斤維持一斤480元."}, {"title": "大吃一斤", "snippet": "大吃一斤在foodpanda點的到，更多New Taipei City 推薦美食，線上訂立即送，下載foodpanda APP，20分鐘外送上門！瀏覽菜單和獨家優惠折扣."}, {"title": "大吃一斤(新北板橋店)菜單", "snippet": "大吃一斤(新北板橋店) 在foodpanda點的到，更多New Taipei City 推薦美食，線上訂立即送，下載foodpanda APP，20分鐘外送上門！"}, {"title": "大吃一斤活蝦餐廳- 店家介紹", "snippet": "大吃一斤活蝦餐廳. 資本總額. 200000. 代表人. 李錦鴻. 所在區域. 新北市. 所在地. 新北市三重區自強路1段222號(1樓). 商業類型. 獨資. 異動紀錄. 1111108. 營業狀態為: ..."}, {"title": "新北市| 三重區大吃一斤(泰國蝦牛肉料理店)", "snippet": "大吃一斤(泰國蝦牛肉料理店) 餐廳介紹 ; phone icon 電話, 0975 147 848 ; 營業時間, 星期一17:00–04:00 星期二17:00–04:00 星期三17:00–04:00 星期四17:00– ..."}, {"title": "大吃一斤活蝦餐廳", "snippet": "大吃一斤活蝦餐廳. 負責人姓名, 李錦鴻. 地址, 新北市三重區自強路1段222號(1樓). 現況, 核准設立. 資本額(元), 200,000. 組織類型, 獨資. 登記機關, 新北市政府經濟發展局."}, {"title": "【大吃一斤(泰國蝦牛肉料理店)】網友評價- 新北三重區合菜餐廳", "snippet": "大吃一斤(泰國蝦牛肉料理店) - 網友評論、最新食記(132則) 評分: 4.4分。大吃一斤(泰國蝦牛肉料理店)是位於新北三重區的餐廳，地址: 新北市 ... 生猛活海鮮."}, {"title": "大吃一斤生猛海鮮/活魚料理超值優惠方案", "snippet": "大吃一斤生猛海鮮/活魚料理. 電話：0975-147-848. 地址：新北市三重區自強路一段222號. 營業時間：週一至週日17: ..."}, {"title": "大吃一斤三重店 (泰國蝦料理.平價快炒熱炒.各式海鮮)", "顧客評價": "塔香蛤蜊、胡椒蝦、檸檬蝦、胡椒鳳螺 口味不錯食材新鮮 拍照時蛤蜊已經快被小孩吃光\t蝦子不大，店面不大，魚腥味很重，廁所很多蚊子，連菜裡面也有蚊子🦟，根本吃不下去\t新鮮好吃😋老闆人很Nice 推薦鹽烤蝦以及蒜味奶油蝦👍👍👍"}, {"title": "類似的店", "snippet": "['海鮮']\t['海鮮']\t['海鮮']\t['海鮮']"}, {"telephone_number": "0975 147 848"}])
-
-    if args.provider == "openai":
-        client = OpenAI( organization = os.getenv('ORGANIZATION_ID'))
-        # categories = ", ".join([ "`"+x+"`" for x in args.classes if x!='早餐' ])+ " or " + "`早餐`"
-        user_content = f'''
-            `query`: `{query}`,
-            `search_results`: {search_results}
-        '''
-        resp = llm( args.provider, args.model, system_prompt, user_content)
-        print(f"resp -> {resp}")
-
-
-    elif args.provider == "anthropic":
-        client = anthropic.Client(api_key=os.getenv('ANTHROPIC_APIKEY'))
+        jenv = jinja2.Environment()
+        template = jenv.from_string(config['extraction_prompt'])
+        system_prompt = template.render( classes = config['classes'], traits = config['traits'])
+        query = "山の迴饗"
+        search_results = str([{"title": "山の迴饗", "snippet": "謝謝大家這麼支持山の迴饗 我們會繼續努力用心做出美味的料理 ————————— ⛰️ 山の迴饗地址：台東縣關山鎮中華路56號訂位專線：0975-957-056 · #山的迴饗 · #夢想起飛"}, {"title": "山的迴饗餐館- 店家介紹", "snippet": "營業登記資料 · 統一編號. 92433454 · 公司狀況. 營業中 · 公司名稱. 山的迴饗餐館 · 公司類型. 獨資 · 資本總額. 30000 · 所在地. 臺東縣關山鎮中福里中華路56號 · 使用發票."}, {"title": "關山漫遊| 💥山の迴饗x night bar", "snippet": "山の迴饗x night bar 即將在12/1號台東關山開幕! 別再煩惱池上、鹿野找不到宵夜餐酒館 各位敬請期待並關注我們✨ night bar❌山的迴饗 12/1 ..."}, {"title": "山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵 - 台灣美食網", "snippet": "山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵｜台式三杯雞｜滷肉飯｜便當｜CP美食營業時間 ; 星期一, 休息 ; 星期二, 10:00–14:00 16:00–21:00 ; 星期三, 10:00–14:00 16:00– ..."}, {"title": "便當｜CP美食- 山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵", "snippet": "餐廳山的迴饗| 中西複合式餐廳｜焗烤飯｜義大利麵｜台式三杯雞｜滷肉飯｜便當｜CP美食google map 導航. 臺東縣關山鎮中華路56號 +886 975 957 056 ..."}, {"title": "山的迴饗餐館", "snippet": "山的迴饗餐館,統編:92433454,地址:臺東縣關山鎮中福里中華路56號,負責人姓名:周偉慈,設立日期:112年11月15日."}, {"title": "山的迴饗餐館", "snippet": "山的迴饗餐館. 資本總額(元), 30,000. 負責人, 周偉慈. 登記地址, 看地圖 臺東縣關山鎮中福里中華路56號 郵遞區號查詢. 設立日期, 2023-11-15. 資料管理 ..."}, {"title": "山的迴饗餐館, 公司統一編號92433454 - 食品業者登錄資料集", "snippet": "公司或商業登記名稱山的迴饗餐館的公司統一編號是92433454, 登錄項目是餐飲場所, 業者地址是台東縣關山鎮中福里中華路56號, 食品業者登錄字號是V-202257990-00001-5."}, {"title": "山的迴饗餐館, 公司統一編號92433454 - 食品業者登錄資料集", "snippet": "公司或商業登記名稱山的迴饗餐館的公司統一編號是92433454, 登錄項目是公司/商業登記, 業者地址是台東縣關山鎮中福里中華路56號, 食品業者登錄字號是V-202257990-00000-4 ..."}, {"title": "山的迴饗餐館", "snippet": "負責人, 周偉慈 ; 登記地址, 台東縣關山鎮中福里中華路56號 ; 公司狀態, 核准設立 「查詢最新營業狀況請至財政部稅務入口網 」 ; 資本額, 30,000元 ; 所在縣市 ..."}, {"title": "山的迴饗 | 關山美食｜焗烤飯｜酒吧｜義大利麵｜台式三杯雞｜滷肉飯｜便當｜CP美食", "顧客評價": "324晚餐餐點豬排簡餐加白醬焗烤等等餐點。\t店家也提供免費的紅茶 綠茶 白開水 多種的調味料自取 總而言之 CP值真的很讚\t空間舒適涼爽，店員服務周到"}, {"title": "類似的店", "snippet": "['中國菜']\t['客家料理']\t['餐廳']\t['熟食店']\t['餐廳']"}, {"telephone_number": "0975 957 056"}])
         user_content = f'''
             `query`: `{query}`,
             `search_results`: {search_results}
         '''
         print(f"user_content -> {user_content}")
-        resp = llm( args.provider, args.model, system_prompt, user_content)
+        resp = llm( config['provider'], config['model'], system_prompt, user_content)
         print(resp)
 
+    elif args.task == 'classify':
+        system_prompt = config['classification_prompt']
+
     else:
-        raise Exception("Invalid provider")
+        raise Exception("Invalid task")
+    
+    

notebooks/chaining.ipynb

@@ -0,0 +1,351 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "f9e39a86-a9db-4571-bdc1-bf2a14675345",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "import os\n",
+    "import json\n",
+    "\n",
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()\n",
+    "\n",
+    "TAVILY_API_KEY = os.environ.get(\"TAVILY_API_KEY\")\n",
+    "OPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5348dd2b-a2ae-4fa6-8268-be6da402898c",
+   "metadata": {},
+   "source": [
+    "### Use tools ###"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "e47f6678-69ff-4ead-96c7-d4b7cd59e561",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'url': 'https://maps.google.com.tw/',\n",
+       "  'content': 'Find local businesses, view maps and get driving directions in Google Maps.'},\n",
+       " {'url': 'https://www.google.com.tw/maps?hl=zh-TW&tab=wl&output=classic&dg=brw',\n",
+       "  'content': '利用「Google 地圖」尋找本地商家、檢視地圖或規劃行車路線。'},\n",
+       " {'url': 'https://twfile.com/五角味食堂/92277974',\n",
+       "  'content': '五角味食堂,商業統一編號:92277974,地址:彰化縣北斗鎮文昌里復興路臨100-1號,負責人姓名:李雅筑,OpenData(4) 五角味食堂(24項情報)-台灣公司情報網 台灣公司情報網'},\n",
+       " {'url': 'https://www.findcompany.com.tw/五角味食堂',\n",
+       "  'content': '五角味食堂. 資本總額 (元) 30,000. 負責人. 李雅筑. 登記地址. 彰化縣北斗鎮文昌里復興路臨100-1號 郵遞區號查詢. 設立日期. 2023-03-31.'},\n",
+       " {'url': 'https://twypage.com/sd-1654806-B-彰化-五角味食堂.html',\n",
+       "  'content': '五角味食堂,統編:92277974,地址:彰化縣北斗鎮文昌里復興路臨100-1號,負責人:李雅筑,設立日期:2023-03-31,變更日期:2023-03-31,公司狀態:核准設立 「查詢最新營業狀況請至財政部稅務入口網 」,營業項目:食品什貨、飲料零售業,餐館業'}]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+    "\n",
+    "\n",
+    "tool = TavilySearchResults()\n",
+    "tool.invoke({\"query\": \"五角味食堂\t彰化縣北斗鎮文昌里復興路臨100-1號\"})\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c1f236e-88bf-46e7-8e39-cbc9d7330f56",
+   "metadata": {},
+   "source": [
+    "### Define agent - AgentExecutor ###"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "22837788-d2f0-48a0-ad71-ff6226d5bd39",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain import hub\n",
+    "from langchain.agents import AgentExecutor, create_openai_functions_agent\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "\n",
+    "instructions = \"\"\"As a helpful and rigorous retail analyst, given the provided query and a list of search results for the query,  your task is to first use store name and address to identify relevant information. After that, from the relevant information, extract `store_name`, `address`, `description`, `category`, `provide_alcohol` and `phone_number` from the found relevant information. Note that `category` can only be `小吃店`, `日式料理(含居酒屋，串燒)`, `火(鍋／爐)`, `東南亞料理(不含日韓)`, `海鮮熱炒`,  `特色餐廳(含雞、鵝、牛、羊肉)`, `傳統餐廳`, `燒烤`, `韓式料理(含火鍋，烤肉)`, `西餐廳(含美式，義式，墨式)`, `西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)`, `西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)` or `早餐`. According to our experience,`provide_alcohol` can be inferred based on whether a store is suitable for  scenarios such as Gathering, Chill, Enjoying Together, Drinking Freely, Winery, Wine Cellar, Wine Storage, Relaxing, Unwinding, Lyrical, Romantic, Pleasant, Stress Relief, Wine and Dine, Light Drinking Gatherings, Birthday Celebrations, Socializing, Parties, Networking, After Work Relaxation with a Drink, Relaxing Places Suitable for Drinking, Every Dish Goes Well with Beer, Shared Dishes, Dining Together, Atmosphere Suitable for Celebratory Drinking, Places Suitable for Light Drinking Gatherings with Friends, Small Shops Suitable for Relaxing and Light Drinking. `description` is a summary of key piece of evidence and reasons that lead you decide `category` and `provide_alcohol` .\n",
+    "\n",
+    "            It's very important to omit unrelated results. Do not make up any assumption.\n",
+    "            Please think step by step, and output a single json that starts with `{` and ends with `}`. An example output json is like {\"store_name\": \"...\", \"address\": \"...\", \"description\": \"... products, service or highlights ...\", \"category\": \"...\", \"phone_number\": \"...\", \"provide_alcohol\": true or false}\n",
+    "            If no relevant information has been found, simply output json with empty values.\"\"\"\n",
+    "base_prompt = hub.pull(\"langchain-ai/openai-functions-template\")\n",
+    "prompt = base_prompt.partial(instructions=instructions)\n",
+    "llm = ChatOpenAI(temperature=0, )\n",
+    "tavily_tool = TavilySearchResults()\n",
+    "tools = [tavily_tool]\n",
+    "agent = create_openai_functions_agent(llm, tools, prompt)\n",
+    "agent_executor = AgentExecutor(\n",
+    "    agent=agent,\n",
+    "    tools=tools,\n",
+    "    verbose=True,\n",
+    ")\n",
+    "agent_executor.invoke({\"input\": \"五角味食堂\t彰化縣北斗鎮文昌里復興路臨100-1號\"})\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "38770f28-9950-49dc-a305-0ea180ae417c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3m\n",
+      "Invoking: `tavily_search_results_json` with `{'query': '五角味食堂 彰化縣北斗鎮文昌里復興路臨100-1號'}`\n",
+      "\n",
+      "\n",
+      "\u001b[0m\u001b[36;1m\u001b[1;3m[{'url': 'https://www.facebook.com/people/五角味食堂/100088207986612/', 'content': '五角味食堂, 彰化縣. 209 likes · 1 talking about this. 五角味食堂～ 一個結合.台灣.日本.中國.韓國.泰國.口味料理的複合式美食餐廳。 裡面以餃子類為主打!'}, {'url': 'https://twfile.com/五角味食堂/92277974', 'content': '五角味食堂,商業統一編號:92277974,地址:彰化縣北斗鎮文昌里復興路臨100-1號,負責人姓名:李雅筑,OpenData(4) 五角味食堂(24項情報)-台灣公司情報網 台灣公司情報網'}, {'url': 'https://opengovtw.com/ban/92277974', 'content': '所在地為彰化縣北斗鎮文昌里復興路臨100-1號。 ... 五角味食堂: 彰化縣北斗鎮文昌里復興路臨100-1號: 李雅筑: 30000: 獨資: 1120816972: 財政部營業稅籍資料. 營業人名稱: 五角味食堂 : 營業地址: 彰化縣北斗鎮文昌里復興路臨100－1號 :'}, {'url': 'https://www.google.com.tw/maps?hl=zh-TW&tab=wl&output=classic&dg=brw', 'content': '利用「Google 地圖」尋找本地商家、檢視地圖或規劃行車路線。'}, {'url': 'https://maps.google.com.tw/', 'content': 'Find local businesses, view maps and get driving directions in Google Maps.'}]\u001b[0m\u001b[32;1m\u001b[1;3m{\n",
+      "\"store_name\": \"五角味食堂\",\n",
+      "\"address\": \"彰化縣北斗鎮文昌里復興路臨100-1號\",\n",
+      "\"description\": \"五角味食堂是一個結合台灣、日本、中國、韓國、泰國口味料理的複合式美食餐廳，主打餃子類食物。\",\n",
+      "\"category\": \"小吃店\",\n",
+      "\"phone_number\": \"\",\n",
+      "\"provide_alcohol\": false\n",
+      "}\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'input': '五角味食堂\\t彰化縣北斗鎮文昌里復興路臨100-1號',\n",
+       " 'output': '{\\n\"store_name\": \"五角味食堂\",\\n\"address\": \"彰化縣北斗鎮文昌里復興路臨100-1號\",\\n\"description\": \"五角味食堂是一個結合台灣、日本、中國、韓國、泰國口味料理的複合式美食餐廳，主打餃子類食物。\",\\n\"category\": \"小吃店\",\\n\"phone_number\": \"\",\\n\"provide_alcohol\": false\\n}'}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "479c0691-72b1-4f43-9120-43007ea4c041",
+   "metadata": {},
+   "source": [
+    "### Utilities - external API wrapper (NOT TOOL!) ###"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "3d5884e2-c6b4-4454-bf21-f7b6bbfe2193",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.utilities import SerpAPIWrapper\n",
+    "SERPAPI_API_KEY=\"9d4e074bca8f9f7fa9ca5e05b874ea7da4f07cb505292ab29a0e1b91ffa0340a\"\n",
+    "os.environ[\"SERPAPI_API_KEY\"] = SERPAPI_API_KEY\n",
+    "SERPER_API_KEY=\"37128e339289d0e855c54f9afa9aa489bd7c23da\"\n",
+    "os.environ['SERPER_API_KEY']=SERPER_API_KEY\n",
+    "\n",
+    "from langchain_community.utilities import GoogleSerperAPIWrapper\n",
+    "\n",
+    "###  SERPER  ###\n",
+    "serper = GoogleSerperAPIWrapper(gl = 'tw', lr = 'lang_zh-TW', hl='zh-tw', k = 30) \n",
+    "# search.run(\"Obama's first name?\")\n",
+    "# serper_results = serper.results(\"宜窩餐飲有限公司 台北市 -inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw\")\n",
+    "serper_results = serper.results(\"致仙飲食店 澎湖縣 -inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw\")\n",
+    "print( serper_results.keys() )\n",
+    "print( serper_results['knowledgeGraph'])\n",
+    "print( serper_results['organic'] )\n",
+    "\n",
+    "\n",
+    "###  SERP API  ###\n",
+    "serp = SerpAPIWrapper(params = {\n",
+    "    \"gl\": 'tw', \n",
+    "    \"lr\": 'lang_zh-TW', \n",
+    "    \"hl\": 'zh-tw',\n",
+    "    \"google_domain\": \"google.com.tw\"\n",
+    "}) # utilities\n",
+    "# serp_results = serp.results(\"宜窩餐飲有限公司 台北市 -inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw\")\n",
+    "# serp_results = serp.results(\"吉祥餛飩麵食館 台北市 -inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw\")\n",
+    "serp_results = serp.results(\"致仙飲食店 澎湖縣 -inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw\")\n",
+    "\n",
+    "\n",
+    "print( serp_results.keys() )\n",
+    "print( serp_results['knowledge_graph'] \n",
+    "      "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "4c7290fb-931a-4f93-8469-ab4821b6c033",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n"
+     ]
+    },
+    {
+     "ename": "ValueError",
+     "evalue": "An output parsing error occurred. In order to pass this error back to the agent and have it try again, pass `handle_parsing_errors=True` to the AgentExecutor. This is the error: Could not parse output: No.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mOutputParserException\u001b[0m                     Traceback (most recent call last)",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/agents/agent.py:1167\u001b[0m, in \u001b[0;36mAgentExecutor._iter_next_step\u001b[0;34m(self, name_to_tool_map, color_mapping, inputs, intermediate_steps, run_manager)\u001b[0m\n\u001b[1;32m   1166\u001b[0m     \u001b[38;5;66;03m# Call the LLM to see what to do.\u001b[39;00m\n\u001b[0;32m-> 1167\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43magent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mplan\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1168\u001b[0m \u001b[43m        \u001b[49m\u001b[43mintermediate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1169\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_child\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrun_manager\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m   1170\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1171\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1172\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m OutputParserException \u001b[38;5;28;01mas\u001b[39;00m e:\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/agents/agent.py:732\u001b[0m, in \u001b[0;36mAgent.plan\u001b[0;34m(self, intermediate_steps, callbacks, **kwargs)\u001b[0m\n\u001b[1;32m    731\u001b[0m full_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mllm_chain\u001b[38;5;241m.\u001b[39mpredict(callbacks\u001b[38;5;241m=\u001b[39mcallbacks, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mfull_inputs)\n\u001b[0;32m--> 732\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput_parser\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfull_output\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/agents/output_parsers/self_ask.py:41\u001b[0m, in \u001b[0;36mSelfAskOutputParser.parse\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfinish_string \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m last_line:\n\u001b[0;32m---> 41\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m OutputParserException(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCould not parse output: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtext\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     42\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m AgentFinish({\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124moutput\u001b[39m\u001b[38;5;124m\"\u001b[39m: last_line[\u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfinish_string) :]}, text)\n",
+      "\u001b[0;31mOutputParserException\u001b[0m: Could not parse output: No.",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[22], line 22\u001b[0m\n\u001b[1;32m     12\u001b[0m tools \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     13\u001b[0m     Tool(\n\u001b[1;32m     14\u001b[0m         name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIntermediate Answer\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     17\u001b[0m     )\n\u001b[1;32m     18\u001b[0m ]\n\u001b[1;32m     19\u001b[0m self_ask_with_search \u001b[38;5;241m=\u001b[39m initialize_agent(\n\u001b[1;32m     20\u001b[0m     tools, llm, agent\u001b[38;5;241m=\u001b[39mAgentType\u001b[38;5;241m.\u001b[39mSELF_ASK_WITH_SEARCH, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m     21\u001b[0m )\n\u001b[0;32m---> 22\u001b[0m \u001b[43mself_ask_with_search\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m五角味食堂\t彰化縣北斗鎮文昌里復興路臨100-1號\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:168\u001b[0m, in \u001b[0;36mdeprecated.<locals>.deprecate.<locals>.warning_emitting_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    166\u001b[0m     warned \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     emit_warning()\n\u001b[0;32m--> 168\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/chains/base.py:600\u001b[0m, in \u001b[0;36mChain.run\u001b[0;34m(self, callbacks, tags, metadata, *args, **kwargs)\u001b[0m\n\u001b[1;32m    598\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m    599\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`run` supports only one positional argument.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 600\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmetadata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m[\n\u001b[1;32m    601\u001b[0m         _output_key\n\u001b[1;32m    602\u001b[0m     ]\n\u001b[1;32m    604\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m kwargs \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m args:\n\u001b[1;32m    605\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m(kwargs, callbacks\u001b[38;5;241m=\u001b[39mcallbacks, tags\u001b[38;5;241m=\u001b[39mtags, metadata\u001b[38;5;241m=\u001b[39mmetadata)[\n\u001b[1;32m    606\u001b[0m         _output_key\n\u001b[1;32m    607\u001b[0m     ]\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:168\u001b[0m, in \u001b[0;36mdeprecated.<locals>.deprecate.<locals>.warning_emitting_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    166\u001b[0m     warned \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     emit_warning()\n\u001b[0;32m--> 168\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/chains/base.py:383\u001b[0m, in \u001b[0;36mChain.__call__\u001b[0;34m(self, inputs, return_only_outputs, callbacks, tags, metadata, run_name, include_run_info)\u001b[0m\n\u001b[1;32m    351\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Execute the chain.\u001b[39;00m\n\u001b[1;32m    352\u001b[0m \n\u001b[1;32m    353\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    374\u001b[0m \u001b[38;5;124;03m        `Chain.output_keys`.\u001b[39;00m\n\u001b[1;32m    375\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    376\u001b[0m config \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    377\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcallbacks\u001b[39m\u001b[38;5;124m\"\u001b[39m: callbacks,\n\u001b[1;32m    378\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtags\u001b[39m\u001b[38;5;124m\"\u001b[39m: tags,\n\u001b[1;32m    379\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m: metadata,\n\u001b[1;32m    380\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrun_name\u001b[39m\u001b[38;5;124m\"\u001b[39m: run_name,\n\u001b[1;32m    381\u001b[0m }\n\u001b[0;32m--> 383\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    384\u001b[0m \u001b[43m    \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    385\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcast\u001b[49m\u001b[43m(\u001b[49m\u001b[43mRunnableConfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[43mk\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitems\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mv\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mis\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mnot\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    386\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_only_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_only_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    387\u001b[0m \u001b[43m    \u001b[49m\u001b[43minclude_run_info\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minclude_run_info\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    388\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/chains/base.py:166\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    165\u001b[0m     run_manager\u001b[38;5;241m.\u001b[39mon_chain_error(e)\n\u001b[0;32m--> 166\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m    167\u001b[0m run_manager\u001b[38;5;241m.\u001b[39mon_chain_end(outputs)\n\u001b[1;32m    169\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m include_run_info:\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/chains/base.py:156\u001b[0m, in \u001b[0;36mChain.invoke\u001b[0;34m(self, input, config, **kwargs)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    154\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_validate_inputs(inputs)\n\u001b[1;32m    155\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m--> 156\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    157\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m new_arg_supported\n\u001b[1;32m    158\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call(inputs)\n\u001b[1;32m    159\u001b[0m     )\n\u001b[1;32m    161\u001b[0m     final_outputs: Dict[\u001b[38;5;28mstr\u001b[39m, Any] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprep_outputs(\n\u001b[1;32m    162\u001b[0m         inputs, outputs, return_only_outputs\n\u001b[1;32m    163\u001b[0m     )\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/agents/agent.py:1433\u001b[0m, in \u001b[0;36mAgentExecutor._call\u001b[0;34m(self, inputs, run_manager)\u001b[0m\n\u001b[1;32m   1431\u001b[0m \u001b[38;5;66;03m# We now enter the agent loop (until it returns something).\u001b[39;00m\n\u001b[1;32m   1432\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_should_continue(iterations, time_elapsed):\n\u001b[0;32m-> 1433\u001b[0m     next_step_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_take_next_step\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1434\u001b[0m \u001b[43m        \u001b[49m\u001b[43mname_to_tool_map\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1435\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcolor_mapping\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1436\u001b[0m \u001b[43m        \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1437\u001b[0m \u001b[43m        \u001b[49m\u001b[43mintermediate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1438\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrun_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun_manager\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1439\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1440\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(next_step_output, AgentFinish):\n\u001b[1;32m   1441\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_return(\n\u001b[1;32m   1442\u001b[0m             next_step_output, intermediate_steps, run_manager\u001b[38;5;241m=\u001b[39mrun_manager\n\u001b[1;32m   1443\u001b[0m         )\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/agents/agent.py:1139\u001b[0m, in \u001b[0;36mAgentExecutor._take_next_step\u001b[0;34m(self, name_to_tool_map, color_mapping, inputs, intermediate_steps, run_manager)\u001b[0m\n\u001b[1;32m   1130\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_take_next_step\u001b[39m(\n\u001b[1;32m   1131\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   1132\u001b[0m     name_to_tool_map: Dict[\u001b[38;5;28mstr\u001b[39m, BaseTool],\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1136\u001b[0m     run_manager: Optional[CallbackManagerForChainRun] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   1137\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[AgentFinish, List[Tuple[AgentAction, \u001b[38;5;28mstr\u001b[39m]]]:\n\u001b[1;32m   1138\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_consume_next_step(\n\u001b[0;32m-> 1139\u001b[0m         [\n\u001b[1;32m   1140\u001b[0m             a\n\u001b[1;32m   1141\u001b[0m             \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_iter_next_step(\n\u001b[1;32m   1142\u001b[0m                 name_to_tool_map,\n\u001b[1;32m   1143\u001b[0m                 color_mapping,\n\u001b[1;32m   1144\u001b[0m                 inputs,\n\u001b[1;32m   1145\u001b[0m                 intermediate_steps,\n\u001b[1;32m   1146\u001b[0m                 run_manager,\n\u001b[1;32m   1147\u001b[0m             )\n\u001b[1;32m   1148\u001b[0m         ]\n\u001b[1;32m   1149\u001b[0m     )\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/agents/agent.py:1139\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1130\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_take_next_step\u001b[39m(\n\u001b[1;32m   1131\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   1132\u001b[0m     name_to_tool_map: Dict[\u001b[38;5;28mstr\u001b[39m, BaseTool],\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1136\u001b[0m     run_manager: Optional[CallbackManagerForChainRun] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   1137\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[AgentFinish, List[Tuple[AgentAction, \u001b[38;5;28mstr\u001b[39m]]]:\n\u001b[1;32m   1138\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_consume_next_step(\n\u001b[0;32m-> 1139\u001b[0m         [\n\u001b[1;32m   1140\u001b[0m             a\n\u001b[1;32m   1141\u001b[0m             \u001b[38;5;28;01mfor\u001b[39;00m a \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_iter_next_step(\n\u001b[1;32m   1142\u001b[0m                 name_to_tool_map,\n\u001b[1;32m   1143\u001b[0m                 color_mapping,\n\u001b[1;32m   1144\u001b[0m                 inputs,\n\u001b[1;32m   1145\u001b[0m                 intermediate_steps,\n\u001b[1;32m   1146\u001b[0m                 run_manager,\n\u001b[1;32m   1147\u001b[0m             )\n\u001b[1;32m   1148\u001b[0m         ]\n\u001b[1;32m   1149\u001b[0m     )\n",
+      "File \u001b[0;32m/data1/env/py310helperbot/lib/python3.10/site-packages/langchain/agents/agent.py:1178\u001b[0m, in \u001b[0;36mAgentExecutor._iter_next_step\u001b[0;34m(self, name_to_tool_map, color_mapping, inputs, intermediate_steps, run_manager)\u001b[0m\n\u001b[1;32m   1176\u001b[0m     raise_error \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m   1177\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raise_error:\n\u001b[0;32m-> 1178\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1179\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAn output parsing error occurred. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1180\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIn order to pass this error back to the agent and have it try \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1181\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124magain, pass `handle_parsing_errors=True` to the AgentExecutor. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1182\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThis is the error: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(e)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1183\u001b[0m     )\n\u001b[1;32m   1184\u001b[0m text \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(e)\n\u001b[1;32m   1185\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandle_parsing_errors, \u001b[38;5;28mbool\u001b[39m):\n",
+      "\u001b[0;31mValueError\u001b[0m: An output parsing error occurred. In order to pass this error back to the agent and have it try again, pass `handle_parsing_errors=True` to the AgentExecutor. This is the error: Could not parse output: No."
+     ]
+    }
+   ],
+   "source": [
+    "from langchain.agents import AgentType, Tool, initialize_agent\n",
+    "\n",
+    "instructions = \"\"\"As a helpful and rigorous retail analyst, given the provided query and a list of search results for the query,  your task is to first use store name and address to identify relevant information. After that, from the relevant information, extract `store_name`, `address`, `description`, `category`, `provide_alcohol` and `phone_number` from the found relevant information. Note that `category` can only be `小吃店`, `日式料理(含居酒屋，串燒)`, `火(鍋／爐)`, `東南亞料理(不含日韓)`, `海鮮熱炒`,  `特色餐廳(含雞、鵝、牛、羊肉)`, `傳統餐廳`, `燒烤`, `韓式料理(含火鍋，烤肉)`, `西餐廳(含美式，義式，墨式)`, `西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)`, `西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)` or `早餐`. According to our experience,`provide_alcohol` can be inferred based on whether a store is suitable for  scenarios such as Gathering, Chill, Enjoying Together, Drinking Freely, Winery, Wine Cellar, Wine Storage, Relaxing, Unwinding, Lyrical, Romantic, Pleasant, Stress Relief, Wine and Dine, Light Drinking Gatherings, Birthday Celebrations, Socializing, Parties, Networking, After Work Relaxation with a Drink, Relaxing Places Suitable for Drinking, Every Dish Goes Well with Beer, Shared Dishes, Dining Together, Atmosphere Suitable for Celebratory Drinking, Places Suitable for Light Drinking Gatherings with Friends, Small Shops Suitable for Relaxing and Light Drinking. `description` is a summary of key piece of evidence and reasons that lead you decide `category` and `provide_alcohol` .\n",
+    "\n",
+    "            It's very important to omit unrelated results. Do not make up any assumption.\n",
+    "            Please think step by step, and output a single json that starts with `{` and ends with `}`. An example output json is like {\"store_name\": \"...\", \"address\": \"...\", \"description\": \"... products, service or highlights ...\", \"category\": \"...\", \"phone_number\": \"...\", \"provide_alcohol\": true or false}\n",
+    "            If no relevant information has been found, simply output json with empty values.\"\"\"\n",
+    "base_prompt = hub.pull(\"langchain-ai/openai-functions-template\")\n",
+    "prompt = base_prompt.partial(instructions=instructions)\n",
+    "llm = ChatOpenAI(temperature=0, )\n",
+    "serp = SerpAPIWrapper() # utilities\n",
+    "# serp.results(\"五角味食堂\t彰化縣北斗鎮文昌里復興路臨100-1號\")\n",
+    "tools = [\n",
+    "    Tool(\n",
+    "        name=\"Intermediate Answer\",\n",
+    "        func=serp.run,\n",
+    "        description=\"useful for when you need to get factual information with search\",\n",
+    "    )\n",
+    "]\n",
+    "self_ask_with_search = initialize_agent(\n",
+    "    tools, llm, agent=AgentType.SELF_ASK_WITH_SEARCH, verbose=True\n",
+    ")\n",
+    "self_ask_with_search.run(\"五角味食堂\t彰化縣北斗鎮文昌里復興路臨100-1號\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68b499ad-3014-4d52-a30a-d750980d030e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3a4c34ba-7af8-4e63-aa82-946cb27651ec",
+   "metadata": {},
+   "source": [
+    "### LCEL ###"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "03543ce2-1406-4534-8071-12e172b770a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass()\n",
+    "\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "model = ChatOpenAI(model=\"gpt-4\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3980df79-f7b2-465f-9c88-d6bf401eb1e2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4505be03-1225-41bf-a62b-9bf29784a24c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e48659bc-d39c-425e-a452-39f0c6e16428",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "72723ffe-4b96-4bfb-b9ae-9718b3da4ea6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pipeline.py

@@ -0,0 +1,699 @@
+import os
+import sys
+import time
+import json
+import joblib
+import math
+import itertools 
+import argparse
+import multiprocessing as mp
+from typing import List
+from pathlib import Path
+
+import jinja2
+import requests
+import pandas as pd
+from dotenv import load_dotenv
+from serpapi import GoogleSearch
+import tiktoken
+from openai import OpenAI
+from tqdm import tqdm
+from loguru import logger
+
+from model import llm
+from data import get_leads, format_search_results
+from utils import (parse_json_garbage, split_dataframe, merge_results, 
+                   combine_results, split_dict, format_df, 
+                   clean_quotes, compose_query)
+from batch import postprocess_result
+
+load_dotenv()
+ORGANIZATION_ID = os.getenv('OPENAI_ORGANIZATION_ID')
+SERP_API_KEY = os.getenv('SERP_APIKEY')
+SERPER_API_KEY = os.getenv('SERPER_API_KEY')
+
+try:
+    logger.remove(0)
+    logger.add(sys.stderr, level="INFO")
+except ValueError:
+    pass
+
+def get_serp( query: str, google_domain: str, gl: str, lr: str, top_k: int = 20, hl: str = "zh-tw", location: str = 'Taiwan', provider: str = 'serp') -> dict:
+    """
+    """
+    results = []
+
+    if provider == 'serp':
+        search = GoogleSearch({
+            "q": query, 
+            'google_domain': google_domain,
+            'gl': gl,
+            'lr': lr,
+            "api_key": SERP_API_KEY
+        })
+        result = search.get_dict()
+        # print(result['organic_results'][0])
+        # return result['organic_results'][0]
+        return result
+    elif provider == 'serper':
+        try:
+            payload = json.dumps({
+                "q": query,
+                "location": "Taiwan",
+                "gl": gl,
+                "hl": hl,
+                "num": top_k,
+                "autocorrect": False
+            })
+            response = requests.request(
+                "POST", 
+                "https://google.serper.dev/search", 
+                headers = { 'X-API-KEY': SERPER_API_KEY, 'Content-Type': 'application/json'}, 
+                data = payload
+            )
+        except Exception as e:
+            logger.error()
+            raise Exception(f"SERPER error -> {e}, query -> {query}")
+        result = response.json()
+        if 'searchParamters' in result:
+            result['search_parameters'] = result.pop('searchParamters')
+        if 'knowledgeGraph' in result:
+            result['knowledge_graph'] = result.pop('knowledgeGraph')
+        if 'organic' in result:
+            result['organic_results'] = result.pop('organic')
+        return result
+    else:
+        raise Exception(f"Unknown provider: {provider}")
+
+def get_condensed_result(result: dict):
+    """
+    Argument
+        result
+    Return
+        condensed_result:
+    Example:
+        
+    """
+    filtered_results = [ 
+        {"title": r.get('title',""), 'snippet': r.get('snippet',"")} for r in result['organic_results']
+    ]
+    
+    condensed_result = json.dumps(filtered_results, ensure_ascii=False)
+    # print( condensed_results )
+    return condensed_result
+
+def get_googlemap_results(result: dict):
+    """Get a store's google map results (in knowledge_graph)
+    Argument
+        result: dict
+            - 'knowledge_graph'
+                - 'title', 'thumbnail', 'type', 'entity_type', 'kgmid', 'knowledge_graph_search_link', 'serpapi_knowledge_graph_search_link', 'tabs', 'place_id', 'directions', 'local_map', 'rating', 'review_count', '服務項目', '地址', '地址_links', 'raw_hours', 'hours', '電話號碼', '電話號碼_links', 'popular_times', 'user_reviews', 'reviews_from_the_web', 'unclaimed_listing', '個人資料', '其他人也搜尋了以下項目', '其他人也搜尋了以下項目_link', '其他人也搜尋了以下項目_stick'
+    Return
+        googlemap_result: str
+    """
+    googlemap_result = "## Google map results\n"
+    if 'knowledge_graph' in result:
+        if 'user_reviews' in result['knowledge_graph']:
+            user_review = "\t".join([ _.get('summary', '') for _ in result['knowledge_graph']['user_reviews']])
+            store_name = result['knowledge_graph']['title']
+            googlemap_result += ( f"### store name: {store_name}\n")
+            googlemap_result += ( f"\t- 顧客評價: {user_review}\n")
+        if '其他人也搜尋了以下項目' in result['knowledge_graph']:
+            similar_store_types = "\t".join([ str(_.get('extensions', '')) for _ in result['knowledge_graph']['其他人也搜尋了以下項目']])
+            googlemap_result += ( f"\t- 類似店面類型: {similar_store_types}\n")
+        if '暫停營業' in result['knowledge_graph']:
+            store_status = '暫停營業' if result['knowledge_graph']['暫停營業'] else '營業中'
+            googlemap_result += ( f"\t- 營業狀態: {store_status}\n")
+        if '電話號碼' in result['knowledge_graph']:
+            phone_number = result['knowledge_graph']['電話號碼']
+            googlemap_result += ( f"\t- 電話號碼: {phone_number}\n")
+        if 'type' in result['knowledge_graph']:
+            store_type = result['knowledge_graph']['type']
+            googlemap_result += ( f"\t- 餐飲屬性: {store_type}\n")
+
+    else:
+        googlemap_result += ("empty\n")
+    return clean_quotes(googlemap_result)
+
+def get_organic_result(result: dict) -> str:
+    """Get a store's organic search results
+    Argument
+        result: str
+    Return
+        organic_result: str
+    """
+    organic_result = "## Search results\n"
+    # filtered_results = [ 
+    #     {"title": r.get('title',""), 'snippet': r.get('snippet',"")} for r in result['organic_results']
+    # ]
+    for r in result['organic_results']:
+        organic_result += ( f"### {r.get('title','')}: {r.get('snippet','')}\n")
+    return clean_quotes(organic_result)
+
+def compose_classification(  user_content, config: dict) -> str:
+    """
+    Argument
+        user_content: str
+        config: dict
+            classes: list
+            backup_classes: list
+            provider: e.g. 'google', 'openai'
+            model: e.g. 'gemini-1.5-flash', 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview'
+    Return
+        response: str
+    Example
+        system_prompt = 
+            As a helpful and rigorous retail analyst, given the provided information about a store, 
+            your task is two-fold. First, classify provided evidence below into the mostly relevant category from the following: {classes}. 
+            Second, if no relevant information has been found, classify the evidence into the mostly relevant supercategory from the following: {backup_classes}.
+            It's very important to omit unrelated piece of evidence and don't make up any assumption.
+            Please think step by step, and must output in json format. An example output json is like {{"category": "..."}}
+            If no relevant piece of information can ever be found at all, simply output json with empty string "".
+            I'll tip you and guarantee a place in heaven you do a great job completely according to my instruction.
+    """
+
+    if isinstance(classes, list):
+        classes = ", ".join([ f"`{x}`" for x in classes])
+    elif isinstance(classes, str):
+        pass
+    else:
+        raise Exception(f"Incorrect classes type: {type(classes)}")
+    template = jinja2.Environment().from_string(config['classification_prompt'])
+    system_prompt = template.render( classes=config['classes'], backup_classes=config['backup_classes'])
+    response = llm( 
+        provider = config['provider'], 
+        model = config['model'], 
+        system_prompt = system_prompt,
+        user_content = user_content,
+    )
+    return response
+
+def classify_results( 
+        analysis_results: pd.DataFrame, 
+        config: dict,
+        input_column: str = 'evidence', 
+        output_column: str = 'classified_category',
+    ):
+    """Classify the results
+    Argument
+        analysis_results: dataframe
+        config: dict
+            classes: list,
+            backup_classes: list,
+            provider: str, 
+            model: str,
+        input_column: str
+        output_column: str
+    Return 
+        analysis_results: dataframe
+    """
+    classified_results = analysis_results.copy()
+    labels, empty_indices = [], []
+    for idx, evidence in zip( analysis_results['index'], analysis_results[input_column]):
+        try:
+            user_content = f'''`evidence`: `{evidence}`''' 
+            pred_cls = compose_classification( user_content, config)
+            label = parse_json_garbage(pred_cls)['category']
+            labels.append(label)
+        except Exception as e:
+            logger.error(f"# CLASSIFICATION error: e -> {e}, user_content -> {user_content}, evidence: {evidence}")
+            labels.append("")
+            empty_indices.append(idx)
+
+    classified_results[output_column] = labels
+    return { 
+        "classified_results": classified_results, 
+        "empty_indices": empty_indices
+    }
+            
+def classify_results_mp( extracted_results: pd.DataFrame, classified_file_path: str, config: dict, n_processes: int = 4):
+    """
+    Argument
+        extracted_results:
+        classified_file_path:
+        config: dict
+            classes: list
+            backup_classes: list
+            provider: str
+            model: str, 
+        n_processes: int
+    Return
+        classified_results: dataframe
+    Reference
+        200 records, 4 processes, 122.4695s
+    """
+    st = time.time()
+    # classified_file_path = "data/classified_result.joblib"
+    if not os.path.exists(classified_file_path):
+        split_data = split_dataframe(extracted_results)
+        with mp.Pool(args.n_processes) as pool:
+            classified_results = pool.starmap( 
+                classify_results,
+                [ ( 
+                    d, config, 'evidence', 'classified_category'
+                ) for d in split_data]
+            ) 
+            classified_results = merge_results( classified_results, dataframe_columns=['classified_results'], list_columns=['empty_indices'])
+            try:
+                with open( classified_file_path, "wb") as f:
+                    joblib.dump( classified_results, f)
+            except FileNotFoundError as e:
+                logger.error(f"# CLASSIFICATION error: e -> {e}")
+                with open( f"./{Path(classified_file_path).name}.joblib", "wb") as f:
+                    joblib.dump( classified_results, f)
+            
+    else:
+        with open( classified_file_path, "rb") as f:
+            classified_results = joblib.load(f)
+    print( f"total time: {time.time() - st}")
+    return classified_results
+
+def compose_filter( query, search_results, config: dict):
+    """Filter the search results based on the query (store name and address)
+    Argument
+        query: str
+        search_results: str
+        system_prompt: str
+        config: dict
+            provider: default to be "google"
+            model: default to be "gemini-1.5-flash" 
+    Return
+        response: str
+    """
+    system_prompt = f'''As a helpful and rigorous retail analyst, given the provided query and a list of search results for the query,  execute this task step by step. Search results contains a list of search results, where some could be totally irrelevant to our query.
+Steps: 
+First, use store name and address to identify relevant and irrelevant information from search results.
+Second, look through the list of search results; keep relevant ones and drop irrelevant ones.
+Third, filter the results and only output relevant ones. Original numbering must be kept.
+
+Output in json format such as {{ "relevant_results": [ "result ...", "result ...", "result ..." ], "irrelevant_results": ["result ..."]}}. It's very important to omit unrelated results. Do not make up any assumption.
+        '''
+    user_content = f"## query: `{query}`\n## search_results: {search_results}"
+    response = llm( 
+        provider = config['provider'], 
+        model = config['model'], 
+        system_prompt = system_prompt, 
+        user_content = user_content
+    )
+    return response
+
+def filter_results(  results: pd.DataFrame, config: dict):
+    """Filter the results
+    Argument
+        results: dataframe
+        config: dict
+            provider
+            model:
+    Return 
+        analysis_results: dataframe
+    """
+    results = results.copy()
+    relevant_results, empty_indices = [], []
+    for i, d in tqdm(enumerate(results.itertuples())):
+        idx = d.index # d[1]
+        search_results = d.search_results
+        # evidence = d.googlemap_results +"\n" + d.search_results
+        # business_id = d.business_id # d[2]
+        business_name = d.business_name # d[3]
+        address = d.address # d[7]
+        try:
+            query = compose_query( address, business_name, use_exclude=False)
+            filtered_results = compose_filter( query = query, search_results=search_results, config=config)
+            relevant_result = parse_json_garbage(filtered_results)['relevant_results']
+            relevant_result = "### 搜尋結果: \n" + "\n".join([ "- " + r for r in relevant_result])
+            relevant_results.append(relevant_result)
+        except Exception as e:
+            logger.error(f"# FILTER error (add to empty_indices): e -> {e}, query -> {query}, filtered_results: {filtered_results}")
+            relevant_results.append(search_results)
+            empty_indices.append(idx)
+
+    results.loc[ :, "search_results"] = relevant_results
+    return { 
+        "filtered_results": results, 
+        "empty_indices": empty_indices
+    }
+
+def filter_results_mp( data: pd.DataFrame, filtered_file_path: str, config: dict, n_processes: int = 4):
+    """Filter results in parallel
+    Argument
+        data: dataframe
+        filtered_file_path: str
+        config: dict
+            provider: str
+            model: str
+        n_processes: int
+    Return
+        filtered_results: dataframe
+    """
+    st = time.time()
+    # crawl_file_path = "data/crawled_results.joblib"
+    if not os.path.exists(filtered_file_path):
+        split_data = split_dataframe( data )
+        with mp.Pool(n_processes) as pool:
+            filtered_results = pool.starmap( 
+                filter_results, 
+                [ (d, config) for d in split_data]
+            )
+            filtered_results = merge_results( filtered_results, dataframe_columns=['filtered_results'], list_columns=['empty_indices'])
+            # with open( filtered_file_path, "wb") as f:
+            #     joblib.dump( filtered_results, f)
+            filtered_results['filtered_results'].to_csv( filtered_file_path, index=False)
+    else:
+        # with open( filtered_file_path, "rb") as f:
+        #     filtered_results = joblib.load(f)
+        filtered_results = { 'filtered_results': pd.read_csv( filtered_file_path)}
+    logger.debug( f"total time: {time.time() - st}")
+    return filtered_results
+
+def crawl_results( data: pd.DataFrame, serp_provider: str = 'serp', google_domain: str = 'google.com.tw', gl: str = 'tw', lr: str = 'lang_zh-TW'):
+    """
+    Argument
+        data: dataframe
+        google_domain: str
+        gl: str
+        lr: str
+    Return
+        {
+            `crawled_results`: df
+            `empty_indices`: list
+        }
+        df format:
+        [
+            {'title': '達米娜魚料理食堂',
+            'snippet': 'Zhenzhen · 台東縣台東市洛陽街204號 · 08 934 1662 · 其他美食 · 外送・提供廁所・免費Wifi · ・休息中・將於20:00 開始營業 · NT$120 · 座位數15 · 現金.'},
+            ...
+            {'title': '台東美食推薦》25間台東市美食小吃/特色餐廳/早餐伴手禮',
+            'snippet': '好漁日鬼頭刀專屬料理MAHI MAHI TODAY如店名所說，是間專賣鬼頭刀料理的餐廳，台灣主要盛產鬼頭刀的地方就位於台東的成功新港漁港，所以推薦大家來台東 ...'},
+            {'title': '類似的店', 'snippet': "['餐廳']\t['早午餐']\t['餐廳']"},
+            {'status': '暫停營業'},
+            {'telephone_number': '08 934 1662'}
+        ]
+    Reference
+        200 records, 4 processes, 171.36490321159363
+    """
+    # serp_results = []
+    # condensed_results = []
+    crawled_results = []
+    empty_indices = []
+    for i, d in tqdm(enumerate(data.itertuples())):
+        idx = d[0]
+        address = d[1]
+        business_id = d[2]
+        business_name = d[4]
+        query = compose_query(address, business_name)
+        try:
+            res = get_serp( query, google_domain, gl, lr, provider=serp_provider)
+            # serp_results.append(res)
+        except:
+            logger.warning( f"# SERP error (will add to empty indices): i = {i}, idx = {idx}, query = {query}")
+            empty_indices.append(i)
+            continue
+        try:
+            # cond_res = get_condensed_result(res)
+            googlemap_res = get_googlemap_results(res)
+            search_res = get_organic_result(res)
+            # condensed_results.append(cond_res)
+        except:
+            logger.warning(f"# get googlemap & organic results error (will add to empty indices): i = {i}, idx = {idx}, res = {res}")
+            empty_indices.append(i)
+            continue
+        
+        crawled_results.append( { 
+            "index": idx, 
+            "business_id": business_id, 
+            "business_name": business_name, 
+            "serp": res,
+            # "evidence": cond_res, 
+            "googlemap_results": googlemap_res,
+            "search_results": search_res,
+            "address": address,
+        } )
+    crawled_results = pd.DataFrame(crawled_results)
+
+    return {
+        "crawled_results": crawled_results,
+        "empty_indices": empty_indices
+    }
+
+def crawl_results_mp( data: pd.DataFrame, crawl_file_path: str, serp_provider: str, n_processes: int = 4):
+    st = time.time()
+    # crawl_file_path = "data/crawled_results.joblib"
+    if not os.path.exists(crawl_file_path):
+        split_data = split_dataframe( data )
+        with mp.Pool(n_processes) as pool:
+            crawled_results = pool.starmap( 
+                crawl_results, 
+                [( d, serp_provider) for d in split_data]
+            )
+            crawled_results = merge_results( crawled_results, dataframe_columns=['crawled_results'], list_columns=['empty_indices'])
+            try:
+                with open( crawl_file_path, "wb") as f:
+                    joblib.dump( crawled_results, f)
+            except FileNotFoundError as e:
+                logger.error(f"# CRAWL error: e = {e}")
+                with open( f"./{Path(crawl_file_path).name}.joblib", "wb") as f:
+                    joblib.dump( crawled_results, f)
+    else:
+        with open( crawl_file_path, "rb") as f:
+            crawled_results = joblib.load(f)
+    logger.debug( f"total time: {time.time() - st}")
+    return crawled_results
+
+def compose_extraction( query, search_results, config: dict):
+    """
+    Argument
+        query: str
+        search_results: str
+        config: dict
+            system_prompt: str
+            classes: list. e.g. `小吃店`,`日式料理(含居酒屋，串燒)`,`火(鍋／爐)`,`東南亞料理(不含日韓)`,`海鮮熱炒`,`特色餐廳(含雞、鵝、牛、羊肉)`,`釣蝦場`,`傳統餐廳`,`燒烤`,`韓式料理(含火鍋，烤肉)`,`PUB(Live Band)`,`PUB(一般，含Lounge)`,`PUB(電音＼舞場)`,`五星級飯店`,`自助KTV(含連鎖，庭園自助)`,`西餐廳(含美式，義式，墨式)`,`咖啡廳(泡沫紅茶)`,`飯店(星級／旅館，不含五星級)`,`運動休閒館(含球類練習場，飛鏢等)`,`西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)`,`西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)`,`早餐`
+            provider: "openai"
+            model: "gpt-4-0125-preview" or 'gpt-3.5-turbo-0125'
+    Return
+        response: str
+
+    Example
+        classes = ", ".join([ "`"+x+"`" for x in classes if x!='早餐' ])+ " or " + "`早餐`"
+        traits = "Gathering, Chill, Enjoying Together, Drinking Freely, Winery, Wine Cellar, Wine Storage, Relaxing, Unwinding, Lyrical, Romantic, Pleasant, Stress Relief, Wine and Dine, Light Drinking Gatherings, Birthday Celebrations, Socializing, Parties, Networking, After Work Relaxation with a Drink, Relaxing Places Suitable for Drinking, Every Dish Goes Well with Beer, Shared Dishes, Dining Together, Atmosphere Suitable for Celebratory Drinking, Places Suitable for Light Drinking Gatherings with Friends, Small Shops Suitable for Relaxing and Light Drinking"
+        system_prompt = f'''
+            As a helpful and rigorous retail analyst, given the provided query and a list of search results for the query,  your task is to first use store name and address to identify relevant information. 
+            After that, from the relevant information, extract `store_name`, `address`, `description`, `category`, `provide_alcohol` and `phone_number` from the found relevant information. 
+            Note that `category` can only be {classes}. 
+            According to our experience,`provide_alcohol` can be inferred based on whether a store is suitable for scenarios such as {traits}. 
+            `description` is a summary of key piece of evidence and reasons that lead you decide `category` and `provide_alcohol` .
+
+            It's very important to omit unrelated results. Do not make up any assumption.
+            Please think step by step, and output a single json that starts with `{{` and ends with `}}`. An example output json is like {{"store_name": "...", "address": "...", "description": "... products, service or highlights ...", "category": "...", "phone_number": "...", "provide_alcohol": true or false}}
+            If no relevant information has been found, simply output json with empty values.
+        '''
+    """
+    classes = ", ".join([ "`"+x+"`" for x in config['classes'] if x!='早餐' ])+ " or " + "`早餐`"
+    traits = config['traits']
+    system_prompt = config['extraction_prompt']
+    jenv = jinja2.Environment()
+    template = jenv.from_string(system_prompt)
+    system_prompt = template.render( classes=classes, traits=traits)
+    user_content = f"`query`: `{query}`\n`search_results`: {search_results}"
+    response = llm( 
+        provider = config['provider'], 
+        model = config['model'], 
+        system_prompt = system_prompt, 
+        user_content = user_content
+    )
+    return response
+
+def extract_results( data: pd.DataFrame, config: dict):
+    """
+    Argument
+        data: a dataframe 
+            - "index", "business_id", "business_name", "serp", "googlemap_results", "search_results", "address"
+            # - `evidence`, `result`
+        config: dict
+            classes: list
+            provider: str
+            model: str
+    Return
+        extracted_results: dataframe of `extracted_evidence`
+    """
+    extracted_results, empty_indices, ext_res = [], [], []
+    for i, d in tqdm(enumerate(data.itertuples())):
+        idx = d.index # d[1]
+        # evidence = d.evidence
+        # evidence = d.formatted_evidence
+        evidence = d.googlemap_results +"\n" + d.search_results
+        business_id = d.business_id # d[2]
+        business_name = d.business_name # d[3]
+        address = d.address # d[7]
+        ana_res = None
+        query = compose_query( address, business_name, use_exclude=False)
+        try:
+            ext_res = compose_extraction(
+                query = query, 
+                search_results = evidence, 
+                config = config
+            )
+            ext_res = parse_json_garbage(ext_res)
+        except Exception as e:
+            logger.error(f"# ANALYSIS error (add to empty indices): e = {e}, i = {i}, q = {query}, ext_res = {ext_res}")
+            empty_indices.append(i)
+            continue
+        
+        extracted_results.append( { 
+            "index": idx, 
+            "business_id": business_id, 
+            "business_name": business_name, 
+            "evidence": evidence, 
+            ** ext_res
+        } )
+    extracted_results = pd.DataFrame(extracted_results)
+
+    return {
+        "extracted_results": extracted_results, 
+        "empty_indices": empty_indices
+    }
+
+def extract_results_mp( crawled_results, extracted_file_path, config: dict, n_processes: int = 4):
+    """
+    Argument
+        crawled_results: dataframe
+        extracted_file_path
+        config:
+            classes: list
+            model: str
+            provider: str
+    Return 
+    Reference
+        200 records, 4 processes, 502.26914715766907
+    """
+    st = time.time()
+    # args.extracted_file_path = "data/extracted_results.joblib"
+    if not os.path.exists(extracted_file_path):
+        split_data = split_dataframe( crawled_results)
+        with mp.Pool(n_processes) as pool:
+            extracted_results = pool.starmap( extract_results, [ (x, config) for x in split_data])
+            extracted_results = merge_results( extracted_results, dataframe_columns=['extracted_results'], list_columns=['empty_indices'])
+            try:
+                with open( extracted_file_path, "wb") as f:
+                    joblib.dump( extracted_results, f)
+            except FileNotFoundError as e:
+                logger.error(f"# EXTRACT error: e = {e}")
+                with open( f"./{Path(extracted_file_path).name}.joblib", "wb") as f:
+                    joblib.dump( extracted_results, f)
+    else:
+        with open( extracted_file_path, "rb") as f:
+            extracted_results = joblib.load(f)
+    logger.info( f"total time: {time.time() - st}")
+    return extracted_results
+
+def compose_regularization( category: str, config: dict):
+    """
+    Argument
+        category: str
+        config: dict
+            provider: str
+            model: str
+    Return
+        response: str
+    """
+    system_prompt = f"""
+    As a helpful and factual assistant, your task is to classify the provided raw cuisine category into a conformed category. The definition of each conformed category is show below (in the format of `category`: `... definition ...`):
+- `小吃店`:小吃、擔仔麵、小吃攤、街邊小店、傳統小吃、麵食、麵攤、炒飯、餃子館、鯊魚煙、黑白切、牛肉麵、銅板美食、小點心、簡餐、色小菜、開放空間攤販
+- `日式料理(含居酒屋，串燒)`:居酒屋、酒場、水產、清酒、生魚片、壽司、日式啤酒、日式料理、代烤服務、日本餐飲場所、日本傳統食物、日式定食
+- `火(鍋／爐)`:麻辣鍋、薑母鴨、鴨味仔、鍋物、湯底、滋補、冬令補、涮涮鍋、個人鍋、冬天圍爐、羊肉爐、鴛鴦鍋、炭火爐、氣火爐、燒酒雞、蒸氣海鮮鍋
+- `東南亞料理(不含日韓)`:印尼、越式、泰式、沙嗲、海南雞、河粉、馬來西亞料理、新加坡料理、寮國料理、緬甸料理、南洋風味、印度料理、越南春捲、泰式綠咖哩、異國風情裝潢、滇緬料理
+- `海鮮熱炒`:海鮮、現撈、活海鮮、生猛、大排檔、活魚活蝦、生猛海鮮、快炒、海產、台式海鮮、下酒菜
+- `特色餐廳(含雞、鵝、牛、羊肉)`:烤鴨、燒鵝、甕仔雞、甕缸雞、桶仔雞、牛雜、蒙古烤肉、鵝肉城、金山鴨肉、生牛肉、全羊宴、活鱉、烤雞店、鵝肉餐廳、溫體牛、現宰羊肉、鹹水鵝、土羊肉
+- `傳統餐廳`:江浙、台菜、合菜、桌菜、粵菜、中式、川菜、港式、上海菜、砂鍋魚頭、東北菜、北京烤鴨、一鴨三吃、婚宴、辦桌、老字號、宴會廳、台灣料理
+- `燒烤`:燒烤、串燒、串串、烤魚、鮮蚵、炭烤、直火、碳火、和牛、戶外生火、烤肉、路邊燒烤
+- `韓式料理(含火鍋，烤肉)`:韓國泡菜、韓式年糕、首爾、燒酒、韓式炸雞、春川辣炒雞、韓式炸醬麵、海鮮煎餅、烤三層肉、烤五花、烤韓牛、醬料和飯、石鍋拌飯、韓式風格、韓式清酒、啤酒、銅盤烤肉、韓流
+- `PUB(Live Band)`:音樂餐廳、樂團表演、現場表演、LIVE表演、樂團駐唱、定期表演、有舞台場地、樂隊、專人駐唱
+- `PUB(一般，含Lounge)`:酒吧、bar、lounge、飛鏢、調酒、運動酒吧、音樂酒吧、沙發聊天、女公關、互動調酒師、公關服務
+- `PUB(電音＼舞場)`:夜店、舞池電音、藝人、包廂低消制、電子音樂表演、DJ、派對狂歡
+- `五星級飯店`:高級飯店、奢華酒店、連鎖五星級飯店、國際集團飯店、米其林飯店、高檔住宿
+- `自助KTV(含連鎖，庭園自助)`:卡拉OK、唱歌、歌坊、歡唱吧、自行點歌、自助唱歌、唱歌包廂、慶生聯誼包廂
+- `西餐廳(含美式，義式，墨式)`:牛排、餐酒、歐式、義式、西餐、義大利麵、凱薩沙拉、紅酒、白酒、調酒、墨西哥式料理、阿根廷式料理、漢堡、比薩
+- `咖啡廳(泡沫紅茶)`:泡沫紅茶店、咖啡店、café、coffee、輕食、軟性飲料、簡餐、茶街
+- `飯店(星級／旅館，不含五星級)`:飯店、酒店、商務旅館、平價住宿
+- `運動休閒館(含球類練習場，飛鏢等)`:撞球、高爾夫、運動、保齡球、娛樂、高爾夫練習場、大魯閣棒球場、籃球、羽毛球、PHOENIX鳳凰、羽球館、看球賽
+- `釣蝦場`:釣蝦、蝦寶、投幣卡拉OK、釣竿和餌料、蝦子現場烹煮食用、泰國蝦、現烤蝦子、包廂唱歌、現釣現烤、自備或租用釣竿。
+
+Note that you must choose from the above categories. Other ones are strongly prohibited. 
+
+Output in json format such as {{"category": "..."}}.
+
+    """
+    user_content = category
+    response = llm( 
+        provider = config['provider'], 
+        model = config['model'], 
+        system_prompt = system_prompt, 
+        user_content = user_content
+    )
+    return response
+
+def regularize_results( results: pd.DataFrame, provider, model):
+    """Regularize the categories
+    Argument
+        results: dataframe
+        provider: str
+        model: str
+    Return
+        a dict of 
+            - regularized_results: dataframe
+            - empty_indices: list
+    """
+    results = results.copy()
+    regular_categories, empty_indices = [], []
+    for i, d in tqdm(enumerate(results.itertuples())):
+        idx = d.index # d[1]
+        category = d.category
+        if pd.isna(category) or len(category)==0:
+            regular_categories.append("")
+            continue
+        try:
+            query = category
+            regularized_result = compose_regularization(  category, provider=provider, model=model)
+            regular_category = parse_json_garbage(regularized_result)['category']
+            regular_categories.append(regular_category)
+        except Exception as e:
+            logger.error(f"# REGULARIZATION error (add to empty_indices): e -> {e}, query -> {query}, category: {category}")
+            regular_categories.append(category)
+            empty_indices.append(idx)
+
+    results.loc[ :, "category"] = regular_categories
+    return { 
+        "regularized_results": results, 
+        "empty_indices": empty_indices
+    }
+
+def regularize_results_mp( data: pd.DataFrame, regularized_file_path, provider, model):
+    """Regularize categories in parallel
+    Argument
+        data: dataframe
+        regularized_file_path: str
+        provider: str
+        model: str
+    Return
+        regularized_results: dataframe
+    """
+    st = time.time()
+    if not os.path.exists(regularized_file_path):
+        split_data = split_dataframe( data )
+        with mp.Pool(n_processes) as pool:
+            regularized_results = pool.starmap( 
+                regularize_results, 
+                [ ( 
+                    d, provider, model
+                ) for d in split_data]
+            )
+            regularized_results = merge_results( regularized_results, dataframe_columns=['regularized_results'], list_columns=['empty_indices'])
+            # with open( filtered_file_path, "wb") as f:
+            #     joblib.dump( filtered_results, f)
+            regularized_results['regularized_results'].to_csv( regularized_file_path, index=False)
+    else:
+        # with open( filtered_file_path, "rb") as f:
+        #     filtered_results = joblib.load(f)
+        regularized_results = { 'regularized_results': pd.read_csv( regularized_file_path)}
+    logger.debug( f"total time: {time.time() - st}")
+    return regularized_results
+

requirements.txt

@@ -17,4 +17,5 @@ tiktoken
 tqdm
 joblib
 google-search-results
-anthropic>=0.25.7,<0.26.0
+anthropic>=0.25.7,<0.26.0
+google-generativeai>=0.7.1,<0.8.0

scripts/run_batch.sh

@@ -0,0 +1,33 @@
+start_index=0
+total_files=17
+wait_time_in_seconds=10
+for i in $(seq $start_index $total_files); do
+    # data_dir="data"/"gpt-4o-mini"/"output_0402_1_篩選結果.xlsx - Sheet1_$i"
+    # data_dir="data"/"gpt-4o-mini"/"output_0402_2_篩選結果.xlsx - Sheet1_$i"
+    # data_dir="data"/"gpt-4o-mini"/"output_0402_3_篩選結果.xlsx - Sheet1_$i"
+    data_dir="data"/"gpt-4o-mini"/"output_0402_4_篩選結果.xlsx - Sheet1_$i"
+    # data_dir="data/tmp"
+
+    echo "Preparing batch data ..."
+    python batch.py -t prepare_batch \
+        -crp "$data_dir"/crawled_results.joblib \
+        -o "$data_dir"/batch.jsonl
+        # -topn 200
+
+    echo "Executing batch data ..."
+    python batch.py -t run_batch \
+        -i "$data_dir"/batch.jsonl \
+        -j "$data_dir"/job.joblib \
+        -jp "$data_dir"/output.jsonl
+
+    echo "Converting batch to extracted results ..."
+    python batch.py -t batch2extract \
+        -jp "$data_dir"/output.jsonl \
+        -crp "$data_dir"/crawled_results.joblib \
+        -erp "$data_dir"/extracted_results.joblib
+
+
+    sleep $wait_time_in_seconds
+done
+
+echo "All tasks completed."

scripts/run_combine.sh

@@ -0,0 +1,24 @@
+if [ -z $1 ]; then
+    echo "Please provide the batch index."
+    echo "Example: "
+    echo "    sh ./run_combine.sh 1"
+    echo "    sh ./run_combine.sh 2"
+    echo "    sh ./run_combine.sh 3"
+    echo "    sh ./run_combine.sh 4"
+    exit 1
+else
+    index=$1
+    data_dir="data"/"gpt-4o-mini"
+
+    pattern="output_0402_"$index"_篩選結果.xlsx - Sheet1_*"
+    python batch.py \
+        -t combine \
+        -i "$data_dir" \
+        -prp "$pattern" \
+        -rp "data/staging/output_0402_"$index"_篩選結果.xlsx - Sheet1.csv" \
+        -o "data/staging/output_0402_"$index"_篩選結果.xlsx - Sheet1.postprocessed.csv"
+
+    echo "All tasks completed."
+fi
+
+

scripts/run_crawl.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# 設定檔案的基礎名稱
+# base_file="output_0402_1_篩選結果.xlsx - Sheet1_"
+# base_file="output_0402_2_篩選結果.xlsx - Sheet1_"
+# base_file="output_0402_3_篩選結果.xlsx - Sheet1_"
+base_file="output_0402_4_篩選結果.xlsx - Sheet1_"
+
+# 設定總共要處理的檔案數量
+start_index=0
+total_files=17
+
+# 設定每次處理消耗的數量和 API 限制
+# consumption_per_run=1000
+# api_rate_limit=3000
+api_rate_limit=20000
+wait_time_in_seconds=60 # 1500 # 25 mins
+
+# 迴圈執行
+for i in $(seq $start_index $total_files); do
+    # 動態生成檔案名稱
+    file_name="${base_file}${i}.csv"
+    crawled_file_path="${base_file}${i}/crawled_results.joblib"
+    
+    # 執行 python 指令
+    python sheet.py --data_path "data/production/${file_name}" --task new \
+    --step crawl \
+    --output_dir data/gpt-4o-mini \
+    --n_processes 4 \
+    --serp_provider serp \
+    --crawled_file_path "${crawled_file_path}" \
+    --extraction_provider openai \
+    --extraction_model gpt-4o-mini \
+    --regularization_provider openai \
+    --regularization_model gpt-4o-mini
+    
+    # 等待以避免 API rate limit
+    echo "Completed task for ${file_name}. Waiting for ${wait_time_in_seconds} seconds..."
+    sleep $wait_time_in_seconds
+done
+
+echo "All tasks completed."

scripts/run_postprocess.sh

@@ -0,0 +1,25 @@
+if [ -z $1 ] | [ -z $2 ]; then
+    echo "Please provide the batch index."
+    echo "Example: "
+    echo "    sh ./run_postprocess.sh 1 25"
+    echo "    sh ./run_postprocess.sh 2 27"
+    echo "    sh ./run_postprocess.sh 3 27"
+    echo "    sh ./run_postprocess.sh 4 17"
+    exit 1
+else
+    batch_num=$1
+    start_index=0
+    total_files=$2
+    wait_time_in_seconds=1
+    for i in $(seq $start_index $total_files); do
+        # data_dir="data"/"gpt-4o-mini"/"output_0402_1_篩選結果.xlsx - Sheet1_$i"
+        data_dir="data"/"gpt-4o-mini"/"output_0402_"$batch_num"_篩選結果.xlsx - Sheet1_$i"
+        # data_dir="data"/"gpt-4o-mini"/"output_0402_3_篩選結果.xlsx - Sheet1_$i"
+        # data_dir="data"/"gpt-4o-mini"/"output_0402_4_篩選結果.xlsx - Sheet1_$i"
+        python batch.py -t postprocess \
+            -rrp "$data_dir"/regularized_results.joblib \
+            -prp "$data_dir"/postprocessed_results.csv
+    done
+
+    echo "All tasks completed."
+fi

scripts/run_regularization.sh

@@ -0,0 +1,38 @@
+start_index=0
+total_files=0
+wait_time_in_seconds=10
+for i in $(seq $start_index $total_files); do
+    # data_dir="data"/"gpt-4o-mini"/"output_0402_1_篩選結果.xlsx - Sheet1_$i"
+    # data_dir="data"/"gpt-4o-mini"/"output_0402_2_篩選結果.xlsx - Sheet1_$i"
+    data_dir="data"/"gpt-4o-mini"/"output_0402_3_篩選結果.xlsx - Sheet1_$i"
+    # data_dir="data"/"gpt-4o-mini"/"output_0402_4_篩選結果.xlsx - Sheet1_$i"
+    # data_dir="data/tmp"
+
+    # echo "Preparing batch data ..."
+    # python batch.py -t prepare_regularization \
+    #     -erp "$data_dir"/extracted_results.joblib \
+    #     -o "$data_dir"/regularization.jsonl
+    #     -topn 200
+
+    # echo "Executing batch data ..."
+    # python batch.py -t run_batch \
+    #     -i "$data_dir"/regularization.jsonl \
+    #     -j "$data_dir"/reg_job.joblib \
+    #     -jp "$data_dir"/reg_output.jsonl
+
+    # echo "Converting batch to extracted results ..."
+    # python model.py -t batch2extract \
+    #     -jp "$data_dir"/output.jsonl \
+    #     -crp "$data_dir"/crawled_results.joblib \
+    #     -erp "$data_dir"/extracted_results.joblib
+
+    echo "Converting batch to regularized results ..."
+    python batch.py -t batch2reg \
+        -jp "$data_dir"/reg_output.jsonl \
+        -erp "$data_dir"/extracted_results.joblib \
+        -rrp "$data_dir"/regularized_results.joblib
+
+    sleep $wait_time_in_seconds
+done
+
+echo "All tasks completed."

sheet.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import time
 import json
 import joblib
@@ -6,529 +7,63 @@ import math
 import itertools 
 import argparse
 import multiprocessing as mp
+from typing import List
+from pathlib import Path
 
+import yaml
+import jinja2
+import requests
 import pandas as pd
 from dotenv import load_dotenv
 from serpapi import GoogleSearch
 import tiktoken
 from openai import OpenAI
 from tqdm import tqdm
+from loguru import logger
 
 from model import llm
-from utils import parse_json_garbage
-
+from data import get_leads, format_search_results
+from utils import (parse_json_garbage, split_dataframe, merge_results, 
+                   combine_results, split_dict, format_df, 
+                   clean_quotes, compose_query, reverse_category2supercategory)
+from batch import postprocess_result
+from pipeline import (get_serp, get_condensed_result, get_organic_result, get_googlemap_results,
+                    crawl_results, crawl_results_mp, 
+                    compose_extraction, extract_results, extract_results_mp, 
+                    compose_classification, classify_results, classify_results_mp,
+                    compose_regularization, regularize_results, regularize_results_mp,
+                    compose_filter, filter_results, filter_results_mp)
 load_dotenv()
 ORGANIZATION_ID = os.getenv('OPENAI_ORGANIZATION_ID')
 SERP_API_KEY = os.getenv('SERP_APIKEY')
-
-
-def get_leads( file_path: str, names: list = ['營業地址', '統一編號', '總機構統一編號', '營業人名稱', '資本額', '設立日期', '組織別名稱', '使用統一發票',
-       '行業代號', '名稱', '行業代號1', '名稱1', '行業代號2', '名稱2', '行業代號3', '名稱3']):
-    """
-    """
-    assert os.path.exists(file_path)
-    data = pd.read_csv( file_path, names=names)
-    return data
+SERPER_API_KEY = os.getenv('SERPER_API_KEY')
+    
     
-def get_serp( query: str, google_domain: str, gl: str, lr: str) -> dict:
-    """
-    """
-    results = []
-    search = GoogleSearch({
-        "q": query, 
-        'google_domain': google_domain,
-        'gl': gl,
-        'lr': lr,
-        "api_key": SERP_API_KEY
-      })
-    result = search.get_dict()
-    # print(result['organic_results'][0])
-    # return result['organic_results'][0]
-    return result
-
-
-def get_condensed_result(result):
-    """
-    Argument
-        result
-    Return
-        condensed_result:
-    Example:
-        result['knowledge_graph'].keys() # 'title', 'thumbnail', 'type', 'entity_type', 'kgmid', 'knowledge_graph_search_link', 'serpapi_knowledge_graph_search_link', 'tabs', 'place_id', 'directions', 'local_map', 'rating', 'review_count', '服務項目', '地址', '地址_links', 'raw_hours', 'hours', '電話號碼', '電話號碼_links', 'popular_times', 'user_reviews', 'reviews_from_the_web', 'unclaimed_listing', '個人資料', '其他人也搜尋了以下項目', '其他人也搜尋了以下項目_link', '其他人也搜尋了以下項目_stick'
-    """
-    filtered_results = [ 
-        {"title": r.get('title',""), 'snippet': r.get('snippet',"")} for r in result['organic_results']
-    ]
-    if 'knowledge_graph' in result:
-        if 'user_reviews' in result['knowledge_graph']:
-            filtered_results.append( {'title': result['knowledge_graph']['title'], '顧客評價': "\t".join([ _.get('summary', '') for _ in result['knowledge_graph']['user_reviews']]) })
-        if '其他人也搜尋了以下項目' in result['knowledge_graph']:
-            filtered_results.append( {'title': "類似的店", 'snippet': "\t".join([ str(_.get('extensions', '')) for _ in result['knowledge_graph']['其他人也搜尋了以下項目']]) })
-        if '暫停營業' in result['knowledge_graph']:
-            filtered_results.append( {'status': '暫停營業' if result['knowledge_graph']['暫停營業'] else '營業中'})
-        if '電話號碼' in result['knowledge_graph']:
-            filtered_results.append( {'telephone_number': result['knowledge_graph']['電話號碼']})
-    condensed_result = json.dumps(filtered_results, ensure_ascii=False)
-    # print( condensed_results )
-    return condensed_result
-
-def compose_extraction( query, search_results, classes: list, provider: str, model: str):
-    """
-    Argument
-        query: str
-        search_results: str
-        system_prompt: str
-        classes: list, `小吃店`, `日式料理(含居酒屋，串燒)`, `火(鍋／爐)`, `東南亞料理(不含日韓)`, `海鮮熱炒`,  `特色餐廳(含雞、鵝、牛、羊肉)`, `傳統餐廳`, `燒烤`, `韓式料理(含火鍋，烤肉)`, `西餐廳(含美式，義式，墨式)`, `西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)`, `西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)` or `早餐`
-        provider: "openai"
-        model: "gpt-4-0125-preview" or 'gpt-3.5-turbo-0125'
-    Return
-        response: str
-    """
-    classes = ", ".join([ "`"+x+"`" for x in classes if x!='早餐' ])+ " or " + "`早餐`"
-    system_prompt = f'''
-            As a helpful and rigorous retail analyst, given the provided query and a list of search results for the query, 
-            your task is to first identify relevant information of the identical store based on store name and proxmity of address if known. After that, extract `store_name`, `address`, `description`, `category` and `phone_number` from the found relevant information, where `category` can only be {classes}. 
-            It's very important to omit unrelated results. Do not make up any assumption.
-            Please think step by step, and output in json format. An example output json is like {{"store_name": "...", "address": "...", "description": "... products, service or highlights ...", "category": "...", "phone_number": "..."}}
-            If no relevant information has been found, simply output json with empty values.
-            I'll tip you and guarantee a place in heaven you do a great job completely according to my instruction.
-        '''
-    user_content = f"`query`: `{query}`\n`search_results`: {search_results}"
-    response = llm( 
-        provider = provider, 
-        model = model, 
-        system_prompt = system_prompt, 
-        user_content = user_content
-    )
-    return response
-
-
-def compose_classication(  user_content, classes: list, backup_classes: list, provider: str, model: str) -> str:
-    """
-    Argument
-        client: 
-        evidence: str
-        classes: list
-        provider: e.g. 'openai'
-        model: e.g. 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview'
-    Return
-        response: str
-    """
-    if isinstance(classes, list):
-        classes = ", ".join([ f"`{x}`" for x in classes])
-    elif isinstance(classes, str):
-        pass
-    else:
-        raise Exception(f"Incorrect classes type: {type(classes)}")
-    system_prompt = f"""
-            As a helpful and rigorous retail analyst, given the provided information about a store, 
-            your task is two-fold. First, classify provided evidence below into the mostly relevant category from the following: {classes}. 
-            Second, if no relevant information has been found, classify the evidence into the mostly relevant supercategory from the following: {backup_classes}.
-            It's very important to omit unrelated piece of evidence and don't make up any assumption.
-            Please think step by step, and must output in json format. An example output json is like {{"category": "..."}}
-            If no relevant piece of information can ever be found at all, simply output json with empty string "".
-            I'll tip you and guarantee a place in heaven you do a great job completely according to my instruction.
-    """
-    response = llm( 
-        provider = provider, 
-        model = model, 
-        system_prompt = system_prompt,
-        user_content = user_content,
-    )
-    return response
-
-
-def classify_results( 
-        analysis_results: pd.DataFrame, 
-        classes: list,
-        backup_classes: list,
-        provider: str, 
-        model: str,
-        input_column: str = 'evidence', 
-        output_column: str = 'classified_category',
-    ):
-    """Classify the results
-    Argument
-        analysis_results: dataframe
-        input_column: str
-        output_column: str
-        classes: list
-    Return 
-        analysis_results: dataframe
-    """
-    classified_results = analysis_results.copy()
-    labels, empty_indices = [], []
-    for idx, evidence in zip( analysis_results['index'], analysis_results[input_column]):
-        try:
-            user_content = f'''`evidence`: `{evidence}`''' 
-            pred_cls = compose_classication( user_content, classes=classes, backup_classes=backup_classes, provider=provider, model=model)
-            label = parse_json_garbage(pred_cls)['category']
-            labels.append(label)
-        except Exception as e:
-            print(f"# CLASSIFICATION error: e -> {e}, user_content -> {user_content}, evidence: {evidence}")
-            labels.append("")
-            empty_indices.append(idx)
-
-    classified_results[output_column] = labels
-    return { 
-        "classified_results": classified_results, 
-        "empty_indices": empty_indices
-    }
-            
-def classify_results_mp( extracted_results: pd.DataFrame, classified_file_path: str, classes: list, backup_classes: list, provider: str, model: str, n_processes: int = 4):
-    """
-    Argument
-        extracted_results:
-        classified_file_path:
-        classes: e.g. ['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)']
-        backup_classes: e.g. [ '中式', '西式']
-        provider:
-        model:
-        n_processes: int
-    Return
-        classified_results: dataframe
-    Reference
-        200 records, 4 processes, 122.4695s
-    """
-    st = time.time()
-    # classified_file_path = "data/classified_result.joblib"
-    if not os.path.exists(classified_file_path):
-        split_data = split_dataframe(extracted_results)
-        with mp.Pool(args.n_processes) as pool:
-            classified_results = pool.starmap( 
-                classify_results,
-                [ ( 
-                    d, 
-                    classes, backup_classes,
-                    provider, model,
-                    'evidence', 'classified_category',
-                ) for d in split_data]
-            ) 
-            classified_results = merge_results( classified_results, dataframe_columns=['classified_results'], list_columns=['empty_indices'])
-            with open( classified_file_path, "wb") as f:
-                joblib.dump( classified_results, f)
-    else:
-        with open( classified_file_path, "rb") as f:
-            classified_results = joblib.load(f)
-    print( f"total time: {time.time() - st}")
-    return classified_results
-
-
-def compose_query( address, name, with_index: bool = True, exclude: str = "-inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw"):
-    """
-    Argumemnt
-        # d: series with d[1]: 地址, d[4]: 營業人名稱 #
-        address: str
-        name: str
-        with_index: bool
-    Return
-        query: `縣市` `營業人名稱`
-    """
-    # if with_index:  # .itertuples()
-    #     query = f"{d[1][:3]} {d[4]}"
-    # else:
-    #     query = f"{d[0][:3]} {d[3]}"
-    query = f"{address[:3]} {name} {exclude}" 
-    return query
-
-def crawl_results( data: pd.DataFrame, google_domain: str = 'google.com.tw', gl: str = 'tw', lr: str = 'lang_zh-TW'):
-    """
-    Argument
-        data: dataframe
-        google_domain: str
-        gl: str
-        lr: str
-    Return
-        crawled_results
-    Reference
-        200 records, 4 processes, 171.36490321159363
-    """
-    serp_results = []
-    condensed_results = []
-    crawled_results = []
-    empty_indices = []
-    for i, d in tqdm(enumerate(data.itertuples())):
-        idx = d[0]
-        address = d[1]
-        business_id = d[2]
-        business_name = d[4]
-        query = compose_query(address, business_name)
-        try:
-            res = get_serp( query, google_domain, gl, lr)
-            serp_results.append(res)
-        except:
-            print( f"# SERP error: i = {i}, idx = {idx}, query = {query}")
-            empty_indices.append(i)
-            continue
-        try:
-            cond_res = get_condensed_result(res)
-            condensed_results.append(cond_res)
-        except:
-            print(f"# CONDENSE error: i = {i}, idx = {idx}, res = {res}")
-            empty_indices.append(i)
-            continue
-        
-        crawled_results.append( { 
-            "index": idx, 
-            "business_id": business_id, 
-            "business_name": business_name, 
-            "serp": res,
-            "evidence": cond_res, 
-            "address": address,
-        } )
-    crawled_results = pd.DataFrame(crawled_results)
-
-    return {
-        "crawled_results": crawled_results,
-        "empty_indices": empty_indices
-    }
-
-def crawl_results_mp( data: pd.DataFrame, crawl_file_path: str, n_processes: int = 4):
-    st = time.time()
-    # crawl_file_path = "data/crawled_results.joblib"
-    if not os.path.exists(crawl_file_path):
-        split_data = split_dataframe( data )
-        with mp.Pool(n_processes) as pool:
-            crawled_results = pool.map( crawl_results, split_data)
-            crawled_results = merge_results( crawled_results, dataframe_columns=['crawled_results'], list_columns=['empty_indices'])
-            with open( crawl_file_path, "wb") as f:
-                joblib.dump( crawled_results, f)
-    else:
-        with open( crawl_file_path, "rb") as f:
-            crawled_results = joblib.load(f)
-    print( f"total time: {time.time() - st}")
-    return crawled_results
-
-def extract_results( data: pd.DataFrame, classes: list, provider: str, model: str):
-    """
-    Argument
-        data: `evidence`, `result`
-    Return
-        extracted_results: dataframe of `extracted_evidence`
-    """
-    extracted_results, empty_indices, ext_res = [], [], []
-    for i, d in tqdm(enumerate(data.itertuples())):
-        idx = d[1]
-        evidence = d.evidence
-        business_id = d[2]
-        business_name = d[3]
-        address = d[6]
-        ana_res = None
-        query = compose_query( address, business_name)
-        try:
-            ext_res = compose_extraction( query = query, search_results = evidence, classes = classes, provider = provider, model = model)
-            ext_res = parse_json_garbage(ext_res)
-        except Exception as e:
-            print(f"# ANALYSIS error: e = {e}, i = {i}, q = {query}, ext_res = {ext_res}")
-            empty_indices.append(i)
-            continue
-        
-        extracted_results.append( { 
-            "index": idx, 
-            "business_id": business_id, 
-            "business_name": business_name, 
-            "evidence": evidence, 
-            ** ext_res
-        } )
-    extracted_results = pd.DataFrame(extracted_results)
-
-    return {
-        "extracted_results": extracted_results, 
-        "empty_indices": empty_indices
-    }
-
-def extract_results_mp( crawled_results, extracted_file_path, classes: list, provider: str, model: str, n_processes: int = 4):
-    """
-    Argument
-        crawled_results: dataframe
-        extracted_file_path
-        classes: list
-    Return 
-    Reference
-        200 records, 4 processes, 502.26914715766907
-    """
-    st = time.time()
-    # args.extracted_file_path = "data/extracted_results.joblib"
-    if not os.path.exists(extracted_file_path):
-        split_data = split_dataframe( crawled_results)
-        with mp.Pool(n_processes) as pool:
-            extracted_results = pool.starmap( extract_results, [ (x, classes, provider, model) for x in split_data])
-            extracted_results = merge_results( extracted_results, dataframe_columns=['extracted_results'], list_columns=['empty_indices'])
-            with open( extracted_file_path, "wb") as f:
-                joblib.dump( extracted_results, f)
-    else:
-        with open( extracted_file_path, "rb") as f:
-            extracted_results = joblib.load(f)
-    print( f"total time: {time.time() - st}")
-    return extracted_results
-
-
-def postprocess_result( results: pd.DataFrame, postprocessed_results_path, category_hierarchy: dict, column_name: str = 'category'):
-    """
-    Argument
-        analysis_result: `evidence`, `result`
-        postprocessed_results_path
-    Return
-    """
-    # index = analysis_result['result']['index']
-    # store_name = data.loc[index]['營業人名稱'] if len(analysis_result['result'].get('store_name',''))==0 else analysis_result['result']['store_name']
-    # address = data.loc[index]['營業地址'] if len(analysis_result['result'].get('address',''))==0 else analysis_result['result']['address']
-    # post_res = { 
-    #     "evidence": analysis_result['evidence'],
-    #     "index": index,
-    #     "begin_date": data.loc[index]['設立日期'],
-    #     "store_name": store_name,
-    #     "address": address,
-    #     "description": analysis_result['result'].get('description', ""),
-    #     "phone_number": analysis_result['result'].get('phone_number', ""),
-    #     "category": analysis_result['result'].get('category', ""),
-    #     "supercategory": category_hierarchy.get(analysis_result['result'].get('category', ""), analysis_result['result'].get('category',"")),
-    # }
-    if not os.path.exists(postprocessed_results_path):
-        postprocessed_results = results.copy()
-        postprocessed_results['supercategory'] = postprocessed_results[column_name].apply(lambda x: category_hierarchy.get(x, ''))
-        with open( postprocessed_results_path, "wb") as f:
-            joblib.dump( postprocessed_results, f)
-    else:
-        with open( postprocessed_results_path, "rb") as f:
-            postprocessed_results = joblib.load(f)
-    return postprocessed_results
-
-
-def combine_results( results: pd.DataFrame, combined_results_path: str, src_column: str = 'classified_category', tgt_column: str = 'category', strategy: str = 'replace'):
-    """
-    Argument
-        classified_results_df: dataframe
-        combined_results_path
-        src_column: str
-        strategy: str, 'replace' or 'patch'
-    Return
-        combined_results: dataframe
-    """
-    if not os.path.exists(combined_results_path):
-        combined_results = results.copy()
-        if strategy == 'replace':
-            condition = (combined_results[tgt_column]=='') | (combined_results[src_column]!=combined_results[tgt_column])
-            combined_results.loc[ condition, tgt_column] = combined_results[condition][src_column].values
-        elif strategy == 'patch':
-            condition = (combined_results[tgt_column]=='')
-            combined_results.loc[ condition, tgt_column] = combined_results[condition][src_column].values
-        else:
-            raise Exception(f"Strategy {strategy} not implemented")
-        with open( combined_results_path, "wb") as f:
-            joblib.dump( combined_results, f)
-    else:
-        with open( combined_results_path, "rb") as f:
-            combined_results = joblib.load(f)
-    return combined_results
-
-def format_evidence(evidence):
-    """
-    """
-    formatted = []
-    evidence = json.loads(evidence)
-    # print( len(evidence) )
-    for i in range(len(evidence)):
-        if 'title' in evidence[i] and '顧客評價' in evidence[i]:
-            f = f"\n> 顧客評價： {evidence[i]['顧客評價']}"
-        elif 'title' in evidence[i] and evidence[i]['title']=='類似的店':
-            f = f"\n> 類似的店： {evidence[i]['snippet']}"
-        elif 'status' in evidence[i]:
-            f = f"\n> 經營狀態： {evidence[i]['status']}"
-        elif 'telephone_number' in evidence[i]:
-            f = f"\n> 電話號碼： {evidence[i]['telephone_number']}"
-        else:
-            try:
-                f = f"{i+1}. {evidence[i]['title']} ({evidence[i].get('snippet','')})"
-            except KeyError:
-                print( evidence[i] )
-                raise KeyError
-        formatted.append(f)
-    return "\n".join(formatted)
-
-def format_output( df: pd.DataFrame, input_column: str = 'evidence', output_column: str = 'formatted_evidence', format_func = format_evidence):
-    """
-    Argument
-        df: `evidence`, `result`
-        input_column: 
-        output_column:
-        format_func:
-    Return
-        formatted_df: dataframe of `formatted_evidence`
-    """
-    formatted_df = df.copy()
-    formatted_df[output_column] = formatted_df[input_column].apply(format_evidence)
-    return formatted_df
-
-def merge_results( results: list, dataframe_columns: list, list_columns: list):
-    """
-    Argument
-        results: a list of dataframes
-        dataframe_columns: list
-        list_columns: list
-    """
-    assert len(results) > 0, "No results to merge"
-    merged_results = {}
-    for result in results:
-        for key in dataframe_columns:
-            mer_res = pd.concat([ r[key] for r in results], ignore_index=True)
-            merged_results[key] = mer_res
-
-        for key in list_columns:
-            mer_res = list(itertools.chain(*[ r[key] for r in results]))
-            merged_results[key] = mer_res
-
-    return merged_results
-
-
-def split_dataframe( df: pd.DataFrame, n_processes: int = 4) -> list:
-    """
-    """
-    n = df.shape[0]
-    n_per_process = math.ceil(n / n_processes)
-    return [ df.iloc[i:i+n_per_process] for i in range(0, n, n_per_process)]
-
-
 def continue_missing(args):
     """
     """
     data = get_leads(args.data_path)
     n_data = data.shape[0]
     
-    formatted_results_path = os.path.join( args.output_dir, args.formatted_results_path)
-    formatted_results = pd.read_csv(formatted_results_path)
+    formatted_results = pd.read_csv(os.path.join( args.output_dir, args.formatted_results_path))
     missing_indices = []
     for i in range(n_data):
         if i not in formatted_results['index'].unique():
-            print(f"{i} is not found")
+            logger.debug(f"{i} is not found")
             missing_indices.append(i)
+    if len(missing_indices)==0:
+        logger.debug("No missing data")
+        return
+    missing_data = data.loc[missing_indices]
+    if not os.path.exists(args.output_missing_dir):
+        os.makedirs(args.output_missing_dir)
+    missing_data.to_csv( args.missing_data_path, index=False, header=False)
 
-    crawled_results_path = os.path.join( args.output_dir, args.crawled_file_path)
-    crawled_results = joblib.load( open( crawled_results_path, "rb"))
-    crawled_results = crawled_results['crawled_results'].query( f"index in {missing_indices}")
-    print( crawled_results)
-    
-    er = extract_results( crawled_results, classes = args.classes, provider = args.provider, model = args.model)
-    er = er['extracted_results']
-    print(er['category'])
-
-    postprossed_results = postprocess_result( 
-        er, 
-        "/tmp/postprocessed_results.joblib",
-        category2supercategory
-    )
-
-    out_formatted_results = format_output( 
-        postprossed_results, 
-        input_column = 'evidence', 
-        output_column = 'formatted_evidence', 
-        format_func = format_evidence
-    )
-
-    out_formatted_results.to_csv( "/tmp/formatted_results.missing.csv", index=False)
-    formatted_results = pd.concat([formatted_results, out_formatted_results], ignore_index=True)
-    formatted_results.sort_values(by='index', ascending=True, inplace=True)
-    formatted_results.to_csv( "/tmp/formatted_results.csv", index=False)
+    args.data_path = args.missing_data_path
+    args.output_dir = args.output_missing_dir
+    if missing_data.shape[0]<args.n_processes:
+        args.n_processes = 1
+    main(args)
 
 
 def main(args):
@@ -541,125 +76,150 @@ def main(args):
             extract: 2791.631685256958(delay = 10)
             classify: 2374.4915606975555(delay = 10)
     """
-    crawled_file_path = os.path.join( args.output_dir, args.crawled_file_path)
-    extracted_file_path = os.path.join( args.output_dir, args.extracted_file_path)
-    classified_file_path = os.path.join( args.output_dir, args.classified_file_path)
-    combined_file_path = os.path.join( args.output_dir, args.combined_file_path)
-    postprocessed_results = os.path.join( args.output_dir, args.postprocessed_results)
-    formatted_results_path = os.path.join( args.output_dir, args.formatted_results_path)
+    steps = args.steps
+    crawled_file_path = os.path.join( args.output_dir, args.crawled_file_path) if args.crawled_file_path is not None else None
+    extracted_file_path = os.path.join( args.output_dir, args.extracted_file_path) if args.extracted_file_path is not None else None
+    # classified_file_path = os.path.join( args.output_dir, args.classified_file_path)
+    # combined_file_path = os.path.join( args.output_dir, args.combined_file_path)
+    postprocessed_file_path = os.path.join( args.output_dir, args.postprocessed_file_path) if args.postprocessed_file_path is not None else None
+    # formatted_results_path = os.path.join( args.output_dir, args.formatted_results_path)
+    filtered_file_path = os.path.join( args.output_dir, args.filtered_file_path) if args.filtered_file_path is not None else None
+    regularized_file_path = os.path.join( args.output_dir, args.regularized_file_path) if args.regularized_file_path is not None else None
 
     ## 讀取資料名單 ##
     data = get_leads(args.data_path)
     
     ## 進行爬蟲與分析 ##
-    crawled_results = crawl_results_mp( data, crawled_file_path, n_processes=args.n_processes)
-    # crawled_results = { k:v[-5:] for k,v in crawled_results.items()}
+    if steps=='all' or steps=='crawl':
+        Path(crawled_file_path).parent.mkdir(parents=True, exist_ok=True)
+        crawled_results = crawl_results_mp( 
+            data, 
+            crawled_file_path, 
+            serp_provider=args.serp_provider, 
+            n_processes=args.n_processes
+        )
+    else:
+        sys.exit(0)
 
+    # crawled_results = { k:v[-5:] for k,v in crawled_results.items()}
+    # crawled_results['crawled_results'].to_csv( formatted_results_path, index=False)
+
+    ## 篩選爬蟲結果 ##
+    # filtered_results = filter_results_mp( 
+    #     data = crawled_results['crawled_results'], 
+    #     filtered_file_path = filtered_file_path,
+    #     provider = args.filter_provider,
+    #     model = args.filter_model,
+    #     n_processes = args.n_processes
+    # )
+    # sys.exit(0)
+    
     ## 方法 1: 擷取關鍵資訊與分類 ##
-    extracted_results = extract_results_mp( 
-        crawled_results = crawled_results['crawled_results'], 
-        extracted_file_path = extracted_file_path,
-        classes = args.classes,
-        provider = args.provider,
-        model = args.model,
-        n_processes = args.n_processes
-    )
+    if steps=='all' or steps=='extract':
+        assert os.path.exists(crawled_file_path), f"# CRAWLED file not found: {crawled_file_path}"
+        crawled_results = joblib.load( open(crawled_file_path, "rb"))
+        extracted_results = extract_results_mp( 
+            crawled_results = crawled_results['crawled_results'],  # filtered_results['filtered_results'], # crawled_results['crawled_results'], 
+            extracted_file_path = extracted_file_path,
+            classes = args.classes,
+            provider = args.extraction_provider, # 'openai', # args.provider,
+            model = args.extraction_model, # 'gpt-3.5-turbo-0125', # args.model,
+            n_processes = args.n_processes
+        )
+    else:
+        sys.exit(0)
 
     ## 方法2: 直接對爬蟲結果分類 ##
-    classified_results = classify_results_mp( 
-        extracted_results['extracted_results'], 
-        classified_file_path, 
-        classes = args.classes, 
-        backup_classes = args.backup_classes, 
-        provider = args.provider,
-        model = args.model,
-        n_processes = args.n_processes
-    )
+    # classified_results = classify_results_mp( 
+    #     extracted_results['extracted_results'], 
+    #     classified_file_path, 
+    #     classes = args.classes, 
+    #     backup_classes = args.backup_classes, 
+    #     provider = args.provider,
+    #     model = args.model,
+    #     n_processes = args.n_processes
+    # )
 
     ## 合併分析結果 ##
-    combined_results = combine_results( 
-        classified_results['classified_results'], 
-        combined_file_path, 
-        src_column = 'classified_category', 
-        tgt_column = 'category', 
-        strategy = args.strategy
-    ) 
+    # combined_results = combine_results( 
+    #     classified_results['classified_results'], 
+    #     combined_file_path, 
+    #     src_column = 'classified_category', 
+    #     tgt_column = 'category', 
+    #     strategy = args.strategy
+    # ) 
+
+    ## 正規化分類結果 ##
+    if steps=='all' or steps=='regularize':
+        assert os.path.exists(args.extracted_file_path), f"# extracted result file not found: {args.extracted_file_path}"
+        extracted_results = joblib.load( open(extracted_file_path, "rb"))
+        regularize_results = regularize_results_mp(
+            extracted_results['extracted_results'], 
+            regularized_file_path,
+            provider = args.regularization_provider, # 'google', # 'openai', # args.provider,
+            model = args.regularization_model # 'gemini-1.5-flash' #  'gpt-3.5-turbo-0125' # args.model
+        )
+    else:
+        sys.exit(0)
 
     ## 後處理分析結果 ##
-    postprossed_results = postprocess_result( 
-        combined_results, 
-        postprocessed_results,
-        category2supercategory
-    )
-
-    formatted_results = format_output( postprossed_results, input_column = 'evidence', output_column = 'formatted_evidence', format_func = format_evidence)
-    formatted_results.to_csv( formatted_results_path, index=False)
-
-
-category2supercategory = {
-        "小吃店": "中式",
-        "日式料理(含居酒屋，串燒)": "中式",
-        "火(鍋／爐)": "中式",
-        "東南亞料��(不含日韓)": "中式",
-        "海鮮熱炒": "中式",
-        "特色餐廳(含雞、鵝、牛、羊肉)": "中式",
-        "傳統餐廳": "中式",
-        "燒烤": "中式", 
-        "韓式料理(含火鍋，烤肉)": "中式",
-        "西餐廳(含美式，義式，墨式)": "西式",
-        "中式": "中式", 
-        "西式": "西式",
-        "西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)": "西式", 
-        "西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)": "西式",
-        "早餐": ""
-    }
+    if steps=='all' or steps=='postprocess':
+        assert os.path.exists(args.regularized_file_path), f"# extracted result file not found: {args.extracted_file_path}"
+        regularize_results = joblib.load( open(regularized_file_path, "rb"))
+        postprossed_results = postprocess_result( 
+            regularize_results['regularized_results'], # extracted_results['extracted_results'], # combined_results, 
+            postprocessed_file_path,
+            category2supercategory
+        )
+    else:
+        sys.exit(0)
 
-supercategory2category = {
-        "中式": [
-            "小吃店",
-            "日式料理(含居酒屋，串燒)",
-            "火(鍋／爐)",
-            "東南亞料理(不含日韓)",
-            "海鮮熱炒",
-            "特色餐廳(含雞、鵝、牛、羊肉)",
-            "傳統餐廳",
-            "燒烤", 
-            "韓式料理(含火鍋，烤肉)"
-        ],
-        "西式": ["西餐廳(含美式，義式，墨式)", "西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)", "西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)"],
-        "": ["早餐"]
-    }
 
 if __name__=='__main__':
     
     base = "https://serpapi.com/search.json" 
     engine = 'google'
-    # query = "Coffee"
     google_domain = 'google.com.tw'
     gl = 'tw'
     lr = 'lang_zh-TW'
-    # url = f"{base}?engine={engine}&q={query}&google_domain={google_domain}&gl={gl}&lr={lr}"
     n_processes = 4
     client = OpenAI( organization = ORGANIZATION_ID)
 
     parser = argparse.ArgumentParser()
+    parser.add_argument("--config", type=str, default='config/config.yml', help="Path to the configuration file")
     parser.add_argument("--data_path", type=str, default="data/餐廳類型分類.xlsx - 測試清單.csv")
+    parser.add_argument("--missing_data_path", type=str, default="data/missing/missing.csv")
     parser.add_argument("--task", type=str, default="new", choices = ["new", "continue"], help="new or continue")
+    parser.add_argument("--steps", type=str, default="all", choices = ["all", "crawl", "extract", "regularize", "postprocess"], help="new or continue")
     parser.add_argument("--output_dir", type=str, help='output directory')
+    parser.add_argument("--output_missing_dir", type=str, help='output missing directory')
     parser.add_argument("--classified_file_path", type=str, default="classified_results.joblib")
     parser.add_argument("--extracted_file_path", type=str, default="extracted_results.joblib")
     parser.add_argument("--crawled_file_path", type=str, default="crawled_results.joblib")
     parser.add_argument("--combined_file_path", type=str, default="combined_results.joblib")
-    parser.add_argument("--postprocessed_results", type=str, default="postprocessed_results.joblib")
+    parser.add_argument("--regularized_file_path", type=str, default="regularized_results.joblib")
+    parser.add_argument("--postprocessed_file_path", type=str, default="postprocessed_results.csv")
     parser.add_argument("--formatted_results_path", type=str, default="formatted_results.csv")
-    parser.add_argument("--classes", type=list, default=['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)', '西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)', '西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)', '早餐'])
+    parser.add_argument("--filtered_file_path", type=str, default="filtered_results.csv")
+    # parser.add_argument("--classes", type=list, default=['小吃店', '日式料理(含居酒屋，串燒)', '火(鍋／爐)', '東南亞料理(不含日韓)', '海鮮熱炒',  '特色餐廳(含雞、鵝、牛、羊肉)', '傳統餐廳', '燒烤', '韓式料理(含火鍋，烤肉)', '西餐廳(含美式，義式，墨式)', '西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)', '西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)', '早餐'])
+    parser.add_argument("--classes", type=list, default=['小吃店','日式料理(含居酒屋，串燒)','火(鍋／爐)','東南亞料理(不含日韓)','海鮮熱炒','特色餐廳(含雞、鵝、牛、羊肉)','釣蝦場','傳統餐廳','燒烤','韓式料理(含火鍋，烤肉)','PUB(Live Band)','PUB(一般，含Lounge)','PUB(電音＼舞場)','五星級飯店','自助KTV(含連鎖，庭��自助)','西餐廳(含美式，義式，墨式)','咖啡廳(泡沫紅茶)','飯店(星級／旅館，不含五星級)','運動休閒館(含球類練習場，飛鏢等)','西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)','西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)','早餐'] )
+    # `小吃店`,`日式料理(含居酒屋，串燒)`,`火(鍋／爐)`,`東南亞料理(不含日韓)`,`海鮮熱炒`,`特色餐廳(含雞、鵝、牛、羊肉)`,`釣蝦場`,`傳統餐廳`,`燒烤`,`韓式料理(含火鍋，烤肉)`,`PUB(Live Band)`,`PUB(一般，含Lounge)`,`PUB(電音＼舞場)`,`五星級飯店`,`自助KTV(含連鎖，庭園自助)`,`西餐廳(含美式，義式，墨式)`,`咖啡廳(泡沫紅茶)`,`飯店(星級／旅館，不含五星級)`,`運動休閒館(含球類練習場，飛鏢等)`,`西餐廳(餐酒館、酒吧、飛鏢吧、pub、lounge bar)`,`西餐廳(土耳其、漢堡、薯條、法式、歐式、印度)`,`早餐`
     parser.add_argument("--backup_classes", type=list, default=['中式', '西式'])
     parser.add_argument("--strategy", type=str, default='patch', choices=['replace', 'patch'])
-    parser.add_argument("--provider", type=str, default='openai', choices=['openai', 'anthropic'])
-    parser.add_argument("--model", type=str, default='gpt-4-0125-preview', choices=['claude-3-sonnet-20240229', 'claude-3-haiku-20240307', 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview'])
+    parser.add_argument("--filter_provider", type=str, default='google', choices=['google', 'openai', 'anthropic'])
+    parser.add_argument("--filter_model", type=str, default='gemini-1.5-flash', choices=[ 'claude-3-5-sonnet-20240620', 'claude-3-sonnet-20240229', 'claude-3-haiku-20240307', 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview', 'gpt-4o', 'gpt-4o-mini', 'gemini-1.5-flash'])
+    parser.add_argument("--extraction_provider", type=str, default='openai', choices=['google', 'openai', 'anthropic'])
+    parser.add_argument("--extraction_model", type=str, default='gpt-3.5-turbo-0125', choices=[ 'claude-3-5-sonnet-20240620', 'claude-3-sonnet-20240229', 'claude-3-haiku-20240307', 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview', 'gpt-4o', 'gpt-4o-mini', 'gemini-1.5-flash'])
+    parser.add_argument("--regularization_provider", type=str, default='google', choices=['google', 'openai', 'anthropic'])
+    parser.add_argument("--regularization_model", type=str, default='gemini-1.5-flash', choices=['claude-3-5-sonnet-20240620', 'claude-3-sonnet-20240229', 'claude-3-haiku-20240307', 'gpt-3.5-turbo-0125', 'gpt-4-0125-preview', 'gpt-4o', 'gpt-4o-mini', 'gemini-1.5-flash'])
+    parser.add_argument("--serp_provider", type=str, default='serp', choices=['serp', 'serper'])
     parser.add_argument("--n_processes", type=int, default=4)
     args = parser.parse_args()
 
+    config = yaml.safe_load(open(args.config,"r").read())
+    category2supercategory = config['category2supercategory']
+    supercategory2category = reverse_category2supercategory(category2supercategory)
+
     if args.task == 'new':
         main(args)
     elif args.task == 'continue':

utils.py

@@ -1,13 +1,198 @@
+import os
 import re
 import json
+import itertools
+import math
+import joblib
+from typing import List
 
-def parse_json_garbage(s):
-    s = s[next(idx for idx, c in enumerate(s) if c in "{["):]
-    print(s)
-    s = s[:next(idx for idx, c in enumerate(s) if c in "}]")+1]
-    print(s)
+import pandas as pd
+from loguru import logger
+
+
+def parse_json_garbage(s, start="{", end="}"):
+    """Parse JSON string without comments
+    Argument
+        s: str
+        start: str
+        end: str
+    Return
+        json_obj: dict
+    """
+    s = s[next(idx for idx, c in enumerate(s) if c in start):]
+    # print(f"fix head -> {s}")
+    s = s[:next(idx for idx, c in enumerate(s) if c in end)+1]
+    # print(f"fix tail -> {s}")
+    if s.startswith("json"):
+        s = s[4:]
     try:
         return json.loads(re.sub("[//#].*","",s,flags=re.MULTILINE))
     except json.JSONDecodeError as e:
-        return json.loads(re.sub("[//#].*","",s,flags=re.MULTILINE))
+        logger.warning(f"Error parsing JSON (trying another regex...): {e}")
+        return json.loads(re.sub("^[//#].*","",s,flags=re.MULTILINE))
     
+
+def merge_results( results: list, dataframe_columns: list, list_columns: list):
+    """
+    Argument
+        results: a list of dataframes
+        dataframe_columns: list
+        list_columns: list
+    Return 
+        merged_results: dict
+    """
+    assert len(results) > 0, "No results to merge"
+    merged_results = {}
+    for result in results:
+        for key in dataframe_columns:
+            mer_res = pd.concat([ r[key] for r in results], ignore_index=True)
+            merged_results[key] = mer_res
+
+        for key in list_columns:
+            mer_res = list(itertools.chain(*[ r[key] for r in results]))
+            merged_results[key] = mer_res
+
+    return merged_results
+
+
+def split_dataframe( df: pd.DataFrame, n_processes: int = 4) -> list:
+    """
+    """
+    n = df.shape[0]
+    n_per_process = max( math.ceil(n / n_processes), 1)
+    return [ df.iloc[i:i+n_per_process] for i in range(0, n, n_per_process)]
+
+
+def combine_results( results: pd.DataFrame, combined_results_path: str, src_column: str = 'classified_category', tgt_column: str = 'category', strategy: str = 'replace'):
+    """
+    Argument
+        classified_results_df: dataframe
+        combined_results_path
+        src_column: str
+        strategy: str, 'replace' or 'patch'
+    Return
+        combined_results: dataframe
+    """
+    if not os.path.exists(combined_results_path):
+        combined_results = results.copy()
+        if strategy == 'replace':
+            condition = (combined_results[tgt_column]=='') | (combined_results[src_column]!=combined_results[tgt_column])
+            combined_results.loc[ condition, tgt_column] = combined_results[condition][src_column].values
+        elif strategy == 'patch':
+            condition = (combined_results[tgt_column]=='')
+            combined_results.loc[ condition, tgt_column] = combined_results[condition][src_column].values
+        else:
+            raise Exception(f"Strategy {strategy} not implemented")
+        with open( combined_results_path, "wb") as f:
+            joblib.dump( combined_results, f)
+    else:
+        with open( combined_results_path, "rb") as f:
+            combined_results = joblib.load(f)
+    return combined_results
+
+
+def split_dict( information: dict | List[dict], keys1: List[str], keys2: List[str]):
+    """[ { key1: value1, key2: value2}, { key1: value1, key2: value2}] -> [ {key1: value1}, {key1: value1}], [{key2: value2, key2: value2}]
+    Argument
+        information: dict | List[dict], dim -> N
+        keys1: List[str], dim -> K1
+        keys2: List[str], dim -> K2
+    Example:
+        >> split_dict( [ {"a": 1, "b":2, "c": 3}, {"a": 1, "b":2, "c": 3}, {"a": 1, "b":2, "c": 3}], ['a','b'], ['c']) 
+        >> ( [{'a': 1, 'b': 2}, {'a': 1, 'b': 2}, {'a': 1, 'b': 2}], [{'c': 3}, {'c': 3}, {'c': 3}] )
+    """
+    assert len(keys1)>0 and len(keys2)>0
+    results1, results2 = [], []
+    if isinstance( information, dict):
+        information = [ information]
+    for info in information: # N
+        split_results1 = {} # K1
+        for key in keys1:
+            if key in info:
+                split_results1[key] = info[key]
+            else:
+                split_results1[key] = None
+        results1.append( split_results1)
+        split_results2 = {} # K2
+        for key in keys2:
+            if key in info:
+                split_results2[key] = info[key]
+            else:
+                split_results2[key] = None
+        results2.append( split_results2)
+        # results.append( [ split_results1, split_results2])
+    assert len(results1)==len(results2)
+    if len(results1)==1:
+        return results1[0], results2[0]
+    return results1, results2
+
+
+def format_df( df: pd.DataFrame, input_column: str = 'evidence', output_column: str = 'formatted_evidence', format_func: str = lambda x: x):
+    """
+    Argument
+        df: `evidence`, `result`
+        input_column: 
+        output_column:
+        format_func:
+    Return
+        formatted_df: dataframe of `formatted_evidence`
+    """
+    formatted_df = df.copy()
+    formatted_df[output_column] = formatted_df[input_column].apply(format_func)
+    return formatted_df
+
+
+def clean_quotes( text: str):
+    """
+    """
+    return text.strip().replace("\u3000","").replace("\r","").replace("\"", "").replace("'", "")
+
+
+def compose_query( address, name, with_index: bool = True, exclude: str = "-inurl:twincn.com -inurl:findcompany.com.tw -inurl:iyp.com.tw -inurl:twypage.com -inurl:alltwcompany.com -inurl:zhupiter.com -inurl:twinc.com.tw", use_exclude: bool = True):
+    """
+    Argumemnt
+        # d: series with d[1]: 地址, d[4]: 營業人名稱 #
+        address: str
+        name: str
+        with_index: bool
+    Return
+        query: `縣市` `營業人名稱`
+    """
+    # if with_index:  # .itertuples()
+    #     query = f"{d[1][:3]} {d[4]}"
+    # else:
+    #     query = f"{d[0][:3]} {d[3]}"
+    if use_exclude:
+        query = f"{address[:3]} {name} {exclude}" 
+    else:
+        query = f"{address[:3]} {name}" 
+    return query
+
+
+def reverse_category2supercategory(category2supercategory):
+    """
+    Argument
+        category2supercategory: dict
+    Return
+        supercategory2category: dict
+    """
+    supercategory2category = {}
+    for key, value in category2supercategory.items():
+        if value not in supercategory2category:
+            supercategory2category[value] = [key]
+        else:
+            supercategory2category[value].append(key)
+    return supercategory2category
+
+def concat_df( list_df: List[pd.DataFrame], axis: int = 0):
+    """
+    Argument
+        list_df: List[pd.DataFrame]
+        axis: int
+    Return
+        df: pd.DataFrame
+    """
+    assert len(list_df)>0, "Empty list of dataframes"
+    if len(list_df)==1:
+        return list_df[0]
+    return pd.concat( list_df, axis=axis)