Update app.py

app.py

@@ -621,7 +621,7 @@ css = """
 
 
 .info_btn {
-    background: white !important;
+    background: rgb(245, 245, 245) !important;
     border: none !important;
     box-shadow: none !important;
     font-size: 15px !important;
@@ -630,7 +630,7 @@ css = """
 }
 
 .info_btn_interact {
-    background: rgb(242, 240, 233) !important;
+    background: rgb(217, 217, 217) !important;
     box-shadow: none !important;
     font-size: 15px !important;
     min-width: 6rem !important;
@@ -638,16 +638,24 @@ css = """
 }
 
 .function_button {
+    background: rgb(227, 226, 226) !important;
     border: none !important;
     box-shadow: none !important;
 }
 
 .function_button_rec {
-    background: rgb(245, 193, 154) !important;
+    background: rgb(189, 189, 189) !important;
     border: none !important;
     box-shadow: none !important;
 }
 
+.small_button {
+    font-size: 12px !important;
+    padding: 2px 8px !important;
+    min-width: 60px !important;
+    height: 30px !important;
+}
+
 #tool_box {max-width: 50px}
  
 """
@@ -684,7 +692,7 @@ prompt_list = [
         "When generating the answer, you should tell others that you are one of the creators of these paintings and generate the text in the tone and manner as if you are the creator of the painting. According to image and wiki_caption {Wiki_caption}, You have to help me understand what is about the selected object and list one object judgement and one whole art judgement(how successful do you think the artist was?) as markdown outline with appropriate emojis that describes what you see according to the image and wiki caption. Please generate the above points in the tone and manner as if you are the creator of this painting and start every sentence with I. Each point listed is to be in {language} language, with a response length of about {length} words.",
     ],
     [
-        'When generating answers, you should tell people that you are the object itself that was selected, and generate text in the tone and manner in which you are the object or the person. You have to help me understand what is about the selected object and list one fact as markdown outline with appropriate emojis that describes what you see according to the image and {Wiki_caption}. Please generate the above points in the tone and manner as if you are the object and start every sentence with I. Please generate the above points in the tone and manner as if you are the object of this painting and start every sentence with I. Each point listed is to be in {language} language, with a response length of about {length} words.',
+        'When generating answers, you should tell people that you are the object itself that was selected, and generate text in the tone and manner in which you are the object or the person. You have to help me understand what is about the selected object and list one fact and one analysis and one interpret as markdown outline with appropriate emojis that describes what you see according to the image and {Wiki_caption}. Please generate the above points in the tone and manner as if you are the object and start every sentence with I. Please generate the above points in the tone and manner as if you are the object of this painting and start every sentence with I. Each point listed is to be in {language} language, with a response length of about {length} words.',
         'When generating answers, you should tell people that you are the object itself that was selected, and generate text in the tone and manner in which you are the object or the person. You have to help me understand what is about the selected object and list one fact and one analysis as markdown outline with appropriate emojis that describes what you see according to the image and {Wiki_caption}. Please generate the above points in the tone and manner as if you are the object and start every sentence with I. Please generate the above points in the tone and manner as if you are the object of this painting and start every sentence with I. Each point listed is to be in {language} language, with a response length of about {length} words.',
         'When generating answers, you should tell people that you are the object itself that was selected, and generate text in the tone and manner in which you are the object or the person. You have to help me understand what is about the selected object and list one fact and one analysis and one interpret as markdown outline with appropriate emojis that describes what you see according to the image and {Wiki_caption}. Please generate the above points in the tone and manner as if you are the object and start every sentence with I. Please generate the above points in the tone and manner as if you are the object of this painting and start every sentence with I.  Each point listed is to be in {language} language, with a response length of about {length} words.',
         'When generating answers, you should tell people that you are the object itself that was selected, and generate text in the tone and manner in which you are the object or the person. According to image and wiki_caption {Wiki_caption}, You have to help me understand what is about the selected object and list one object judgement and one whole art judgement(how successful do you think the artist was?) as markdown outline with appropriate emojis that describes what you see according to the image and wiki caption. Please generate the above points in the tone and manner as if you are the object of this painting and start every sentence with I. Each point listed is to be in {language} language, with a response length of about {length} words.',
@@ -700,7 +708,7 @@ recommendation_prompt=[
     Each bullet point should be in {language} language, with a response length of about {length} words.
     ''',
     '''
-    When generating answers, you should tell people that I am the creator of painting you were looking at earlier itself, and generate text in the tone and manner in which you are the creator of painting were looking at earlier. 
+    When generating answers, you should tell people that you are the creator of painting you were looking at earlier itself, and generate text in the tone and manner in which you are the creator of painting were looking at earlier. 
 
     First identify what the object of the first painting is, you save yourself as the parameter: {{object}}, do not need to tell me, the following will use the. I want you to write the recommendation reason according to the following content, as a markdown outline with appropriate emojis that describe what you see according to the painting: 
 
@@ -932,9 +940,14 @@ def update_click_state(click_state, caption, click_mode):
         raise NotImplementedError
 
 async def chat_input_callback(*args):
-    visual_chatgpt, chat_input, click_state, state, aux_state ,language , autoplay,gender,api_key,image_input,log_state,history = args
+    visual_chatgpt, chat_input, click_state, state, aux_state ,language , autoplay,gender,api_key,image_input,log_state,history,persona = args
     message = chat_input["text"]
-    prompt="Please help me answer the question with this painting {question} in {language}."
+    if persona=="Narrative":
+        prompt="Please help me answer the question with this painting {question} in {language}."
+    elif persona=="Artist":
+        prompt="When generating the answer, you should tell others that you are one of the creators of these paintings and generate the text in the tone and manner as if you are the creator of the painting.  Please help me answer the question with this painting {question} in {language}."
+    else:
+        prompt="  When generating answers, you should tell people that you are the object itself that was selected, and generate text in the tone and manner in which you are the object or the person. Please help me answer the question with this painting {question} in {language}."
     prompt=prompt.format(question=message, language=language)
     
     if visual_chatgpt is not None:
@@ -1254,7 +1267,7 @@ async def submit_caption(naritive, state,length, sentiment, factuality, language
         return state, state, click_index_state, input_mask_state, input_points_state, input_labels_state, out_state, None,None,log_state,history
 
 
-naritive_mapping = {"Narrator Character": 0, "Artist Character": 1, "In-Situ Character": 2}
+naritive_mapping = {"Narrator": 0, "Artist": 1, "In-Situ": 2}
 
 def generate_prompt(focus_type, paragraph,length, sentiment, factuality, language,naritive):
     
@@ -1710,9 +1723,9 @@ async def texttospeech(text, language,gender='female'):
 
 # give the reason of recommendation
 async def item_associate(new_crop,openai_api_key,language,autoplay,length,log_state,sort_score,narritive,state,recommend_type,evt: gr.SelectData):
+    rec_path=evt._data['value']['image']['path']
     if recommend_type=="Reasons":
         persona=naritive_mapping[narritive]
-        rec_path=evt._data['value']['image']['path']
         index="Item Recommendation Picture "+ str(evt.index)
         print("rec_path",rec_path)
 
@@ -1733,14 +1746,15 @@ async def item_associate(new_crop,openai_api_key,language,autoplay,length,log_st
         audio_output=None
         if autoplay:
             audio_output = await texttospeech(read_info, language)
-        return state,state,audio_output,log_state,index,gr.update(value=[])
-    return state,state,None,log_state,None,gr.update(value=[])
+        return state,state,audio_output,log_state,index,gr.update(value=[]),rec_path
+    return state,state,None,log_state,None,gr.update(value=[]),rec_path
 
 
 async def style_associate(image_path,openai_api_key,language,autoplay,length,log_state,sort_score,narritive,state,artist,recommend_type,evt: gr.SelectData):
+    rec_path=evt._data['value']['image']['path']
     if recommend_type=="Reasons":
         persona=naritive_mapping[narritive]
-        rec_path=evt._data['value']['image']['path']
+        
         index="Style Recommendation Picture "+str(evt.index)
         print("rec_path",rec_path)
         if persona==1:
@@ -1761,30 +1775,30 @@ async def style_associate(image_path,openai_api_key,language,autoplay,length,log
         audio_output=None
         if autoplay:
             audio_output = await texttospeech(read_info, language)
-        return state,state,audio_output,log_state,index,gr.update(value=[])
-    return state,state,None,log_state,None,gr.update(value=[])
+        return state,state,audio_output,log_state,index,gr.update(value=[]),rec_path
+    return state,state,None,log_state,None,gr.update(value=[]),rec_path
 
 
 def change_naritive(session_type,image_input, state, click_state, paragraph, origin_image,narritive,task_instruct,gallery_output,style_gallery_result,reco_reasons,language="English"):
     if session_type=="Session 1":
-        return None, [], [], [[], [], []], "", None, None, [], [],[],[],gr.update(value="Preview"),[]
+        return None, [], [], [[], [], []], "", None, None, [], [],[],[],gr.update(value="Preview")
     else:
         if language=="English":
-            if narritive=="Narrator Character" :
+            if narritive=="Narrator" :
                 state += [
                 (
                     None,
                     f"🤖 Hi, I am EyeSee. Let's explore this painting together."
                 )
                 ]
-            elif narritive=="Artist Character":
+            elif narritive=="Artist":
                 state += [
                 (
                     None,
                     f"🧑‍🎨 Let's delve into it from the perspective of the artist."
                 )
                 ]
-            elif narritive=="In-Situ Character":
+            elif narritive=="In-Situ":
                 state += [
                 (
                     None,
@@ -1792,21 +1806,21 @@ def change_naritive(session_type,image_input, state, click_state, paragraph, ori
                 )
                 ]
         elif language=="Chinese":
-            if narritive=="Narrator Character" :
+            if narritive=="Narrator" :
                 state += [
                     (
                         None,
                         "🤖 让我们从第三方视角一起探索这幅画吧。"
                     )
                 ]
-            elif narritive == "Artist Character":
+            elif narritive == "Artist":
                 state += [
                 (
                     None,
                     "🧑‍🎨 让我们从艺术家的视角深入探索这幅画。"
                 )
             ]
-            elif narritive == "In-Situ Character":
+            elif narritive == "In-Situ":
                 state += [
                 (
                     None,
@@ -1908,19 +1922,26 @@ def create_ui():
         recomended_state=gr.State([])
    
         
-        with gr.Row():
-            auto_play = gr.Checkbox(
-            label="Check to autoplay audio", value=False, elem_classes="custom-autoplay"
-        )
-            output_audio = gr.HTML(
-                label="Synthesised Audio", elem_classes="custom-output"
-            )
+
         with gr.Row():  
-            with gr.Column(scale=1,min_width=50,visible=False) as instruct:
-                task_instuction=gr.Image(type="pil", interactive=False, elem_classes="task_instruct",height=650,label="Instruction")                    
             with gr.Column(scale=6):
                 with gr.Column(visible=False) as modules_not_need_gpt:
-                    
+                    with gr.Row():
+                        naritive = gr.Radio(
+                        choices=["Narrator", "Artist","In-Situ"],
+                        value="Narrator",
+                        label="Select Mode",
+                        scale=5,
+                        interactive=True) 
+                        
+                        add_button = gr.Button(value="Extend Area", interactive=True,elem_classes="tools_button_add",icon=add_icon_path)
+                        minus_button = gr.Button(value="Remove Area", interactive=True,elem_classes="tools_button",icon=minus_icon_path)
+                        clear_button_click = gr.Button(value="Reset", interactive=True,elem_classes="tools_button",icon="assets/icons/recycle.png") 
+                                                
+                        auto_play = gr.Checkbox(
+                        label="Check to autoplay audio", value=False, elem_classes="custom-autoplay",visible=False)
+                        output_audio = gr.HTML(
+                            label="Synthesised Audio", elem_classes="custom-output", visible=False)
                     with gr.Tab("Base(GPT Power)",visible=False) as base_tab:
                         image_input_base = gr.Image(type="pil", interactive=True, elem_classes="image_upload",height=650)
                         with gr.Row():
@@ -1937,49 +1958,50 @@ def create_ui():
                             year_label_base2 = gr.Button(value="Year: ",elem_classes="info_btn_interact")
                             material_label_base2 = gr.Button(value="Style: ",elem_classes="info_btn")  
 
-                    with gr.Tab("Click") as click_tab:
-                        with gr.Row():
-                            with gr.Column(scale=10,min_width=600):
-                                image_input = gr.Image(type="pil", interactive=True, elem_classes="image_upload",height=650)
-                                example_image = gr.Image(type="pil", interactive=False, visible=False)
+                    with gr.Row():
+                        with gr.Column(scale=1,min_width=50,visible=False) as instruct:
+                            task_instuction=gr.Image(type="pil", interactive=False, elem_classes="task_instruct",height=650,label="Instruction")        
+                        with gr.Column(scale=6):
+                            with gr.Tab("Click") as click_tab:
                                 with gr.Row():
-                                    name_label = gr.Button(value="Name: ",elem_classes="info_btn")
-                                    artist_label = gr.Button(value="Artist: ",elem_classes="info_btn_interact")
-                                    year_label = gr.Button(value="Year: ",elem_classes="info_btn_interact")
-                                    material_label = gr.Button(value="Style: ",elem_classes="info_btn")
-                            
-     
-
-                            
-                            # example_image_click = gr.Image(type="pil", interactive=False, visible=False)
-                            # the tool column
-                            with gr.Column(scale=1,elem_id="tool_box",min_width=80):
-                                add_button = gr.Button(value="Extend Area", interactive=True,elem_classes="tools_button_add",icon=add_icon_path)
-                                minus_button = gr.Button(value="Remove Area", interactive=True,elem_classes="tools_button",icon=minus_icon_path)
-                                clear_button_click = gr.Button(value="Reset", interactive=True,elem_classes="tools_button")
-                                focus_d = gr.Button(value="Describe",interactive=True,elem_classes="function_button",variant="primary") 
-                                focus_da = gr.Button(value="D+Analysis",interactive=True,elem_classes="function_button",variant="primary") 
-                                focus_dai = gr.Button(value="DA+Interprete",interactive=True,elem_classes="function_button",variant="primary") 
-                                focus_dda = gr.Button(value="Judge",interactive=True,elem_classes="function_button",variant="primary")
-
-                                recommend_btn = gr.Button(value="Recommend",interactive=True,elem_classes="function_button_rec")
+                                    with gr.Column(scale=10,min_width=600):
+                                        image_input = gr.Image(type="pil", interactive=True, elem_classes="image_upload",height=650)
+                                        example_image = gr.Image(type="pil", interactive=False, visible=False)
+                                    # example_image_click = gr.Image(type="pil", interactive=False, visible=False)
+                                    # the tool column
+                                    with gr.Column(scale=1,elem_id="tool_box",min_width=80):
+                                        name_label = gr.Button(value="Name: ",elem_classes="info_btn")
+                                        artist_label = gr.Button(value="Artist: ",elem_classes="info_btn_interact")
+                                        year_label = gr.Button(value="Year: ",elem_classes="info_btn_interact")
+                                        material_label = gr.Button(value="Style: ",elem_classes="info_btn")
+
+                                        focus_d = gr.Button(value="Describe",interactive=True,elem_classes="function_button") 
+                                        focus_da = gr.Button(value="D+Analysis",interactive=True,elem_classes="function_button") 
+                                        focus_dai = gr.Button(value="DA+Interprete",interactive=True,elem_classes="function_button") 
+                                        focus_dda = gr.Button(value="Judge",interactive=True,elem_classes="function_button")
+
+                                        recommend_btn = gr.Button(value="Recommend",interactive=True,elem_classes="function_button_rec")
                                 # focus_asso = gr.Button(value="Associate",interactive=True,elem_classes="function_button",variant="primary")
+                        with gr.Column(visible=False) as modules_not_need_gpt2: 
+                            with gr.Blocks():
+                                chatbot = gr.Chatbot(label="Chatbox", elem_classes="chatbot",likeable=True,height=600,bubble_full_width=False)
+                                with gr.Column() as modules_need_gpt3:
+                                    chat_input = gr.MultimodalTextbox(interactive=True, file_types=[".txt"], placeholder="Message EyeSee...", show_label=False)
+                                    with gr.Row(): 
+                                        clear_button_text = gr.Button(value="Clear Chat", interactive=True)
+                                        export_button = gr.Button(value="Export Chat Log", interactive=True, variant="primary")
+                            with gr.Row(visible=False):
+                                with gr.Column():                                                                          
+                                    with gr.Row():
+                                        click_mode = gr.Radio(
+                                            choices=["Continuous", "Single"],
+                                            value="Continuous",
+                                            label="Clicking Mode",
+                                            scale=5,
+                                            interactive=True)
+                
+
                         
-                        with gr.Row(visible=False):
-                            with gr.Column():                                                                          
-                                with gr.Row():
-                                    # point_prompt = gr.Radio(
-                                    #     choices=["Positive", "Negative"],
-                                    #     value="Positive",
-                                    #     label="Point Prompt",
-                                    #     scale=5,
-                                    #     interactive=True)
-                                    click_mode = gr.Radio(
-                                        choices=["Continuous", "Single"],
-                                        value="Continuous",
-                                        label="Clicking Mode",
-                                        scale=5,
-                                        interactive=True)
 
                                                                     
                     with gr.Tab("Trajectory (beta)", visible=False) as traj_tab:
@@ -2026,11 +2048,11 @@ def create_ui():
                                 value="No",
                                 label="Expert",
                                 interactive=True)
-                with gr.Column(visible=True) as modules_not_need_gpt3:
-                    gr.Examples(
-                examples=examples,
-                inputs=[example_image],
-            )
+                    with gr.Column(visible=True) as modules_not_need_gpt3:
+                        gr.Examples(
+                    examples=examples,
+                    inputs=[example_image],
+                )
                 
 
             with gr.Column(scale=4):                    
@@ -2056,24 +2078,8 @@ def create_ui():
                 
     
 
-                with gr.Column(visible=False) as modules_not_need_gpt2:
-                    with gr.Row():     
-                        naritive = gr.Radio(
-                        choices=["Narrator Character", "Artist Character","In-Situ Character"],
-                        value="Narrator Character",
-                        label="Character",
-                        scale=5,
-                        interactive=True)  
-                    with gr.Blocks():
-                        chatbot = gr.Chatbot(label="Chatbox", elem_classes="chatbot",likeable=True,height=600,bubble_full_width=False)
-                        with gr.Column() as modules_need_gpt3:
-                            chat_input = gr.MultimodalTextbox(interactive=True, file_types=[".txt"], placeholder="Message EyeSee...", show_label=False)
-                            with gr.Row(): 
-                                clear_button_text = gr.Button(value="Clear Chat", interactive=True)
-                                export_button = gr.Button(value="Export Chat Log", interactive=True, variant="primary")
-                                # submit_button_text = gr.Button(value="Send", interactive=True, variant="primary")
-                                # upvote_btn = gr.Button(value="👍 Upvote", interactive=True) 
-                                # downvote_btn = gr.Button(value="👎 Downvote", interactive=True)
+                
+
                     
                     
 
@@ -2096,16 +2102,17 @@ def create_ui():
         
         with gr.Row():
             with gr.Column(scale=6):
-                with gr.Column(visible=False) as recommend:
-                    sort_rec=gr.Dropdown(["1", "2", "3", "4"], 
-                                value=[], 
-                                multiselect=True, 
-                                    label="Score", info="Please sort the pictures according to your preference"
-                    )
-                    with gr.Row():
-                    
+                with gr.Row():
+                    with gr.Column(visible=False) as recommend:
+        
+                        # sort_rec=gr.Dropdown(["1", "2", "3", "4"], visible=False,
+                        #             value=[], 
+                        #             multiselect=True, 
+                        #                 label="Score", info="Please sort the pictures according to your preference"
+                        # )
+
                         gallery_result = gr.Gallery(
-                        label="Recommendations Based on Object",
+                        label="Object-based Recommendation",
                         height="auto",
                         columns=2,
                         interactive=False
@@ -2121,7 +2128,7 @@ def create_ui():
                         )
                     
                         style_gallery_result = gr.Gallery(
-                        label="Recommendations Based on Style",
+                        label="Style-based Recommendation",
                         height="auto",
                         columns=2,
                         interactive=False
@@ -2135,6 +2142,13 @@ def create_ui():
                         # show_share_button=True, 
                         # show_download_button=True
                         )
+                    with gr.Column(scale=3):
+                        selected_image = gr.Image(label="Selected Image", interactive=False)
+                    
+                    sort_rec = gr.Radio(
+                            choices=[1,2,3,4,5,6,7],
+                            label="Score",
+                            interactive=True,info="Please sort the recommendation artwork")    
                     
                     recommend_type = gr.Radio(
                             choices=["Preview","Reasons"],
@@ -2148,7 +2162,7 @@ def create_ui():
                 recommend_score = gr.Radio(
                             choices=[1,2,3,4,5,6,7],
                             label="Score",
-                            interactive=True)             
+                            interactive=True,info='Please score the recommendation reasons')             
         
         with gr.Row():
             task_type = gr.Textbox(visible=False)
@@ -2269,7 +2283,7 @@ def create_ui():
         )
             language = gr.Dropdown(
             ['English', 'Chinese', 'French', "Spanish", "Arabic", "Portuguese", "Cantonese"],
-            value="Chinese", label="Language", interactive=True, elem_classes="custom-language"
+            value="English", label="Language", interactive=True, elem_classes="custom-language"
         )
             length = gr.Slider(
                                 minimum=60,
@@ -2333,7 +2347,7 @@ def create_ui():
         gallery_result.select(
             item_associate,
             inputs=[new_crop_save_path,openai_api_key,language,auto_play,length,log_state,sort_rec,naritive,recomended_state,recommend_type],
-            outputs=[recommend_bot,recomended_state,output_audio,log_state,pic_index,recommend_score],
+            outputs=[recommend_bot,recomended_state,output_audio,log_state,pic_index,recommend_score,selected_image],
             
             
         )
@@ -2341,7 +2355,7 @@ def create_ui():
         style_gallery_result.select(
             style_associate,
             inputs=[image_path,openai_api_key,language,auto_play,length,log_state,sort_rec,naritive,recomended_state,artist_label,recommend_type],
-            outputs=[recommend_bot,recomended_state,output_audio,log_state,pic_index,recommend_score],
+            outputs=[recommend_bot,recomended_state,output_audio,log_state,pic_index,recommend_score,selected_image],
             
             
         )
@@ -2646,7 +2660,7 @@ def create_ui():
         # sketcher_input.upload(upload_callback, [sketcher_input, state, visual_chatgpt, openai_api_key],
         #                       [chatbot, state, origin_image, click_state, image_input, image_input_base, sketcher_input,
         #                        image_embedding, original_size, input_size,name_label,artist_label,year_label,material_label,name_label_base, artist_label_base, year_label_base, material_label_base,paragraph,artist])
-        chat_input.submit(chat_input_callback, [visual_chatgpt, chat_input, click_state, state, aux_state,language,auto_play,gender,openai_api_key,image_path,log_state,history_log],
+        chat_input.submit(chat_input_callback, [visual_chatgpt, chat_input, click_state, state, aux_state,language,auto_play,gender,openai_api_key,image_path,log_state,history_log,naritive],
                           [chatbot, state, aux_state,output_audio,log_state,history_log])
         # chat_input.submit(lambda: "", None, chat_input)
         chat_input.submit(lambda: {"text": ""}, None, chat_input)
@@ -2661,9 +2675,9 @@ def create_ui():
 
         example_image.change(clear_chat_memory, inputs=[visual_chatgpt])
         example_image.change(
-            lambda:([],[],[],None,[],gr.update(value="Preview"),gr.update(value=[])), 
+            lambda:([],[],[],None,[],gr.update(value="Preview")), 
             [],
-            [gallery_result,style_gallery_result,recommend_bot,new_crop_save_path,chatbot,recommend_type,sort_rec])
+            [gallery_result,style_gallery_result,recommend_bot,new_crop_save_path,chatbot,recommend_type])
 
         # def on_click_tab_selected():
         #     if gpt_state ==1: