yaleh commited on
Commit
b01579f
·
1 Parent(s): ac847d0

Applied Gradio state for gradio_meta_prompt.

Browse files
Files changed (1) hide show
  1. app/gradio_meta_prompt.py +189 -408
app/gradio_meta_prompt.py CHANGED
@@ -166,91 +166,44 @@ def chat_log_2_chatbot_list(chat_log: str) -> List[List[str]]:
166
  print(line)
167
  return chatbot_list
168
 
169
-
170
- active_model_tab = "Simple"
171
-
172
- def on_model_tab_select(event: gr.SelectData):
173
- """
174
- Handles model tab selection events and updates the active model tab.
175
-
176
- Parameters:
177
- event (gr.SelectData): The select data event triggered by the user's action.
178
-
179
- Returns:
180
- None: This function doesn't return anything but updates the global variable 'active_model_tab'.
181
-
182
- """
183
- if not event.selected:
184
- return
185
-
186
- global active_model_tab
187
- active_model_tab = event.value
188
-
189
-
190
- def get_current_model(simple_model_name: str,
191
- advanced_model_name: str,
192
- expert_model_name: str,
193
- expert_model_config: Optional[Dict[str, Any]] = None) -> BaseLanguageModel:
194
- """
195
- Retrieve and return a language model (LLM) based on the currently active model tab.
196
-
197
- This function uses a mapping to associate model tab names with their corresponding
198
- model names. It then looks up the configuration for the selected model in the
199
- application's configuration, creates an instance of the appropriate type of language
200
- model using that configuration, and returns it. If the active model tab is not found
201
- in the mapping, the simple model will be used as a default.
202
-
203
- Args:
204
- simple_model_name (str): The name of the simple language model. This should
205
- correspond to a key in the 'llms' section of the application's configuration.
206
- advanced_model_name (str): The name of the advanced language model. This should
207
- correspond to a key in the 'llms' section of the application's configuration.
208
- expert_model_name (str): The name of the expert language model. This should
209
- correspond to a key in the 'llms' section of the application's configuration.
210
- expert_model_config (Optional[Dict[str, Any]]): Optional configuration for the
211
- expert model. This configuration will be used to update the model configuration
212
- if the active model tab is "Expert". Defaults to None.
213
-
214
- Returns:
215
- BaseLanguageModel: An instance of a language model that inherits from
216
- BaseLanguageModel, based on the currently active model tab and the provided
217
- model names.
218
-
219
- Raises:
220
- ValueError: If the active model tab is not found in the mapping or if the model
221
- name or configuration is invalid.
222
- RuntimeError: If an unexpected error occurs while retrieving the model.
223
- """
224
- model_mapping = {
225
- "Simple": simple_model_name,
226
- "Advanced": advanced_model_name,
227
- "Expert": expert_model_name
228
- }
229
-
230
- try:
231
- model_name = model_mapping.get(active_model_tab, simple_model_name)
232
- model = config.llms[model_name]
233
- model_type = model.type
234
- model_config = model.model_dump(exclude={'type'})
235
-
236
- # Update the configuration with the expert model configurations if provided
237
- if active_model_tab == "Expert" and expert_model_config:
238
- model_config.update(expert_model_config)
239
-
240
- return LLMModelFactory().create(model_type, **model_config)
241
-
242
- except KeyError as e:
243
- logging.error(f"Configuration key error: {e}")
244
- raise ValueError(f"Invalid model name or configuration: {e}")
245
 
246
- except Exception as e:
247
- logging.error(f"An unexpected error occurred: {e}")
248
- raise RuntimeError(f"Failed to retrieve the model: {e}")
249
-
250
-
251
- def evaluate_system_message(system_message, user_message, simple_model,
252
- advanced_executor_model, expert_executor_model,
253
- expert_executor_model_temperature=0.1):
254
  """
255
  Evaluate a system message by using it to generate a response from an
256
  executor model based on the current active tab and provided user message.
@@ -265,17 +218,8 @@ def evaluate_system_message(system_message, user_message, simple_model,
265
  response.
266
  user_message (str): The user's input message for which a response will
267
  be generated.
268
- simple_model (str): The name of the simple language model. This should
269
- correspond to a key in the 'llms' section of the application's
270
- configuration.
271
- advanced_executor_model (str): The name of the advanced language model.
272
- This should correspond to a key in the 'llms' section of the
273
- application's configuration.
274
- expert_executor_model (str): The name of the expert language model.
275
- This should correspond to a key in the 'llms' section of the
276
- application's configuration.
277
- expert_executor_model_temperature (float, optional): The temperature
278
- parameter for the expert executor model. Defaults to 0.1.
279
 
280
  Returns:
281
  str: The content of the output generated by the LLM based on the system
@@ -287,9 +231,7 @@ def evaluate_system_message(system_message, user_message, simple_model,
287
  Exception: For any other unexpected errors that occur during the
288
  execution of this function.
289
  """
290
- llm = get_current_model(simple_model, advanced_executor_model,
291
- expert_executor_model,
292
- {"temperature": expert_executor_model_temperature})
293
  template = ChatPromptTemplate.from_messages([
294
  ("system", "{system_message}"),
295
  ("human", "{user_message}")
@@ -304,11 +246,7 @@ def evaluate_system_message(system_message, user_message, simple_model,
304
  raise gr.Error(f"Error: {e}")
305
 
306
 
307
- def generate_acceptance_criteria(user_message, expected_output,
308
- simple_model, advanced_executor_model,
309
- expert_prompt_acceptance_criteria_model,
310
- expert_prompt_acceptance_criteria_temperature=0.1,
311
- prompt_template_group: Optional[str] = None):
312
  """
313
  Generate acceptance criteria based on the user message and expected output.
314
 
@@ -319,17 +257,12 @@ def generate_acceptance_criteria(user_message, expected_output,
319
  user_message (str): The user's input message.
320
  expected_output (str): The anticipated response or outcome from the language
321
  model based on the user's message.
322
- simple_model (str): The name of the simple language model.
323
- advanced_executor_model (str): The name of the advanced language model.
324
- expert_prompt_acceptance_criteria_model (str): The name of the expert language
325
- model.
326
- expert_prompt_acceptance_criteria_temperature (float, optional): The temperature
327
- parameter for the expert model. Defaults to 0.1.
328
  prompt_template_group (Optional[str], optional): The group of prompt templates
329
  to use. Defaults to None.
330
 
331
  Returns:
332
- str: The generated acceptance criteria.
333
  """
334
 
335
  log_stream = io.StringIO()
@@ -342,9 +275,7 @@ def generate_acceptance_criteria(user_message, expected_output,
342
  )
343
  logger.addHandler(log_handler)
344
 
345
- llm = get_current_model(simple_model, advanced_executor_model,
346
- expert_prompt_acceptance_criteria_model,
347
- {"temperature": expert_prompt_acceptance_criteria_temperature})
348
  if prompt_template_group is None:
349
  prompt_template_group = 'default'
350
  prompt_templates = prompt_templates_confz2langchain(
@@ -371,10 +302,7 @@ def generate_acceptance_criteria(user_message, expected_output,
371
  def generate_initial_system_message(
372
  user_message: str,
373
  expected_output: str,
374
- simple_model: str,
375
- advanced_executor_model: str,
376
- expert_prompt_initial_developer_model: str,
377
- expert_prompt_initial_developer_temperature: float = 0.1,
378
  prompt_template_group: Optional[str] = None
379
  ) -> tuple:
380
  """
@@ -383,11 +311,7 @@ def generate_initial_system_message(
383
  Args:
384
  user_message (str): The user's input message.
385
  expected_output (str): The anticipated response or outcome from the language model.
386
- simple_model (str): The name of the simple language model.
387
- advanced_executor_model (str): The name of the advanced language model.
388
- expert_prompt_initial_developer_model (str): The name of the expert language model.
389
- expert_prompt_initial_developer_temperature (float, optional):
390
- The temperature parameter for the expert model. Defaults to 0.1.
391
  prompt_template_group (Optional[str], optional):
392
  The group of prompt templates to use. Defaults to None.
393
 
@@ -405,12 +329,7 @@ def generate_initial_system_message(
405
  )
406
  logger.addHandler(log_handler)
407
 
408
- llm = get_current_model(
409
- simple_model,
410
- advanced_executor_model,
411
- expert_prompt_initial_developer_model,
412
- {"temperature": expert_prompt_initial_developer_temperature}
413
- )
414
 
415
  if prompt_template_group is None:
416
  prompt_template_group = 'default'
@@ -442,45 +361,40 @@ def generate_initial_system_message(
442
  return system_message, chat_log_2_chatbot_list(log_output)
443
 
444
 
445
- def process_message(
446
  user_message: str, expected_output: str, acceptance_criteria: str,
447
  initial_system_message: str, recursion_limit: int, max_output_age: int,
448
- llms: Union[BaseLanguageModel, Dict[str, BaseLanguageModel]],
 
 
449
  prompt_template_group: Optional[str] = None,
450
  aggressive_exploration: bool = False
451
  ) -> tuple:
452
  """
453
- Process a user message by executing the MetaPromptGraph with provided
454
- language models and input state.
455
 
456
- This function sets up the initial state of the conversation, logs the
457
- execution if verbose mode is enabled, and extracts the best system message,
458
- output, and analysis from the output state of the MetaPromptGraph.
459
 
460
  Args:
461
- user_message (str): The user's input message to be processed by the
462
- language model(s).
463
- expected_output (str): The anticipated response or outcome from the
464
- language model(s) based on the user's message.
465
- acceptance_criteria (str): Criteria that determines whether the output
466
- is acceptable or not.
467
- initial_system_message (str): Initial instruction given to the language
468
- model(s) before processing the user's message.
469
- recursion_limit (int): The maximum number of times the MetaPromptGraph
470
- can call itself recursively.
471
- max_output_age (int): The maximum age of output messages that should be
472
- considered in the conversation history.
473
- llms (Union[BaseLanguageModel, Dict[str, BaseLanguageModel]]): A single
474
- language model or a dictionary of language models to use for
475
- processing the user's message.
476
- prompt_template_group (Optional[str], optional): The group of prompt
477
- templates to use. Defaults to None.
478
- aggressive_exploration (bool, optional): Whether to use aggressive
479
- exploration. Defaults to False.
480
 
481
  Returns:
482
- tuple: A tuple containing the best system message, output, analysis,
483
- acceptance criteria, and chat log in JSON format.
484
  """
485
  input_state = AgentState(
486
  user_message=user_message,
@@ -501,6 +415,15 @@ def process_message(
501
  if prompt_template_group is None:
502
  prompt_template_group = 'default'
503
  prompt_templates = prompt_templates_confz2langchain(config.prompt_templates[prompt_template_group])
 
 
 
 
 
 
 
 
 
504
  meta_prompt_graph = MetaPromptGraph(llms=llms, prompts=prompt_templates,
505
  aggressive_exploration=aggressive_exploration,
506
  verbose=config.verbose, logger=logger)
@@ -573,219 +496,6 @@ def initialize_llm(model_name: str, model_config: Optional[Dict[str, Any]] = Non
573
  )
574
 
575
 
576
- def process_message_with_single_llm(
577
- user_message: str, expected_output: str, acceptance_criteria: str,
578
- initial_system_message: str, recursion_limit: int, max_output_age: int,
579
- model_name: str, prompt_template_group: Optional[str] = None,
580
- aggressive_exploration: bool = False
581
- ) -> tuple:
582
- """
583
- Process a user message using a single language model.
584
-
585
- This function initializes a language model based on the provided model name
586
- and uses it to process the user's message. The function takes in additional
587
- parameters such as the user's message, expected output, acceptance criteria,
588
- initial system message, recursion limit, and max output age. It then calls
589
- the `process_message` function with the initialized language model to obtain
590
- the best system message, output, analysis, and chat log.
591
-
592
- Parameters:
593
- user_message (str): The user's input message to be processed by the language
594
- model.
595
- expected_output (str): The anticipated response or outcome from the language
596
- model based on the user's message.
597
- acceptance_criteria (str): Criteria that determines whether the output is
598
- acceptable or not.
599
- initial_system_message (str): Initial instruction given to the language
600
- model before processing the user's message.
601
- recursion_limit (int): The maximum number of times the MetaPromptGraph can
602
- call itself recursively.
603
- max_output_age (int): The maximum age of output messages that should be
604
- considered in the conversation history.
605
- model_name (str): The name of the language model to initialize and use for
606
- processing the user's message. This should correspond to a key in the
607
- 'llms' section of the application's configuration.
608
- prompt_template_group (Optional[str], optional): The name of the prompt
609
- template group to use for processing the user's message. Defaults to None.
610
- aggressive_exploration (bool, optional): Whether to use aggressive
611
- exploration techniques. Defaults to False.
612
-
613
- Returns:
614
- tuple: A tuple containing the best system message, output, analysis, and
615
- chat log in JSON format.
616
- - best_system_message (str): The system message that resulted in the
617
- most appropriate response based on the acceptance criteria.
618
- - best_output (str): The output generated by the language model that
619
- best meets the expected outcome and acceptance criteria.
620
- - analysis (str): An analysis of how well the generated output
621
- matches the expected output and acceptance criteria.
622
- - chat_log (list): A list containing JSON objects representing the
623
- conversation log, with each object containing a timestamp, logger
624
- name, levelname, and message.
625
- """
626
- llm = initialize_llm(model_name)
627
- return process_message(
628
- user_message, expected_output, acceptance_criteria, initial_system_message,
629
- recursion_limit, max_output_age, llm, prompt_template_group, aggressive_exploration
630
- )
631
-
632
-
633
- def process_message_with_2_llms(
634
- user_message: str, expected_output: str, acceptance_criteria: str,
635
- initial_system_message: str, recursion_limit: int, max_output_age: int,
636
- optimizer_model_name: str, executor_model_name: str,
637
- prompt_template_group: Optional[str] = None,
638
- aggressive_exploration: bool = False
639
- ) -> tuple:
640
- """
641
- Process a user message using two language models - one for optimization and
642
- another for execution.
643
-
644
- This function initializes the specified optimizer and executor language
645
- models and then uses them to process the user's message along with other
646
- provided input parameters such as expected output, acceptance criteria,
647
- initial system message, recursion limit, and max output age. The result is
648
- obtained by calling the `process_message` function with a dictionary of
649
- language models where all nodes except for NODE_PROMPT_EXECUTOR use the
650
- optimizer model and NODE_PROMPT_EXECUTOR uses the executor model.
651
-
652
- Args:
653
- user_message (str): The user's input message to be processed by the
654
- language models.
655
- expected_output (str): The anticipated response or outcome from the
656
- language models based on the user's message.
657
- acceptance_criteria (str): Criteria that determines whether the output
658
- is acceptable or not.
659
- initial_system_message (str): Initial instruction given to the language
660
- models before processing the user's message.
661
- recursion_limit (int): The maximum number of times the MetaPromptGraph
662
- can call itself recursively.
663
- max_output_age (int): The maximum age of output messages that should be
664
- considered in the conversation history.
665
- optimizer_model_name (str): The name of the language model to initialize
666
- and use for optimization tasks like prompt development, analysis,
667
- and suggestion. This should correspond to a key in the 'llms' section
668
- of the application's configuration.
669
- executor_model_name (str): The name of the language model to initialize
670
- and use for execution tasks like running code or providing final
671
- outputs. This should correspond to a key in the 'llms' section of the
672
- application's configuration.
673
- prompt_template_group (Optional[str], optional): The name of the prompt
674
- template group to use for processing the user's message. Defaults to
675
- None.
676
- aggressive_exploration (bool, optional): Whether to use aggressive
677
- exploration techniques. Defaults to False.
678
-
679
- Returns:
680
- tuple: A tuple containing the best system message, output, analysis, and
681
- chat log in JSON format.
682
- - best_system_message (str): The system message that resulted in the
683
- most appropriate response based on the acceptance criteria.
684
- - best_output (str): The output generated by the language models that
685
- best meets the expected outcome and acceptance criteria.
686
- - analysis (str): An analysis of how well the generated output
687
- matches the expected output and acceptance criteria.
688
- - chat_log (list): A list containing JSON objects representing the
689
- conversation log, with each object containing a timestamp,
690
- logger name, levelname, and message.
691
- """
692
- optimizer_model = initialize_llm(optimizer_model_name)
693
- executor_model = initialize_llm(executor_model_name)
694
- llms = {
695
- NODE_ACCEPTANCE_CRITERIA_DEVELOPER: optimizer_model,
696
- NODE_PROMPT_INITIAL_DEVELOPER: optimizer_model,
697
- NODE_PROMPT_DEVELOPER: optimizer_model,
698
- NODE_PROMPT_EXECUTOR: executor_model,
699
- NODE_OUTPUT_HISTORY_ANALYZER: optimizer_model,
700
- NODE_PROMPT_ANALYZER: optimizer_model,
701
- NODE_PROMPT_SUGGESTER: optimizer_model
702
- }
703
- return process_message(
704
- user_message, expected_output, acceptance_criteria,
705
- initial_system_message, recursion_limit, max_output_age, llms,
706
- prompt_template_group, aggressive_exploration
707
- )
708
-
709
-
710
- def process_message_with_expert_llms(
711
- user_message: str, expected_output: str, acceptance_criteria: str,
712
- initial_system_message: str, recursion_limit: int, max_output_age: int,
713
- initial_developer_model_name: str, initial_developer_temperature: float,
714
- acceptance_criteria_model_name: str, acceptance_criteria_temperature: float,
715
- developer_model_name: str, developer_temperature: float,
716
- executor_model_name: str, executor_temperature: float,
717
- output_history_analyzer_model_name: str, output_history_analyzer_temperature: float,
718
- analyzer_model_name: str, analyzer_temperature: float,
719
- suggester_model_name: str, suggester_temperature: float,
720
- prompt_template_group: Optional[str] = None, aggressive_exploration: bool = False
721
- ) -> tuple:
722
- """
723
- Process a message using expert language models with specified temperatures.
724
-
725
- Args:
726
- user_message (str): The user's input message.
727
- expected_output (str): The anticipated response or outcome from the language model.
728
- acceptance_criteria (str): Criteria for accepting the generated output.
729
- initial_system_message (str): The initial system message to use.
730
- recursion_limit (int): The maximum number of recursive calls.
731
- max_output_age (int): The maximum age of output messages to consider.
732
- initial_developer_model_name (str): The name of the initial developer model.
733
- initial_developer_temperature (float): The temperature for the initial developer model.
734
- acceptance_criteria_model_name (str): The name of the acceptance criteria model.
735
- acceptance_criteria_temperature (float): The temperature for the acceptance criteria model.
736
- developer_model_name (str): The name of the developer model.
737
- developer_temperature (float): The temperature for the developer model.
738
- executor_model_name (str): The name of the executor model.
739
- executor_temperature (float): The temperature for the executor model.
740
- output_history_analyzer_model_name (str): The name of the output history analyzer model.
741
- output_history_analyzer_temperature (float): The temperature for the output history analyzer model.
742
- analyzer_model_name (str): The name of the analyzer model.
743
- analyzer_temperature (float): The temperature for the analyzer model.
744
- suggester_model_name (str): The name of the suggester model.
745
- suggester_temperature (float): The temperature for the suggester model.
746
- prompt_template_group (Optional[str], optional): The group of prompt templates to use. Defaults to None.
747
- aggressive_exploration (bool, optional): Whether to use aggressive exploration. Defaults to False.
748
-
749
- Returns:
750
- tuple: A tuple containing the processed message results.
751
- """
752
- llms = {
753
- NODE_PROMPT_INITIAL_DEVELOPER: initialize_llm(
754
- initial_developer_model_name, {"temperature": initial_developer_temperature}
755
- ),
756
- NODE_ACCEPTANCE_CRITERIA_DEVELOPER: initialize_llm(
757
- acceptance_criteria_model_name, {"temperature": acceptance_criteria_temperature}
758
- ),
759
- NODE_PROMPT_DEVELOPER: initialize_llm(
760
- developer_model_name, {"temperature": developer_temperature}
761
- ),
762
- NODE_PROMPT_EXECUTOR: initialize_llm(
763
- executor_model_name, {"temperature": executor_temperature}
764
- ),
765
- NODE_OUTPUT_HISTORY_ANALYZER: initialize_llm(
766
- output_history_analyzer_model_name,
767
- {"temperature": output_history_analyzer_temperature}
768
- ),
769
- NODE_PROMPT_ANALYZER: initialize_llm(
770
- analyzer_model_name, {"temperature": analyzer_temperature}
771
- ),
772
- NODE_PROMPT_SUGGESTER: initialize_llm(
773
- suggester_model_name, {"temperature": suggester_temperature}
774
- )
775
- }
776
- return process_message(
777
- user_message,
778
- expected_output,
779
- acceptance_criteria,
780
- initial_system_message,
781
- recursion_limit,
782
- max_output_age,
783
- llms,
784
- prompt_template_group,
785
- aggressive_exploration
786
- )
787
-
788
-
789
  class FileConfig(BaseConfig):
790
  config_file: str = 'config.yml' # default path
791
 
@@ -1028,28 +738,101 @@ with gr.Blocks(title='Meta Prompt') as demo:
1028
  simple_model_name_input
1029
  ])
1030
 
 
 
 
 
 
 
 
 
 
 
1031
  # set up event handlers
1032
- simple_llm_tab.select(on_model_tab_select)
1033
- advanced_llm_tab.select(on_model_tab_select)
1034
- expert_llm_tab.select(on_model_tab_select)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1035
 
1036
  generate_acceptance_criteria_button.click(
1037
  generate_acceptance_criteria,
1038
  inputs=[user_message_input, expected_output_input,
1039
- simple_model_name_input,
1040
- advanced_optimizer_model_name_input,
1041
- expert_prompt_acceptance_criteria_model_name_input,
1042
- expert_prompt_acceptance_criteria_temperature_input,
1043
  prompt_template_group],
1044
  outputs=[acceptance_criteria_input, logs_chatbot]
1045
  )
1046
  generate_initial_system_message_button.click(
1047
  generate_initial_system_message,
1048
  inputs=[user_message_input, expected_output_input,
1049
- simple_model_name_input,
1050
- advanced_optimizer_model_name_input,
1051
- expert_prompt_initial_developer_model_name_input,
1052
- expert_prompt_initial_developer_temperature_input,
1053
  prompt_template_group],
1054
  outputs=[initial_system_message_input, logs_chatbot]
1055
  )
@@ -1059,10 +842,7 @@ with gr.Blocks(title='Meta Prompt') as demo:
1059
  inputs=[
1060
  initial_system_message_input,
1061
  user_message_input,
1062
- simple_model_name_input,
1063
- advanced_executor_model_name_input,
1064
- expert_prompt_executor_model_name_input,
1065
- expert_prompt_executor_temperature_input
1066
  ],
1067
  outputs=[output_output]
1068
  )
@@ -1071,10 +851,7 @@ with gr.Blocks(title='Meta Prompt') as demo:
1071
  inputs=[
1072
  system_message_output,
1073
  user_message_input,
1074
- simple_model_name_input,
1075
- advanced_executor_model_name_input,
1076
- expert_prompt_executor_model_name_input,
1077
- expert_prompt_executor_temperature_input
1078
  ],
1079
  outputs=[output_output]
1080
  )
@@ -1090,7 +867,7 @@ with gr.Blocks(title='Meta Prompt') as demo:
1090
  analysis_output, logs_chatbot])
1091
 
1092
  simple_submit_button.click(
1093
- process_message_with_single_llm,
1094
  inputs=[
1095
  user_message_input,
1096
  expected_output_input,
@@ -1098,7 +875,13 @@ with gr.Blocks(title='Meta Prompt') as demo:
1098
  initial_system_message_input,
1099
  recursion_limit_input,
1100
  max_output_age,
1101
- simple_model_name_input,
 
 
 
 
 
 
1102
  prompt_template_group,
1103
  aggressive_exploration
1104
  ],
@@ -1112,7 +895,7 @@ with gr.Blocks(title='Meta Prompt') as demo:
1112
  )
1113
 
1114
  advanced_submit_button.click(
1115
- process_message_with_2_llms,
1116
  inputs=[
1117
  user_message_input,
1118
  expected_output_input,
@@ -1120,8 +903,13 @@ with gr.Blocks(title='Meta Prompt') as demo:
1120
  initial_system_message_input,
1121
  recursion_limit_input,
1122
  max_output_age,
1123
- advanced_optimizer_model_name_input,
1124
- advanced_executor_model_name_input,
 
 
 
 
 
1125
  prompt_template_group,
1126
  aggressive_exploration
1127
  ],
@@ -1135,7 +923,7 @@ with gr.Blocks(title='Meta Prompt') as demo:
1135
  )
1136
 
1137
  expert_submit_button.click(
1138
- process_message_with_expert_llms,
1139
  inputs=[
1140
  user_message_input,
1141
  expected_output_input,
@@ -1143,20 +931,13 @@ with gr.Blocks(title='Meta Prompt') as demo:
1143
  initial_system_message_input,
1144
  recursion_limit_input,
1145
  max_output_age,
1146
- expert_prompt_initial_developer_model_name_input,
1147
- expert_prompt_initial_developer_temperature_input,
1148
- expert_prompt_acceptance_criteria_model_name_input,
1149
- expert_prompt_acceptance_criteria_temperature_input,
1150
- expert_prompt_developer_model_name_input,
1151
- expert_prompt_developer_temperature_input,
1152
- expert_prompt_executor_model_name_input,
1153
- expert_prompt_executor_temperature_input,
1154
- expert_output_history_analyzer_model_name_input,
1155
- expert_output_history_analyzer_temperature_input,
1156
- expert_prompt_analyzer_model_name_input,
1157
- expert_prompt_analyzer_temperature_input,
1158
- expert_prompt_suggester_model_name_input,
1159
- expert_prompt_suggester_temperature_input,
1160
  prompt_template_group,
1161
  aggressive_exploration
1162
  ],
 
166
  print(line)
167
  return chatbot_list
168
 
169
+ def on_model_tab_select(simple_model_name,
170
+ advanced_optimizer_model_name, advanced_executor_model_name,
171
+ expert_prompt_initial_developer_model_name,
172
+ expert_prompt_acceptance_criteria_developer_model_name,
173
+ expert_prompt_developer_model_name,
174
+ expert_prompt_executor_model_name,
175
+ expert_prompt_history_analyzer_model_name,
176
+ expert_prompt_analyzer_model_name,
177
+ expert_prompt_suggester_model_name,
178
+ event: gr.SelectData):
179
+ if event.value == 'Simple':
180
+ return simple_model_name, \
181
+ simple_model_name, \
182
+ simple_model_name, \
183
+ simple_model_name, \
184
+ simple_model_name, \
185
+ simple_model_name, \
186
+ simple_model_name
187
+ elif event.value == 'Advanced':
188
+ return advanced_optimizer_model_name, \
189
+ advanced_optimizer_model_name, \
190
+ advanced_optimizer_model_name, \
191
+ advanced_executor_model_name, \
192
+ advanced_optimizer_model_name, \
193
+ advanced_optimizer_model_name, \
194
+ advanced_optimizer_model_name
195
+ elif event.value == 'Expert':
196
+ return expert_prompt_initial_developer_model_name, \
197
+ expert_prompt_acceptance_criteria_developer_model_name, \
198
+ expert_prompt_developer_model_name, \
199
+ expert_prompt_executor_model_name, \
200
+ expert_prompt_history_analyzer_model_name, \
201
+ expert_prompt_analyzer_model_name, \
202
+ expert_prompt_suggester_model_name
203
+ else:
204
+ raise ValueError(f"Invalid model tab selected: {event.value}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
205
 
206
+ def evaluate_system_message(system_message, user_message, executor_model_name):
 
 
 
 
 
 
 
207
  """
208
  Evaluate a system message by using it to generate a response from an
209
  executor model based on the current active tab and provided user message.
 
218
  response.
219
  user_message (str): The user's input message for which a response will
220
  be generated.
221
+ executor_model_state (gr.State): The state object containing the name
222
+ of the executor model to use.
 
 
 
 
 
 
 
 
 
223
 
224
  Returns:
225
  str: The content of the output generated by the LLM based on the system
 
231
  Exception: For any other unexpected errors that occur during the
232
  execution of this function.
233
  """
234
+ llm = initialize_llm(executor_model_name)
 
 
235
  template = ChatPromptTemplate.from_messages([
236
  ("system", "{system_message}"),
237
  ("human", "{user_message}")
 
246
  raise gr.Error(f"Error: {e}")
247
 
248
 
249
+ def generate_acceptance_criteria(user_message, expected_output, acceptance_criteria_model_name, prompt_template_group):
 
 
 
 
250
  """
251
  Generate acceptance criteria based on the user message and expected output.
252
 
 
257
  user_message (str): The user's input message.
258
  expected_output (str): The anticipated response or outcome from the language
259
  model based on the user's message.
260
+ acceptance_criteria_model_name (str): The name of the acceptance criteria model to use.
 
 
 
 
 
261
  prompt_template_group (Optional[str], optional): The group of prompt templates
262
  to use. Defaults to None.
263
 
264
  Returns:
265
+ tuple: A tuple containing the generated acceptance criteria and the chat log.
266
  """
267
 
268
  log_stream = io.StringIO()
 
275
  )
276
  logger.addHandler(log_handler)
277
 
278
+ llm = initialize_llm(acceptance_criteria_model_name)
 
 
279
  if prompt_template_group is None:
280
  prompt_template_group = 'default'
281
  prompt_templates = prompt_templates_confz2langchain(
 
302
  def generate_initial_system_message(
303
  user_message: str,
304
  expected_output: str,
305
+ initial_developer_model_name: str,
 
 
 
306
  prompt_template_group: Optional[str] = None
307
  ) -> tuple:
308
  """
 
311
  Args:
312
  user_message (str): The user's input message.
313
  expected_output (str): The anticipated response or outcome from the language model.
314
+ initial_developer_model_name (str): The name of the initial developer model to use.
 
 
 
 
315
  prompt_template_group (Optional[str], optional):
316
  The group of prompt templates to use. Defaults to None.
317
 
 
329
  )
330
  logger.addHandler(log_handler)
331
 
332
+ llm = initialize_llm(initial_developer_model_name)
 
 
 
 
 
333
 
334
  if prompt_template_group is None:
335
  prompt_template_group = 'default'
 
361
  return system_message, chat_log_2_chatbot_list(log_output)
362
 
363
 
364
+ def process_message_with_models(
365
  user_message: str, expected_output: str, acceptance_criteria: str,
366
  initial_system_message: str, recursion_limit: int, max_output_age: int,
367
+ initial_developer_model_name: str, acceptance_criteria_model_name: str,
368
+ developer_model_name: str, executor_model_name: str, history_analyzer_model_name: str,
369
+ analyzer_model_name: str, suggester_model_name: str,
370
  prompt_template_group: Optional[str] = None,
371
  aggressive_exploration: bool = False
372
  ) -> tuple:
373
  """
374
+ Process a user message by executing the MetaPromptGraph with provided language models and input state.
 
375
 
376
+ This function sets up the initial state of the conversation, logs the execution if verbose mode is enabled,
377
+ and extracts the best system message, output, and analysis from the output state of the MetaPromptGraph.
 
378
 
379
  Args:
380
+ user_message (str): The user's input message to be processed by the language model(s).
381
+ expected_output (str): The anticipated response or outcome from the language model(s) based on the user's message.
382
+ acceptance_criteria (str): Criteria that determines whether the output is acceptable or not.
383
+ initial_system_message (str): Initial instruction given to the language model(s) before processing the user's message.
384
+ recursion_limit (int): The maximum number of times the MetaPromptGraph can call itself recursively.
385
+ max_output_age (int): The maximum age of output messages that should be considered in the conversation history.
386
+ initial_developer_model_name (str): The name of the initial developer model to use.
387
+ acceptance_criteria_model_name (str): The name of the acceptance criteria model to use.
388
+ developer_model_name (str): The name of the developer model to use.
389
+ executor_model_name (str): The name of the executor model to use.
390
+ history_analyzer_model_name (str): The name of the history analyzer model to use.
391
+ analyzer_model_name (str): The name of the analyzer model to use.
392
+ suggester_model_name (str): The name of the suggester model to use.
393
+ prompt_template_group (Optional[str], optional): The group of prompt templates to use. Defaults to None.
394
+ aggressive_exploration (bool, optional): Whether to use aggressive exploration. Defaults to False.
 
 
 
 
395
 
396
  Returns:
397
+ tuple: A tuple containing the best system message, output, analysis, acceptance criteria, and chat log in JSON format.
 
398
  """
399
  input_state = AgentState(
400
  user_message=user_message,
 
415
  if prompt_template_group is None:
416
  prompt_template_group = 'default'
417
  prompt_templates = prompt_templates_confz2langchain(config.prompt_templates[prompt_template_group])
418
+ llms = {
419
+ NODE_PROMPT_INITIAL_DEVELOPER: initialize_llm(initial_developer_model_name),
420
+ NODE_ACCEPTANCE_CRITERIA_DEVELOPER: initialize_llm(acceptance_criteria_model_name),
421
+ NODE_PROMPT_DEVELOPER: initialize_llm(developer_model_name),
422
+ NODE_PROMPT_EXECUTOR: initialize_llm(executor_model_name),
423
+ NODE_OUTPUT_HISTORY_ANALYZER: initialize_llm(history_analyzer_model_name),
424
+ NODE_PROMPT_ANALYZER: initialize_llm(analyzer_model_name),
425
+ NODE_PROMPT_SUGGESTER: initialize_llm(suggester_model_name)
426
+ }
427
  meta_prompt_graph = MetaPromptGraph(llms=llms, prompts=prompt_templates,
428
  aggressive_exploration=aggressive_exploration,
429
  verbose=config.verbose, logger=logger)
 
496
  )
497
 
498
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
499
  class FileConfig(BaseConfig):
500
  config_file: str = 'config.yml' # default path
501
 
 
738
  simple_model_name_input
739
  ])
740
 
741
+ model_states = {
742
+ "initial_developer": gr.State(value=None), # None | str
743
+ "acceptance_criteria": gr.State(value=None), # None | str
744
+ "developer": gr.State(value=None), # None | str
745
+ "executor": gr.State(value=None), # None | str
746
+ "history_analyzer": gr.State(value=None), # None | str
747
+ "analyzer": gr.State(value=None), # None | str
748
+ "suggester": gr.State(value=None) # None | str
749
+ }
750
+
751
  # set up event handlers
752
+ simple_llm_tab.select(
753
+ on_model_tab_select,
754
+ [
755
+ simple_model_name_input,
756
+ advanced_optimizer_model_name_input,
757
+ advanced_executor_model_name_input,
758
+ expert_prompt_initial_developer_model_name_input,
759
+ expert_prompt_acceptance_criteria_model_name_input,
760
+ expert_prompt_developer_model_name_input,
761
+ expert_prompt_executor_model_name_input,
762
+ expert_output_history_analyzer_model_name_input,
763
+ expert_prompt_analyzer_model_name_input,
764
+ expert_prompt_suggester_model_name_input
765
+ ],
766
+ [
767
+ model_states["initial_developer"],
768
+ model_states["acceptance_criteria"],
769
+ model_states["developer"],
770
+ model_states["executor"],
771
+ model_states["history_analyzer"],
772
+ model_states["analyzer"],
773
+ model_states["suggester"]
774
+ ]
775
+ )
776
+ advanced_llm_tab.select(
777
+ on_model_tab_select,
778
+ [
779
+ simple_model_name_input,
780
+ advanced_optimizer_model_name_input,
781
+ advanced_executor_model_name_input,
782
+ expert_prompt_initial_developer_model_name_input,
783
+ expert_prompt_acceptance_criteria_model_name_input,
784
+ expert_prompt_developer_model_name_input,
785
+ expert_prompt_executor_model_name_input,
786
+ expert_output_history_analyzer_model_name_input,
787
+ expert_prompt_analyzer_model_name_input,
788
+ expert_prompt_suggester_model_name_input
789
+ ],
790
+ [
791
+ model_states["initial_developer"],
792
+ model_states["acceptance_criteria"],
793
+ model_states["developer"],
794
+ model_states["executor"],
795
+ model_states["history_analyzer"],
796
+ model_states["analyzer"],
797
+ model_states["suggester"]
798
+ ]
799
+ )
800
+ expert_llm_tab.select(
801
+ on_model_tab_select,
802
+ [
803
+ simple_model_name_input,
804
+ advanced_optimizer_model_name_input,
805
+ advanced_executor_model_name_input,
806
+ expert_prompt_initial_developer_model_name_input,
807
+ expert_prompt_acceptance_criteria_model_name_input,
808
+ expert_prompt_developer_model_name_input,
809
+ expert_prompt_executor_model_name_input,
810
+ expert_output_history_analyzer_model_name_input,
811
+ expert_prompt_analyzer_model_name_input,
812
+ expert_prompt_suggester_model_name_input
813
+ ],
814
+ [
815
+ model_states["initial_developer"],
816
+ model_states["acceptance_criteria"],
817
+ model_states["developer"],
818
+ model_states["executor"],
819
+ model_states["history_analyzer"],
820
+ model_states["analyzer"],
821
+ model_states["suggester"]
822
+ ]
823
+ )
824
 
825
  generate_acceptance_criteria_button.click(
826
  generate_acceptance_criteria,
827
  inputs=[user_message_input, expected_output_input,
828
+ model_states["acceptance_criteria"],
 
 
 
829
  prompt_template_group],
830
  outputs=[acceptance_criteria_input, logs_chatbot]
831
  )
832
  generate_initial_system_message_button.click(
833
  generate_initial_system_message,
834
  inputs=[user_message_input, expected_output_input,
835
+ model_states["initial_developer"],
 
 
 
836
  prompt_template_group],
837
  outputs=[initial_system_message_input, logs_chatbot]
838
  )
 
842
  inputs=[
843
  initial_system_message_input,
844
  user_message_input,
845
+ model_states["executor"]
 
 
 
846
  ],
847
  outputs=[output_output]
848
  )
 
851
  inputs=[
852
  system_message_output,
853
  user_message_input,
854
+ model_states["executor"]
 
 
 
855
  ],
856
  outputs=[output_output]
857
  )
 
867
  analysis_output, logs_chatbot])
868
 
869
  simple_submit_button.click(
870
+ process_message_with_models,
871
  inputs=[
872
  user_message_input,
873
  expected_output_input,
 
875
  initial_system_message_input,
876
  recursion_limit_input,
877
  max_output_age,
878
+ model_states["initial_developer"],
879
+ model_states["acceptance_criteria"],
880
+ model_states["developer"],
881
+ model_states["executor"],
882
+ model_states["history_analyzer"],
883
+ model_states["analyzer"],
884
+ model_states["suggester"],
885
  prompt_template_group,
886
  aggressive_exploration
887
  ],
 
895
  )
896
 
897
  advanced_submit_button.click(
898
+ process_message_with_models,
899
  inputs=[
900
  user_message_input,
901
  expected_output_input,
 
903
  initial_system_message_input,
904
  recursion_limit_input,
905
  max_output_age,
906
+ model_states["initial_developer"],
907
+ model_states["acceptance_criteria"],
908
+ model_states["developer"],
909
+ model_states["executor"],
910
+ model_states["history_analyzer"],
911
+ model_states["analyzer"],
912
+ model_states["suggester"],
913
  prompt_template_group,
914
  aggressive_exploration
915
  ],
 
923
  )
924
 
925
  expert_submit_button.click(
926
+ process_message_with_models,
927
  inputs=[
928
  user_message_input,
929
  expected_output_input,
 
931
  initial_system_message_input,
932
  recursion_limit_input,
933
  max_output_age,
934
+ model_states["initial_developer"],
935
+ model_states["acceptance_criteria"],
936
+ model_states["developer"],
937
+ model_states["executor"],
938
+ model_states["history_analyzer"],
939
+ model_states["analyzer"],
940
+ model_states["suggester"],
 
 
 
 
 
 
 
941
  prompt_template_group,
942
  aggressive_exploration
943
  ],