DISCLAIMERS:
etc. added in chat, try to remove some of that to help avoid dup entries when hit new conversation is_same = True # length of conversation has to be same if len(x) != len(y): return False if len(x) != len(y): return False for stepx, stepy in zip(x, y): if len(stepx) != len(stepy): # something off with a conversation return False for stepxx, stepyy in zip(stepx, stepy): if len(stepxx) != len(stepyy): # something off with a conversation return False if len(stepxx) != 2: # something off return False if len(stepyy) != 2: # something off return False questionx = str(stepxx[0]).replace('
', '').replace('
', '') if stepxx[0] is not None else None answerx = str(stepxx[1]).replace('', '').replace('
', '') if stepxx[1] is not None else None questiony = str(stepyy[0]).replace('', '').replace('
', '') if stepyy[0] is not None else None answery = str(stepyy[1]).replace('', '').replace('
', '') if stepyy[1] is not None else None if questionx != questiony or answerx != answery: return False return is_same def save_chat(*args, chat_is_list=False, auth_filename=None, auth_freeze=None, raise_if_none=True): args_list = list(args) db1s = args_list[0] requests_state1 = args_list[1] args_list = args_list[2:] if not chat_is_list: # list of chatbot histories, # can't pass in list with list of chatbot histories and state due to gradio limits chat_list = args_list[:-1] else: assert len(args_list) == 2 chat_list = args_list[0] # if old chat file with single chatbot, get into shape if isinstance(chat_list, list) and len(chat_list) > 0 and isinstance(chat_list[0], list) and len( chat_list[0]) == 2 and isinstance(chat_list[0][0], str) and isinstance(chat_list[0][1], str): chat_list = [chat_list] # remove None histories chat_list_not_none = [x for x in chat_list if x and len(x) > 0 and len(x[0]) == 2 and x[0][1] is not None] chat_list_none = [x for x in chat_list if x not in chat_list_not_none] if len(chat_list_none) > 0 and len(chat_list_not_none) == 0: if raise_if_none: raise ValueError("Invalid chat file") else: chat_state1 = args_list[-1] choices = list(chat_state1.keys()).copy() return chat_state1, gr.update(choices=choices, value=None) # dict with keys of short chat names, values of list of list of chatbot histories chat_state1 = args_list[-1] short_chats = list(chat_state1.keys()) if len(chat_list_not_none) > 0: # make short_chat key from only first history, based upon question that is same anyways chat_first = chat_list_not_none[0] short_chat = get_short_chat(chat_first, short_chats) if short_chat: old_chat_lists = list(chat_state1.values()) already_exists = any([is_chat_same(chat_list, x) for x in old_chat_lists]) if not already_exists: chat_state1[short_chat] = chat_list.copy() # reverse so newest at top choices = list(chat_state1.keys()).copy() choices.reverse() # save saved chats and chatbots to auth file selection_docs_state1 = None langchain_mode2 = None roles_state1 = None model_options_state1 = None lora_options_state1 = None server_options_state1 = None text_output1 = chat_list[0] text_output21 = chat_list[1] text_outputs1 = chat_list[2:] h2ogpt_key2, visible_models2 = None, None save_auth_func(selection_docs_state1, requests_state1, roles_state1, model_options_state1, lora_options_state1, server_options_state1, chat_state1, langchain_mode2, h2ogpt_key2, visible_models2, None, None, None, None, None, None, None, None, None, None, None, None, None, None, text_output1, text_output21, text_outputs1, ) return chat_state1, gr.update(choices=choices, value=None) def switch_chat(chat_key, chat_state1, num_model_lock=0): chosen_chat = chat_state1[chat_key] # deal with possible different size of chat list vs. current list ret_chat = [None] * (2 + num_model_lock) for chati in range(0, 2 + num_model_lock): ret_chat[chati % len(ret_chat)] = chosen_chat[chati % len(chosen_chat)] return tuple(ret_chat) def clear_texts(*args): return tuple([[]] * len(args)) def clear_scores(): return gr.Textbox(value=res_value), \ gr.Textbox(value='Response Score: NA'), \ gr.Textbox(value='Response Score: NA') switch_chat_fun = functools.partial(switch_chat, num_model_lock=len(text_outputs)) radio_chats.input(switch_chat_fun, inputs=[radio_chats, chat_state], outputs=[text_output, text_output2] + text_outputs) \ .then(clear_scores, outputs=[score_text, score_text2, score_text_nochat]) \ .then(**save_auth_kwargs) def remove_chat(chat_key, chat_state1): if isinstance(chat_key, str): chat_state1.pop(chat_key, None) return gr.update(choices=list(chat_state1.keys()), value=None), chat_state1 remove_chat_event = remove_chat_btn.click(remove_chat, inputs=[radio_chats, chat_state], outputs=[radio_chats, chat_state], **noqueue_kwargs, api_name='remove_chat') def get_chats1(chat_state1): base = 'chats' base = makedirs(base, exist_ok=True, tmp_ok=True, use_base=True) filename = os.path.join(base, 'chats_%s.json' % str(uuid.uuid4())) with open(filename, "wt") as f: f.write(json.dumps(chat_state1, indent=2)) return filename export_chat_event = export_chats_btn.click(get_chats1, inputs=chat_state, outputs=chats_file, **noqueue_kwargs2, api_name='export_chats' if allow_api else False) def add_chats_from_file(db1s, requests_state1, file, chat_state1, radio_chats1, chat_exception_text1, auth_filename=None, auth_freeze=None): if not file: return None, chat_state1, gr.update(choices=list(chat_state1.keys()), value=None), chat_exception_text1 if isinstance(file, str): files = [file] else: files = file if not files: return None, chat_state1, gr.update(choices=list(chat_state1.keys()), value=None), chat_exception_text1 chat_exception_list = [] for file1 in files: try: if hasattr(file1, 'name'): file1 = file1.name with open(file1, "rt") as f: new_chats = json.loads(f.read()) for chat1_k, chat1_v in new_chats.items(): # ignore chat1_k, regenerate and de-dup to avoid loss chat_state1, _ = save_chat(db1s, requests_state1, chat1_v, chat_state1, chat_is_list=True, raise_if_none=True) except BaseException as e: t, v, tb = sys.exc_info() ex = ''.join(traceback.format_exception(t, v, tb)) ex_str = "File %s exception: %s" % (file1, str(e)) print(ex_str, flush=True) chat_exception_list.append(ex_str) chat_exception_text1 = '\n'.join(chat_exception_list) # save chat to auth file selection_docs_state1 = None langchain_mode2 = None roles_state1 = None model_options_state1 = None lora_options_state1 = None server_options_state1 = None text_output1, text_output21, text_outputs1 = None, None, None h2ogpt_key2, visible_models2 = None, None save_auth_func(selection_docs_state1, requests_state1, roles_state1, model_options_state1, lora_options_state1, server_options_state1, chat_state1, langchain_mode2, h2ogpt_key2, visible_models2, None, None, None, None, None, None, None, None, None, None, None, None, None, None, text_output1, text_output21, text_outputs1, ) return None, chat_state1, gr.update(choices=list(chat_state1.keys()), value=None), chat_exception_text1 # note for update_user_db_func output is ignored for db chatup_change_eventa = chatsup_output.change(user_state_setup, inputs=[my_db_state, requests_state, guest_name, langchain_mode], outputs=[my_db_state, requests_state, langchain_mode], show_progress='minimal') add_chats_from_file_func = functools.partial(add_chats_from_file, auth_filename=kwargs['auth_filename'], auth_freeze=kwargs['auth_freeze'], ) chatup_change_event = chatup_change_eventa.then(add_chats_from_file_func, inputs=[my_db_state, requests_state] + [chatsup_output, chat_state, radio_chats, chat_exception_text], outputs=[chatsup_output, chat_state, radio_chats, chat_exception_text], **noqueue_kwargs, api_name='add_to_chats' if allow_api else False) clear_chat_event = clear_chat_btn.click(fn=clear_texts, inputs=[text_output, text_output2] + text_outputs, outputs=[text_output, text_output2] + text_outputs, **noqueue_kwargs, api_name='clear' if allow_api else False) \ .then(deselect_radio_chats, inputs=None, outputs=radio_chats, **noqueue_kwargs) \ .then(clear_scores, outputs=[score_text, score_text2, score_text_nochat]) clear_eventa = save_chat_btn.click(user_state_setup, inputs=[my_db_state, requests_state, guest_name, langchain_mode], outputs=[my_db_state, requests_state, langchain_mode], show_progress='minimal', **noqueue_kwargs2) save_chat_func = functools.partial(save_chat, auth_filename=kwargs['auth_filename'], auth_freeze=kwargs['auth_freeze'], raise_if_none=False, ) clear_event = clear_eventa.then(save_chat_func, inputs=[my_db_state, requests_state] + [text_output, text_output2] + text_outputs + [chat_state], outputs=[chat_state, radio_chats], api_name='save_chat' if allow_api else False) if kwargs['score_model']: clear_event2 = clear_event.then(clear_scores, outputs=[score_text, score_text2, score_text_nochat]) # NOTE: clear of instruction/iinput for nochat has to come after score, # because score for nochat consumes actual textbox, while chat consumes chat history filled by user() no_chat_args = dict(fn=fun, inputs=[model_state, my_db_state, selection_docs_state, requests_state, roles_state] + inputs_list, outputs=text_output_nochat, queue=queue, ) submit_event_nochat = submit_nochat.click(**no_chat_args, api_name='submit_nochat' if allow_api else False) \ .then(**score_args_nochat, api_name='instruction_bot_score_nochat' if allow_api else False, queue=queue) \ .then(clear_instruct, None, instruction_nochat) \ .then(clear_instruct, None, iinput_nochat) # copy of above with text box submission submit_event_nochat2 = instruction_nochat.submit(**no_chat_args) \ .then(**score_args_nochat, queue=queue) \ .then(clear_instruct, None, instruction_nochat) \ .then(clear_instruct, None, iinput_nochat) submit_event_nochat_api = submit_nochat_api.click(fun_with_dict_str, inputs=[model_state, my_db_state, selection_docs_state, requests_state, roles_state, inputs_dict_str], outputs=text_output_nochat_api, queue=True, # required for generator api_name='submit_nochat_api' if allow_api else False) submit_event_nochat_api_plain = submit_nochat_api_plain.click(fun_with_dict_str_plain, inputs=inputs_dict_str, outputs=text_output_nochat_api, **noqueue_kwargs_curl, api_name='submit_nochat_plain_api' if allow_api else False) submit_event_verifier = submit_verifier.click(fun_with_dict_verifier, inputs=verifier_inputs_dict_str, outputs=text_output_verifier, **noqueue_kwargs, api_name='submit_verifier' if allow_api else False) def load_model(model_name, lora_weights, server_name, model_state_old, prompt_type_old, chat_template_in, load_8bit, load_4bit, low_bit_mode, load_gptq, load_awq, load_exllama, use_safetensors, revision, use_cpu, use_gpu_id, gpu_id, max_seq_len1, rope_scaling1, model_path_llama1, model_name_gptj1, model_name_gpt4all_llama1, n_gpu_layers1, n_batch1, n_gqa1, llamacpp_dict_more1, system_prompt1, exllama_dict, gptq_dict, attention_sinks, sink_dict, truncation_generation, hf_model_dict, force_seq2seq_type, force_t5_type, model_options_state1, lora_options_state1, server_options_state1, unload=False): if unload: model_name = no_model_str lora_weights = no_lora_str server_name = no_server_str exllama_dict = str_to_dict(exllama_dict) gptq_dict = str_to_dict(gptq_dict) sink_dict = str_to_dict(sink_dict) hf_model_dict = str_to_dict(hf_model_dict) # switch-a-roo on base_model so can pass GGUF/GGML as base model model_name0 = model_name model_name, model_path_llama1, load_gptq, load_awq, n_gqa1 = \ switch_a_roo_llama(model_name, model_path_llama1, load_gptq, load_awq, n_gqa1, kwargs['llamacpp_path']) # after getting results, we always keep all items related to llama.cpp, gptj, gpt4all inside llamacpp_dict llamacpp_dict = str_to_dict(llamacpp_dict_more1) llamacpp_dict.update(dict(model_path_llama=model_path_llama1, model_name_gptj=model_name_gptj1, model_name_gpt4all_llama=model_name_gpt4all_llama1, n_gpu_layers=n_gpu_layers1, n_batch=n_batch1, n_gqa=n_gqa1, )) if model_name == 'llama' and not model_path_llama1: raise ValueError("Must set model_path_llama if model_name==llama") if model_name == 'gptj' and not model_name_gptj: raise ValueError("Must set model_name_gptj if model_name==llama") if model_name == 'gpt4all_llama' and not model_name_gpt4all_llama: raise ValueError("Must set model_name_gpt4all_llama if model_name==llama") # ensure no API calls reach here if is_public: raise RuntimeError("Illegal access for %s" % model_name) # ensure old model removed from GPU memory if kwargs['debug']: print("Pre-switch pre-del GPU memory: %s" % get_torch_allocated(), flush=True) model0 = model_state0['model'] if isinstance(model_state_old['model'], str) and \ model0 is not None and \ hasattr(model0, 'cpu'): # best can do, move model loaded at first to CPU model0.cpu() if model_state_old['model'] is not None and \ not isinstance(model_state_old['model'], str): if hasattr(model_state_old['model'], 'cpu'): try: model_state_old['model'].cpu() except Exception as e: # sometimes hit NotImplementedError: Cannot copy out of meta tensor; no data! print("Unable to put model on CPU: %s" % str(e), flush=True) del model_state_old['model'] model_state_old['model'] = None if model_state_old['tokenizer'] is not None and not isinstance(model_state_old['tokenizer'], str): del model_state_old['tokenizer'] model_state_old['tokenizer'] = None clear_torch_cache(allow_skip=False) if kwargs['debug']: print("Pre-switch post-del GPU memory: %s" % get_torch_allocated(), flush=True) if not model_name: model_name = no_model_str if model_name == no_model_str: # no-op if no model, just free memory # no detranscribe needed for model, never go into evaluate lora_weights = no_lora_str server_name = no_server_str prompt_type_old = '' chat_template_out = '' model_path_llama1 = '' model_name_gptj1 = '' model_name_gpt4all_llama1 = '' load_gptq = '' load_awq = '' return kwargs['model_state_none'].copy(), \ model_name, lora_weights, server_name, \ prompt_type_old, chat_template_out, max_seq_len1, \ gr.Slider(maximum=256), \ gr.Slider(maximum=256), \ model_path_llama1, model_name_gptj1, model_name_gpt4all_llama1, \ load_gptq, load_awq, n_gqa1, \ n_batch1, n_gpu_layers1, llamacpp_dict_more1, \ model_options_state1, lora_options_state1, server_options_state1 # don't deepcopy, can contain model itself all_kwargs1 = all_kwargs.copy() all_kwargs1['base_model'] = model_name.strip() all_kwargs1['load_8bit'] = load_8bit all_kwargs1['load_4bit'] = load_4bit all_kwargs1['low_bit_mode'] = low_bit_mode all_kwargs1['load_gptq'] = load_gptq all_kwargs1['load_awq'] = load_awq all_kwargs1['load_exllama'] = load_exllama all_kwargs1['use_safetensors'] = use_safetensors all_kwargs1['revision'] = None if not revision else revision # transcribe, don't pass '' all_kwargs1['use_gpu_id'] = use_gpu_id all_kwargs1['gpu_id'] = int(gpu_id) if gpu_id not in [None, 'None'] else None # detranscribe all_kwargs1['llamacpp_dict'] = llamacpp_dict all_kwargs1['exllama_dict'] = exllama_dict all_kwargs1['gptq_dict'] = gptq_dict all_kwargs1['attention_sinks'] = attention_sinks all_kwargs1['sink_dict'] = sink_dict all_kwargs1['truncation_generation'] = truncation_generation all_kwargs1['hf_model_dict'] = hf_model_dict all_kwargs1['force_seq2seq_type'] = force_seq2seq_type all_kwargs1['force_t5_type'] = force_t5_type # reasonable default for easy UI/UX even if not optimal if 'llama2' in model_name and max_seq_len1 in [-1, None]: max_seq_len1 = 4096 elif 'llama3' in model_name and max_seq_len1 in [-1, None]: max_seq_len1 = 8192 elif 'mistral' in model_name and max_seq_len1 in [-1, None]: max_seq_len1 = 4096 else: max_seq_len1 = 4096 all_kwargs1['max_seq_len'] = int(max_seq_len1) if max_seq_len1 is not None and max_seq_len1 > 0 else None try: all_kwargs1['rope_scaling'] = str_to_dict(rope_scaling1) # transcribe except: print("Failed to use user input for rope_scaling dict", flush=True) all_kwargs1['rope_scaling'] = {} if use_cpu: all_kwargs1['n_gpus'] = 0 elif use_gpu_id and all_kwargs1['gpu_id']: all_kwargs1['n_gpus'] = 1 else: all_kwargs1['n_gpus'] = n_gpus_global prompt_type1 = model_name_to_prompt_type(model_name, server_name, model_name0=model_name0, llamacpp_dict=llamacpp_dict, prompt_type_old=prompt_type_old) # detranscribe if lora_weights == no_lora_str: lora_weights = '' all_kwargs1['lora_weights'] = lora_weights.strip() if server_name == no_server_str: server_name = '' all_kwargs1['inference_server'] = server_name.strip() gradio_model_kwargs = dict(reward_type=False, **get_kwargs(get_model, exclude_names=['reward_type'], **all_kwargs1)) model1, tokenizer1, device1 = get_model_retry(**gradio_model_kwargs) clear_torch_cache() if chat_template_in and hasattr(tokenizer1, 'apply_chat_template'): try: tokenizer1.chat_template = base64_decode_jinja_template(chat_template_in) messages_test = [dict(role='user', content='Hi'), dict(role='assistant', content='Hello! How can I help you today?')] prompt = tokenizer1.apply_chat_template(messages_test, tokenize=False, add_generation_prompt=True) assert isinstance(prompt, str) except Exception as e: print("Could not overwrite %s template: %s" % (model_name, str(e))) raise # use if didn't fail chat_template_out = chat_template_in else: chat_template_out = get_chat_template(tokenizer1) tokenizer_base_model = model_name prompt_dict1, error0 = get_prompt(prompt_type1, '', context='', reduced=False, making_context=False, return_dict=True, system_prompt=system_prompt1) model_state_new = dict(model=model1, tokenizer=tokenizer1, device=device1, base_model=model_name, display_name=model_name, tokenizer_base_model=tokenizer_base_model, lora_weights=lora_weights, inference_server=server_name, prompt_type=prompt_type1, prompt_dict=prompt_dict1, # FIXME: not typically required, unless want to expose adding h2ogpt endpoint in UI visible_models=None, h2ogpt_key=None, ) [model_state_new.update({k: v}) for k, v in kwargs['model_state_none'].items() if k not in model_state_new] max_seq_len1new = get_model_max_length_from_tokenizer(tokenizer1) max_max_new_tokens1 = get_max_max_new_tokens(model_state_new, **kwargs) # FIXME: Ensure stored in login state if model_options_state1 and model_name0 not in model_options_state1[0]: model_options_state1[0].extend([model_name0]) if lora_options_state1 and lora_weights not in lora_options_state1[0]: lora_options_state1[0].extend([lora_weights]) if server_options_state1 and server_name not in server_options_state1[0]: server_options_state1[0].extend([server_name]) if kwargs['debug']: print("Post-switch GPU memory: %s" % get_torch_allocated(), flush=True) return model_state_new, model_name, lora_weights, server_name, \ prompt_type1, chat_template_out, max_seq_len1new, \ gr.Slider(maximum=max_max_new_tokens1), \ gr.Slider(maximum=max_max_new_tokens1), \ model_path_llama1, model_name_gptj1, model_name_gpt4all_llama1, \ load_gptq, load_awq, n_gqa1, \ n_batch1, n_gpu_layers1, llamacpp_dict_more1, \ model_options_state1, lora_options_state1, server_options_state1 def get_prompt_str(prompt_type1, prompt_dict1, system_prompt1, which=0): if prompt_type1 in ['', None]: print("Got prompt_type %s: %s" % (which, prompt_type1), flush=True) return str({}) prompt_dict1, prompt_dict_error = get_prompt(prompt_type1, prompt_dict1, context='', reduced=False, making_context=False, return_dict=True, system_prompt=system_prompt1) if prompt_dict_error: return str(prompt_dict_error) else: # return so user can manipulate if want and use as custom return str(prompt_dict1) get_prompt_str_func1 = functools.partial(get_prompt_str, which=1) get_prompt_str_func2 = functools.partial(get_prompt_str, which=2) prompt_type.change(fn=get_prompt_str_func1, inputs=[prompt_type, prompt_dict, system_prompt], outputs=prompt_dict, **noqueue_kwargs) prompt_type2.change(fn=get_prompt_str_func2, inputs=[prompt_type2, prompt_dict2, system_prompt], outputs=prompt_dict2, **noqueue_kwargs) def dropdown_prompt_type_list(x): return gr.Dropdown(value=x) def chatbot_list(x, model_used_in, model_path_llama_in, inference_server_in, prompt_type_in, model_label_prefix_in=''): chat_name = get_chatbot_name(model_used_in, model_used_in, model_path_llama_in, inference_server_in, prompt_type_in, model_label_prefix=model_label_prefix_in) return gr.Textbox(label=chat_name) load_model_inputs = [model_choice, lora_choice, server_choice, model_state, prompt_type, chat_template, model_load8bit_checkbox, model_load4bit_checkbox, model_low_bit_mode, model_load_gptq, model_load_awq, model_load_exllama_checkbox, model_safetensors_checkbox, model_revision, model_use_cpu_checkbox, model_use_gpu_id_checkbox, model_gpu, max_seq_len, rope_scaling, model_path_llama, model_name_gptj, model_name_gpt4all_llama, n_gpu_layers, n_batch, n_gqa, llamacpp_dict_more, system_prompt, model_exllama_dict, model_gptq_dict, model_attention_sinks, model_sink_dict, model_truncation_generation, model_hf_model_dict, model_force_seq2seq_type, model_force_force_t5_type, model_options_state, lora_options_state, server_options_state, ] load_model_outputs = [model_state, model_used, lora_used, server_used, # if prompt_type changes, prompt_dict will change via change rule prompt_type, chat_template, max_seq_len_used, max_new_tokens, min_new_tokens, model_path_llama, model_name_gptj, model_name_gpt4all_llama, model_load_gptq, model_load_awq, n_gqa, n_batch, n_gpu_layers, llamacpp_dict_more, model_options_state, lora_options_state, server_options_state, ] load_model_args = dict(fn=load_model, inputs=load_model_inputs, outputs=load_model_outputs) unload_model_args = dict(fn=functools.partial(load_model, unload=True), inputs=load_model_inputs, outputs=load_model_outputs) prompt_update_args = dict(fn=dropdown_prompt_type_list, inputs=prompt_type, outputs=prompt_type) chatbot_update_args = dict( fn=functools.partial(chatbot_list, model_label_prefix_in=kwargs['model_label_prefix']), inputs=[text_output, model_used, model_path_llama, server_used, prompt_type], outputs=text_output) nochat_update_args = dict( fn=functools.partial(chatbot_list, model_label_prefix_in=kwargs['model_label_prefix']), inputs=[text_output_nochat, model_used, model_path_llama, server_used, prompt_type], outputs=text_output_nochat) load_model_event = load_model_button.click(**load_model_args, api_name='load_model' if allow_api and not is_public else False) \ .then(**prompt_update_args) \ .then(**chatbot_update_args) \ .then(**nochat_update_args) \ .then(clear_torch_cache) \ .then(**save_auth_kwargs) unload_model_event = unload_model_button.click(**unload_model_args, api_name='unload_model' if allow_api and not is_public else False) \ .then(**prompt_update_args) \ .then(**chatbot_update_args) \ .then(**nochat_update_args) \ .then(clear_torch_cache) load_model_inputs2 = [model_choice2, lora_choice2, server_choice2, model_state2, prompt_type2, chat_template2, model_load8bit_checkbox2, model_load4bit_checkbox2, model_low_bit_mode2, model_load_gptq2, model_load_awq2, model_load_exllama_checkbox2, model_safetensors_checkbox2, model_revision2, model_use_cpu_checkbox2, model_use_gpu_id_checkbox2, model_gpu2, max_seq_len2, rope_scaling2, model_path_llama2, model_name_gptj2, model_name_gpt4all_llama2, n_gpu_layers2, n_batch2, n_gqa2, llamacpp_dict_more2, system_prompt, model_exllama_dict2, model_gptq_dict2, model_attention_sinks2, model_sink_dict2, model_truncation_generation2, model_hf_model_dict2, model_force_seq2seq_type2, model_force_force_t5_type2, model_options_state, lora_options_state, server_options_state, ] load_model_outputs2 = [model_state2, model_used2, lora_used2, server_used2, # if prompt_type2 changes, prompt_dict2 will change via change rule prompt_type2, chat_template2, max_seq_len_used2, max_new_tokens2, min_new_tokens2, model_path_llama2, model_name_gptj2, model_name_gpt4all_llama2, model_load_gptq2, model_load_awq2, n_gqa2, n_batch2, n_gpu_layers2, llamacpp_dict_more2, model_options_state, lora_options_state, server_options_state, ] load_model_args2 = dict(fn=load_model, inputs=load_model_inputs2, outputs=load_model_outputs2) unload_model_args2 = dict(fn=functools.partial(load_model, unload=True), inputs=load_model_inputs2, outputs=load_model_outputs2) prompt_update_args2 = dict(fn=dropdown_prompt_type_list, inputs=prompt_type2, outputs=prompt_type2) chatbot_update_args2 = dict( fn=functools.partial(chatbot_list, model_label_prefix_in=kwargs['model_label_prefix']), inputs=[text_output2, model_used2, model_path_llama2, server_used2, prompt_type2], outputs=text_output2) load_model_event2 = load_model_button2.click(**load_model_args2, api_name='load_model2' if allow_api and not is_public else False) \ .then(**prompt_update_args2) \ .then(**chatbot_update_args2) \ .then(clear_torch_cache) \ .then(**save_auth_kwargs) unload_model_event2 = unload_model_button2.click(**unload_model_args2, api_name='unload_model2' if allow_api and not is_public else False) \ .then(**prompt_update_args) \ .then(**chatbot_update_args) \ .then(**nochat_update_args) \ .then(clear_torch_cache) def dropdown_model_lora_server_list(model_list0, model_x, lora_list0, lora_x, server_list0, server_x, model_used1, lora_used1, server_used1, model_used2, lora_used2, server_used2, ): model_new_state = [model_list0[0] + [model_x]] model_new_options = [*model_new_state[0]] if no_model_str in model_new_options: model_new_options.remove(no_model_str) model_new_options = [no_model_str] + sorted(model_new_options) x1 = model_x if model_used1 == no_model_str else model_used1 x2 = model_x if model_used2 == no_model_str else model_used2 ret1 = [gr.Dropdown(value=x1, choices=model_new_options), gr.Dropdown(value=x2, choices=model_new_options), '', model_new_state] lora_new_state = [lora_list0[0] + [lora_x]] lora_new_options = [*lora_new_state[0]] if no_lora_str in lora_new_options: lora_new_options.remove(no_lora_str) lora_new_options = [no_lora_str] + sorted(lora_new_options) # don't switch drop-down to added lora if already have model loaded x1 = lora_x if model_used1 == no_model_str else lora_used1 x2 = lora_x if model_used2 == no_model_str else lora_used2 ret2 = [gr.Dropdown(value=x1, choices=lora_new_options), gr.Dropdown(value=x2, choices=lora_new_options), '', lora_new_state] server_new_state = [server_list0[0] + [server_x]] server_new_options = [*server_new_state[0]] if no_server_str in server_new_options: server_new_options.remove(no_server_str) server_new_options = [no_server_str] + sorted(server_new_options) # don't switch drop-down to added server if already have model loaded x1 = server_x if model_used1 == no_model_str else server_used1 x2 = server_x if model_used2 == no_model_str else server_used2 ret3 = [gr.Dropdown(value=x1, choices=server_new_options), gr.Dropdown(value=x2, choices=server_new_options), '', server_new_state] return tuple(ret1 + ret2 + ret3) add_model_lora_server_event = \ add_model_lora_server_button.click(fn=dropdown_model_lora_server_list, inputs=[model_options_state, new_model] + [lora_options_state, new_lora] + [server_options_state, new_server] + [model_used, lora_used, server_used] + [model_used2, lora_used2, server_used2], outputs=[model_choice, model_choice2, new_model, model_options_state] + [lora_choice, lora_choice2, new_lora, lora_options_state] + [server_choice, server_choice2, new_server, server_options_state], **noqueue_kwargs) def get_inf_models_gr(model_options_state1, model_choice1, server1): models_new = get_inf_models(server1, verbose=verbose) model_options_state1[0].extend(models_new) if no_model_str in model_options_state1[0]: model_options_state1[0].remove(no_model_str) model_options_state1[0] = [no_model_str] + sorted(set(model_options_state1[0])) if models_new: model_choice1 = models_new[0] # pick new one return model_options_state1, gr.Dropdown(choices=model_options_state1[0], value=model_choice1) load_models_button.click(get_inf_models_gr, inputs=[model_options_state, model_choice, server_choice], outputs=[model_options_state, model_choice]) load_models_button2.click(get_inf_models_gr, inputs=[model_options_state, model_choice2, server_choice2], outputs=[model_options_state, model_choice2]) go_event = go_btn.click(lambda: gr.update(visible=False), None, go_btn, api_name="go" if allow_api else False, **noqueue_kwargs) \ .then(lambda: gr.update(visible=True), None, normal_block, **noqueue_kwargs) \ .then(**load_model_args, **noqueue_kwargs).then(**prompt_update_args, **noqueue_kwargs) def compare_textbox_fun(x): return gr.Textbox(visible=x) def compare_column_fun(x): return gr.Column(visible=x) def compare_prompt_fun(x): return gr.Dropdown(visible=x) def slider_fun(x): return gr.Slider(visible=x) compare_checkbox.select(compare_textbox_fun, compare_checkbox, text_output2, api_name="compare_checkbox" if allow_api else False) \ .then(compare_column_fun, compare_checkbox, col_model2) \ .then(compare_prompt_fun, compare_checkbox, prompt_type2) \ .then(compare_textbox_fun, compare_checkbox, score_text2) \ .then(slider_fun, compare_checkbox, max_new_tokens2) \ .then(slider_fun, compare_checkbox, min_new_tokens2) # FIXME: add score_res2 in condition, but do better # callback for logging flagged input/output callback.setup(inputs_list + [text_output, text_output2] + text_outputs, "flagged_data_points") flag_btn.click(lambda *args: callback.flag(args), inputs_list + [text_output, text_output2] + text_outputs, None, preprocess=False, api_name='flag' if allow_api else False, **noqueue_kwargs) flag_btn_nochat.click(lambda *args: callback.flag(args), inputs_list + [text_output_nochat], None, preprocess=False, api_name='flag_nochat' if allow_api else False, **noqueue_kwargs) def get_system_info(): if is_public: time.sleep(10) # delay to avoid spam since **noqueue_kwargs return gr.Textbox(value=system_info_print()) system_event = system_btn.click(get_system_info, outputs=system_text, api_name='system_info' if kwargs['system_api_open'] else False, **noqueue_kwargs) def shutdown_func(admin_pass_textbox1, h2ogpt_pid): assert admin_pass_textbox1 == admin_pass or not admin_pass if kwargs['close_button']: import psutil parent = psutil.Process(h2ogpt_pid) for child in parent.children(recursive=True): child.kill() parent.kill() api_name_shutdown = 'shutdown' if kwargs['shutdown_via_api'] and \ allow_api and \ not is_public and \ kwargs['h2ogpt_pid'] is not None else False shutdown_event = close_btn.click(functools.partial(shutdown_func, h2ogpt_pid=kwargs['h2ogpt_pid']), inputs=[admin_pass_textbox], outputs=None, api_name=api_name_shutdown, **noqueue_kwargs) def get_system_info_dict(system_input1, **kwargs1): if system_input1 != os.getenv("ADMIN_PASS", ""): return json.dumps({}) exclude_list = ['admin_pass', 'examples'] sys_dict = {k: v for k, v in kwargs1.items() if isinstance(v, (str, int, bool, float)) and k not in exclude_list} try: sys_dict.update(system_info()) except Exception as e: # protection print("Exception: %s" % str(e), flush=True) return json.dumps(sys_dict) system_kwargs = all_kwargs.copy() system_kwargs.update(dict(command=str(' '.join(sys.argv)))) get_system_info_dict_func = functools.partial(get_system_info_dict, **all_kwargs) system_dict_event = system_btn2.click(get_system_info_dict_func, inputs=system_input, outputs=system_text2, api_name='system_info_dict' if kwargs['system_api_open'] else False, **noqueue_kwargs, # queue to avoid spam ) def get_model_states(): if len(model_states) >= 1: local_model_states = model_states elif model_state0 is not None: local_model_states = [model_state0] else: local_model_states = [] return local_model_states def get_model_names(): local_model_states = get_model_states() return _get_model_names(local_model_states) def get_model_names_from_lock(admin_pass_textbox1, model_lock_client): assert admin_pass_textbox1 == admin_pass or not admin_pass local_model_states = [ model_lock_to_state(model_lock_client, cache_model_state=True, **kwargs)] return _get_model_names(local_model_states) def _get_model_names(local_model_states): for model_state3 in local_model_states: base_model = model_state3.get('base_model', '') inference_server = model_state3.get('inference_server', '') inference_server_split = inference_server.split(':') inference_server_type = inference_server_split[0].strip() if len( inference_server_split) > 0 else inference_server if 'api.together.xyz' in inference_server: inference_server_type = 'together.ai' from gradio_utils.grclient import GradioClient if isinstance(model_state3.get('model', ''), GradioClient): inference_server_type = 'gradio' if model_state3.get('prompt_type', '') or '' == 'openai_chat': inference_server_type = 'gradio_to_openai_chat' elif model_state3.get('prompt_type', '') or '' == 'openai': inference_server_type = 'gradio_to_openai' # could be TGI, but then will show up as http(s). model_state3['llm'] = True model_state3['rag'] = True model_state3['image'] = model_state3.get('is_vision_model', False) model_state3['actually_image'] = model_state3.get('is_actually_vision_model', False) model_state3['video'] = is_video_model(base_model) or model_state3['image'] model_state3['actually_video'] = is_video_model(base_model) model_state3['json'] = model_state3.get('json', False) model_state3['guided_vllm'] = model_state3.get('guided_vllm', False) model_state3['auto_visible_vision_models'] = model_state3.get('auto_visible_vision_models', False) model_state3['inference_server_type'] = inference_server_type json_vllm = model_state3.get('json_vllm', False) model_state3['strict_json_schema'] = get_supports_schema(inference_server, base_model, json_vllm=json_vllm, just_test=True) key_list = ['display_name', 'base_model', 'inference_server_type', 'strict_json_schema', 'prompt_type', 'prompt_dict', 'chat_template'] + list( kwargs['other_model_state_defaults'].keys()) # don't want to expose backend inference server IP etc. # key_list += ['inference_server'] key_list.extend(['llm', 'rag', 'image', 'actually_image', 'video', 'actually_video', 'json', 'guided_vllm', 'auto_visible_vision_models']) return [{k: x[k] for k in key_list if k in x} for x in local_model_states] models_list_event = system_btn4.click(get_model_names, outputs=system_text4, api_name='model_names' if allow_api else False, **noqueue_kwargs, ) # loads model, so admin password protected models_list_event2 = system_btn5.click(get_model_names_from_lock, inputs=[admin_pass_textbox, model_lock], outputs=system_text5, api_name='model_names_from_lock' if allow_api else False, **noqueue_kwargs, ) def text_to_dict(x): e1 = e2 = None try: # see if json x = json.loads(x) except Exception as e: e1 = e try: # see if literal python dict x = ast.literal_eval(x) except Exception as e0: e2 = e0 x = {} pass if e1 and e2: raise ValueError("Input not valid JSON or literal python dict: %s %s" % (e1, e2)) return x def update_all_users(admin_pass_textbox1, admin_user_update_text1): assert admin_pass_textbox1 == admin_pass or not admin_pass auth_filename1 = kwargs['auth_filename'] if not auth_filename1.endswith('.db'): return admin_user_update_text1 = text_to_dict(admin_user_update_text1) with filelock.FileLock(auth_filename1 + '.lock'): append_to_users_data(auth_filename1, admin_user_update_text1, verbose=verbose) admin_user_update_btn.click(functools.partial(update_all_users), inputs=[admin_pass_textbox, admin_user_update_text], outputs=None, api_name='update_all_users' if allow_api else False, **noqueue_kwargs) def get_users(admin_pass_textbox1): assert admin_pass_textbox1 == admin_pass or not admin_pass auth_filename1 = kwargs['auth_filename'] if not auth_filename1.endswith('.db'): return 'auth_filename not .db but %s' % kwargs['auth_filename'] with filelock.FileLock(auth_filename1 + '.lock'): users_json = json.dumps(get_all_usernames(auth_filename1)) return users_json admin_users_list_btn.click(functools.partial(get_users), inputs=[admin_pass_textbox], outputs=[admin_user_list_text], api_name='list_users' if allow_api else False, **noqueue_kwargs) def get_user(admin_pass_textbox1, username1): assert admin_pass_textbox1 == admin_pass or not admin_pass auth_filename1 = kwargs['auth_filename'] if not auth_filename1.endswith('.db'): return 'auth_filename not .db but %s' % kwargs['auth_filename'] with filelock.FileLock(auth_filename1 + '.lock'): # will create .db if doing migration auth_dict = fetch_user(kwargs['auth_filename'], username1, verbose=verbose) return json.dumps(auth_dict) admin_user_get_btn.click(functools.partial(get_user), inputs=[admin_pass_textbox, admin_user_txt], outputs=[admin_user_get_info], api_name='get_user' if allow_api else False, **noqueue_kwargs) def put_user(admin_pass_textbox1, username1, admin_user_info1, full1=False): assert admin_pass_textbox1 == admin_pass or not admin_pass auth_filename1 = kwargs['auth_filename'] if not auth_filename1.endswith('.db'): return 'auth_filename not .db but %s' % auth_filename1 admin_user_info1 = text_to_dict(admin_user_info1) with filelock.FileLock(auth_filename1 + '.lock'): # first fetch, e.g. in case migration auth_dict0 = fetch_user(auth_filename1, username1, verbose=verbose) if full1: upsert_user(auth_filename1, username1, admin_user_info1, verbose=verbose) else: append_to_user_data(auth_filename1, username1, admin_user_info1, verbose=verbose) auth_dict1 = fetch_user(auth_filename1, username1, verbose=verbose) return auth_dict1 admin_user_put_btn.click(functools.partial(put_user), inputs=[admin_pass_textbox, admin_user_txt, admin_user_put_info], outputs=[admin_user_get_info], api_name='update_user' if allow_api else False, **noqueue_kwargs) admin_user_put_full_btn.click(functools.partial(functools.partial(put_user, full1=True)), inputs=[admin_pass_textbox, admin_user_txt, admin_user_put_info], outputs=[admin_user_get_info], api_name='update_full_user' if allow_api else False, **noqueue_kwargs) def count_chat_tokens(model_state1, chat1, prompt_type1, prompt_dict1, system_prompt1, chat_conversation1, memory_restriction_level1=0, keep_sources_in_context1=False, ): if model_state1 and not isinstance(model_state1['tokenizer'], str): tokenizer = model_state1['tokenizer'] elif model_state0 and not isinstance(model_state0['tokenizer'], str): tokenizer = model_state0['tokenizer'] else: tokenizer = None if tokenizer is not None: langchain_mode1 = 'LLM' add_chat_history_to_context1 = True # fake user message to mimic bot() chat1 = copy.deepcopy(chat1) chat1 = chat1 + [['user_message1', None]] model_max_length1 = tokenizer.model_max_length context1, chat1 = history_to_context(chat1, langchain_mode=langchain_mode1, add_chat_history_to_context=add_chat_history_to_context1, prompt_type=prompt_type1, prompt_dict=prompt_dict1, model_max_length=model_max_length1, memory_restriction_level=memory_restriction_level1, keep_sources_in_context=keep_sources_in_context1, system_prompt=system_prompt1, chat_conversation=chat_conversation1, hyde_level=None, gradio_errors_to_chatbot=kwargs['gradio_errors_to_chatbot']) tokens = tokenizer(context1, return_tensors="pt")['input_ids'] if len(tokens.shape) == 1: return str(tokens.shape[0]) elif len(tokens.shape) == 2: return str(tokens.shape[1]) else: return "N/A" else: return "N/A" count_chat_tokens_func = functools.partial(count_chat_tokens, memory_restriction_level1=memory_restriction_level, keep_sources_in_context1=kwargs['keep_sources_in_context']) count_tokens_event = count_chat_tokens_btn.click(fn=count_chat_tokens_func, inputs=[model_state, text_output, prompt_type, prompt_dict, system_prompt, chat_conversation], outputs=chat_token_count, api_name='count_tokens' if allow_api else False) speak_events = [] if kwargs['enable_tts'] and kwargs['predict_from_text_func'] is not None: if kwargs['tts_model'].startswith('tts_models/'): speak_human_event = speak_human_button.click(kwargs['predict_from_text_func'], inputs=[instruction, chatbot_role, tts_language, roles_state, tts_speed], outputs=speech_human, api_name=False, # not for API ) speak_events.extend([speak_human_event]) elif kwargs['tts_model'].startswith('microsoft'): speak_human_event = speak_human_button.click(kwargs['predict_from_text_func'], inputs=[instruction, speaker, tts_speed], outputs=speech_human, api_name=False, # not for API ) speak_events.extend([speak_human_event]) def wrap_pred_func(chatbot_role1, speaker1, tts_language1, roles_state1, tts_speed1, visible_models1, text_output1, text_output21, *args, all_models=[]): # FIXME: Choose first visible text_outputs1 = list(args) text_outputss = [text_output1, text_output21] + text_outputs1 text_outputss = [x[-1][1] for x in text_outputss if len(x) >= 1 and len(x[-1]) == 2 and x[-1][1]] response = text_outputss[0] if text_outputss else '' keep_sources_in_context1 = False langchain_mode1 = None # so always tries hyde_level1 = None # so always tries response = remove_refs(response, keep_sources_in_context1, langchain_mode1, hyde_level1, kwargs['gradio_errors_to_chatbot']) if kwargs['enable_tts'] and kwargs['predict_from_text_func'] is not None and response: if kwargs['tts_model'].startswith('tts_models/') and chatbot_role1 not in [None, 'None']: yield from kwargs['predict_from_text_func'](response, chatbot_role1, tts_language1, roles_state1, tts_speed1) elif kwargs['tts_model'].startswith('microsoft') and speaker1 not in [None, 'None']: yield from kwargs['predict_from_text_func'](response, speaker1, tts_speed1) def _wrap_pred_func_api(chatbot_role1, speaker1, tts_language1, tts_speed1, response, roles_state1): if kwargs['tts_model'].startswith('microsoft') and speaker1 not in [None, "None"]: sr1 = 16000 elif kwargs['tts_model'].startswith('tts_models/') and chatbot_role1 not in [None, "None"]: sr1 = 24000 else: return if kwargs['enable_tts'] and kwargs['predict_from_text_func'] is not None and response: if kwargs['tts_model'].startswith('tts_models/') and chatbot_role1 not in [None, 'None']: yield from kwargs['predict_from_text_func'](response, chatbot_role1, tts_language1, roles_state1, tts_speed1, return_prefix_every_yield=False, include_audio0=False, return_dict=True, sr=sr1) elif kwargs['tts_model'].startswith('microsoft') and speaker1 not in [None, 'None']: yield from kwargs['predict_from_text_func'](response, speaker1, tts_speed1, return_prefix_every_yield=False, include_audio0=False, return_dict=True, sr=sr1) def wrap_pred_func_api(chatbot_role1, speaker1, tts_language1, tts_speed1, response, stream_output1, h2ogpt_key1, roles_state1, requests_state1): # check key valid_key = is_valid_key(kwargs['enforce_h2ogpt_api_key'], kwargs['enforce_h2ogpt_ui_key'], kwargs['h2ogpt_api_keys'], h2ogpt_key1, requests_state1=requests_state1) kwargs['from_ui'] = is_from_ui(requests_state1) if not valid_key: raise ValueError(invalid_key_msg) if stream_output1: yield from _wrap_pred_func_api(chatbot_role1, speaker1, tts_language1, tts_speed1, response, roles_state1) else: audios = [] for audio1 in _wrap_pred_func_api(chatbot_role1, speaker1, tts_language1, tts_speed1, response, roles_state1): audios.append(audio1) srs = [x['sr'] for x in audios] if len(srs) > 0: sr = srs[0] audios = [x['audio'] for x in audios] audios = combine_audios(audios, audio=None, sr=sr, expect_bytes=kwargs['return_as_byte'], verbose=verbose) yield dict(audio=audios, sr=sr) def wrap_pred_func_plain_api(*args1): args_dict = ast.literal_eval(args1[0]) args_dict['requests_state'] = requests_state0.copy() args_dict['roles_state'] = roles_state.value.copy() input_args_list_speak = ['chatbot_role', 'speaker', 'tts_language', 'tts_speed', 'prompt', 'stream_output', 'h2ogpt_key', 'roles_state', 'requests_state'] assert len(args_dict) == len(input_args_list_speak) # fix order and make into list args_dict = {k: args_dict[k] for k in input_args_list_speak} args_list = list(args_dict.values()) ret = yield from wrap_pred_func_api(*tuple(args_list)) return ret speak_bot_event = speak_bot_button.click(wrap_pred_func, inputs=[chatbot_role, speaker, tts_language, roles_state, tts_speed, visible_models, text_output, text_output2] + text_outputs, outputs=speech_bot, api_name=False, # not for API ) speak_events.extend([speak_bot_event]) speak_text_api_event1 = speak_text_api_button.click(**user_state_kwargs) speak_text_api_event = speak_text_api_event1.then(wrap_pred_func_api, inputs=[chatbot_role, speaker, tts_language, tts_speed, text_speech, stream_output, h2ogpt_key, roles_state, requests_state], outputs=text_speech_out, api_name='speak_text_api' if allow_api else False, ) speak_text_plain_api_event = speak_text_plain_api_button.click(wrap_pred_func_plain_api, inputs=speak_inputs_dict_str, outputs=text_speech_out, api_name='speak_text_plain_api' if allow_api else False, **noqueue_kwargs_curl, ) def stop_audio_func(): return None, None if kwargs['enable_tts']: stop_speak_button.click(stop_audio_func, outputs=[speech_human, speech_bot], cancels=speak_events, **noqueue_kwargs2) # don't pass text_output, don't want to clear output, just stop it # cancel only stops outer generation, not inner generation or non-generation clear_torch_cache_func_soft = functools.partial(clear_torch_cache, allow_skip=True) stop_event = stop_btn.click(lambda: None, None, None, cancels=submits1 + submits2 + submits3 + submits4 + [submit_event_nochat, submit_event_nochat2] + # [eventdb1, eventdb2, eventdb3] + # [eventdb7a, eventdb7, eventdb8a, eventdb8, eventdb9a, eventdb9, eventdb12a, # eventdb12] + # db_events + # [eventdbloadla, eventdbloadlb] + [clear_event] + [submit_event_nochat_api, submit_event_nochat] + [load_model_event, load_model_event2] + [count_tokens_event] + speak_events , **noqueue_kwargs, api_name='stop' if allow_api else False) \ .then(clear_torch_cache_func_soft, **noqueue_kwargs) \ .then(stop_audio_func, outputs=[speech_human, speech_bot]) if kwargs['auth'] is not None: auth = authf load_func = user_state_setup load_inputs = [my_db_state, requests_state, guest_name, login_btn, login_btn] load_outputs = [my_db_state, requests_state, login_btn] else: auth = None load_func = user_state_setup load_inputs = [my_db_state, requests_state, guest_name, login_btn, login_btn] load_outputs = [my_db_state, requests_state, login_btn] # auth = None # load_func, load_inputs, load_outputs = None, None, None app_js = wrap_js_to_lambda( len(load_inputs) if load_inputs else 0, get_dark_js() if kwargs['dark'] else None, get_heap_js(heap_app_id) if is_heap_analytics_enabled else None) #get_console_log_function()) load_kwargs = dict(js=app_js) if is_gradio_version4 else dict(_js=app_js) load_event = demo.load(fn=load_func, inputs=load_inputs, outputs=load_outputs, **load_kwargs) if load_func: load_event2 = load_event.then(load_login_func, inputs=login_inputs, outputs=login_outputs) if load_func and auth: if not kwargs['large_file_count_mode']: get_sources_fun_kwargs_login = get_sources_fun_kwargs.copy() get_sources_fun_kwargs_login['for_login'] = True get_sources1_login = functools.partial(get_sources_gr, **get_sources_fun_kwargs_login) get_sources_kwargs_login = dict(fn=get_sources1_login, inputs=[my_db_state, selection_docs_state, requests_state, langchain_mode, h2ogpt_key], outputs=[file_source, docs_state, text_doc_count], queue=queue) load_event3 = load_event2.then(**get_sources_kwargs_login) load_event4 = load_event3.then(fn=update_dropdown, inputs=docs_state, outputs=document_choice) show_sources1_fun_kwargs_login = show_sources1_fun_kwargs.copy() show_sources1_fun_kwargs_login['for_login'] = True show_sources1_login = functools.partial(get_source_files_given_langchain_mode_gr, **show_sources1_fun_kwargs_login, ) show_sources_kwargs_login = dict(fn=show_sources1_login, inputs=[my_db_state, selection_docs_state, requests_state, langchain_mode, h2ogpt_key], outputs=sources_text) load_event5 = load_event4.then(**show_sources_kwargs_login) get_viewable_sources1_fun_kwargs_login = get_viewable_sources1_fun_kwargs.copy() get_viewable_sources1_fun_kwargs_login['for_login'] = True get_viewable_sources1_login = functools.partial(get_sources_gr, **get_viewable_sources1_fun_kwargs_login) get_viewable_sources_args_login = dict(fn=get_viewable_sources1_login, inputs=[my_db_state, selection_docs_state, requests_state, langchain_mode, h2ogpt_key], outputs=[file_source, viewable_docs_state, text_viewable_doc_count], queue=queue) load_event6 = load_event5.then(**get_viewable_sources_args_login) load_event7 = load_event6.then(**viewable_kwargs) def wrap_transcribe_func_api(audio_obj1, stream_output1, h2ogpt_key1, requests_state1): # check key valid_key = is_valid_key(kwargs['enforce_h2ogpt_api_key'], kwargs['enforce_h2ogpt_ui_key'], kwargs['h2ogpt_api_keys'], h2ogpt_key1, requests_state1=requests_state1) kwargs['from_ui'] = is_from_ui(requests_state1) if not valid_key: raise ValueError(invalid_key_msg) audio_api_state0 = ['', '', None, 'on'] state_text = kwargs['transcriber_func'](audio_api_state0, audio_obj1) text = state_text[1] yield text audio_api_output = gr.Textbox(value='', visible=False) audio_api_input = gr.Textbox(value='', visible=False) audio_api_btn = gr.Button(visible=False) audio_api_btn.click(fn=wrap_transcribe_func_api, inputs=[audio_api_input, stream_output, h2ogpt_key, requests_state], outputs=[audio_api_output], api_name='transcribe_audio_api', show_progress='hidden') def wrap_embedding_func_api(text, h2ogpt_key1, is_list1, requests_state1): # check key valid_key = is_valid_key(kwargs['enforce_h2ogpt_api_key'], kwargs['enforce_h2ogpt_ui_key'], kwargs['h2ogpt_api_keys'], h2ogpt_key1, requests_state1=requests_state1) kwargs['from_ui'] = is_from_ui(requests_state1) if not valid_key: raise ValueError(invalid_key_msg) assert not kwargs['use_openai_embedding'], "Should not be using OpenAI embeddings." is_list1 = ast.literal_eval(is_list1) if is_list1: text = ast.literal_eval(text) else: text = [text] embedding = kwargs['hf_embedding_model']['model'].embed_documents(text) return embedding embed_api_output = gr.Textbox(value='', visible=False) embed_api_input = gr.Textbox(value='', visible=False) embed_api_btn = gr.Button(visible=False) is_list = gr.Textbox(value='False', visible=False) embed_api_btn.click(fn=wrap_embedding_func_api, inputs=[embed_api_input, h2ogpt_key, is_list, requests_state], outputs=[embed_api_output], api_name='embed_api', show_progress='hidden') demo.queue(**queue_kwargs, api_open=kwargs['api_open']) favicon_file = "./aibenfavicon.png"#"h2o-logo.svg" favicon_path = kwargs['favicon_path'] or favicon_file if not os.path.isfile(favicon_file): print("favicon_path1=%s not found" % favicon_file, flush=True) alt_path = os.path.dirname(os.path.abspath(__file__)) favicon_path = os.path.join(alt_path, favicon_file) if not os.path.isfile(favicon_path): print("favicon_path2: %s not found in %s" % (favicon_file, alt_path), flush=True) alt_path = os.path.dirname(alt_path) favicon_path = os.path.join(alt_path, favicon_file) if not os.path.isfile(favicon_path): print("favicon_path3: %s not found in %s" % (favicon_file, alt_path), flush=True) favicon_path = None if kwargs['prepare_offline_level'] > 0: from prepare_offline import go_prepare_offline go_prepare_offline(**locals().copy()) return scheduler = BackgroundScheduler() if kwargs['clear_torch_cache_level'] in [0, 1]: interval_time = 120 clear_torch_cache_func_periodic = clear_torch_cache_func_soft else: interval_time = 20 clear_torch_cache_func_periodic = clear_torch_cache # don't require ever clear torch cache scheduler.add_job(func=clear_torch_cache_func_periodic, trigger="interval", seconds=interval_time) if is_public and \ kwargs['base_model'] not in non_hf_types: # FIXME: disable for gptj, langchain or gpt4all modify print itself # FIXME: and any multi-threaded/async print will enter model output! scheduler.add_job(func=ping, trigger="interval", seconds=60) if os.getenv('PING_GPU'): scheduler.add_job(func=ping_gpu, trigger="interval", seconds=60 * 10) scheduler.start() # import control if kwargs['langchain_mode'] == 'Disabled' and \ os.environ.get("TEST_LANGCHAIN_IMPORT") and \ kwargs['base_model'] not in non_hf_types: assert 'gpt_langchain' not in sys.modules, "Dev bug, import of langchain when should not have" assert 'langchain' not in sys.modules, "Dev bug, import of langchain when should not have" # set port in case GRADIO_SERVER_PORT was already set in prior main() call, # gradio does not listen if change after import # Keep None if not set so can find an open port above used ports server_port = os.getenv('GRADIO_SERVER_PORT') if server_port is not None: server_port = int(server_port) # NOTE: Dynamically added paths won't work unless relative to root and not public allowed_paths = [] allowed_paths += [os.path.abspath(v) for k, v in kwargs['langchain_mode_paths'].items() if v] allowed_paths += [os.path.abspath(x) for x in kwargs['extra_allowed_paths']] blocked_paths = [os.path.abspath(x) for x in kwargs['blocked_paths']] max_threads = max(128, 4 * kwargs['concurrency_count']) if isinstance(kwargs['concurrency_count'], int) else 128 if kwargs['google_auth']: import uvicorn from gradio_utils.google_auth import get_app app_kwargs = dict( favicon_path=favicon_path, # prevent_thread_lock=True, allowed_paths=allowed_paths if allowed_paths else None, blocked_paths=blocked_paths if blocked_paths else None, ) app = get_app(demo, markdown_logo=markdown_logo, visible_h2ogpt_logo=kwargs['visible_h2ogpt_logo'], page_title=page_title, ) uvicorn.run(app, # share not allowed host=kwargs['server_name'], port=server_port or 7860, # show_error not allowed ws_max_queue=max_threads, # workers=max_threads, # https://github.com/tiangolo/fastapi/issues/1495#issuecomment-635681976 root_path=kwargs['root_path'], ssl_keyfile=kwargs['ssl_keyfile'], # ssl_verify=kwargs['ssl_verify'], # https://github.com/gradio-app/gradio/issues/2790#issuecomment-2004984763 ssl_certfile=kwargs['ssl_certfile'], ssl_keyfile_password=kwargs['ssl_keyfile_password'], limit_concurrency=None, ) else: demo.launch(share=kwargs['share'], server_name=kwargs['server_name'], server_port= 7861,#server_port, show_error=True, favicon_path=favicon_path, prevent_thread_lock=True, auth=auth, auth_message=auth_message, root_path=kwargs['root_path'], ssl_keyfile=kwargs['ssl_keyfile'], ssl_verify=kwargs['ssl_verify'], ssl_certfile=kwargs['ssl_certfile'], ssl_keyfile_password=kwargs['ssl_keyfile_password'], max_threads=max_threads, allowed_paths=allowed_paths if allowed_paths else None, blocked_paths=blocked_paths if blocked_paths else None, ) showed_server_name = 'localhost' if kwargs['server_name'] == "0.0.0.0" else kwargs['server_name'] if kwargs['verbose'] or not (kwargs['base_model'] in ['gptj', 'gpt4all_llama']): print("Started Gradio Server and/or GUI: server_name: %s port: %s" % (showed_server_name, server_port), flush=True) if server_port is None: server_port = '7860' if kwargs['open_browser']: # Open URL in a new tab, if a browser window is already open. import webbrowser webbrowser.open_new_tab(demo.local_url) else: print("Use local URL: %s" % demo.local_url, flush=True) if kwargs['openai_server'] or kwargs['function_server']: url_split = demo.local_url.split(':') if len(url_split) == 3: gradio_prefix = ':'.join(url_split[0:1]).replace('//', '') gradio_host = ':'.join(url_split[1:2]).replace('//', '') gradio_port = ':'.join(url_split[2:]).split('/')[0] else: gradio_prefix = 'http' gradio_host = ':'.join(url_split[0:1]) gradio_port = ':'.join(url_split[1:]).split('/')[0] # ensure can reach out if platform.system() in ['Darwin', 'Windows']: openai_host = gradio_host if gradio_host not in ['localhost', '127.0.0.1'] else '0.0.0.0' else: if gradio_host in ['localhost', '127.0.0.1']: openai_host = gradio_host = '0.0.0.0' else: openai_host = gradio_host from openai_server.server_start import run run_kwargs = dict(wait=False, multiple_workers_gunicorn=kwargs['multiple_workers_gunicorn'], host=openai_host, gradio_prefix=gradio_prefix, gradio_host=gradio_host, gradio_port=gradio_port, h2ogpt_key=h2ogpt_key1, auth=kwargs['auth'], auth_access=kwargs['auth_access'], guest_name=kwargs['guest_name'], main_kwargs=json.dumps(kwargs['main_kwargs']), verbose=verbose, agent_server=kwargs['agent_server'], openai_server=kwargs['openai_server'], ) if kwargs['openai_server']: time.sleep(5) if verbose: print("Starting up OpenAI proxy server") if kwargs['openai_workers'] == 1: from openai_server.server import app as openai_app else: openai_app = 'server:app' run(**run_kwargs, port=kwargs['openai_port'], app=openai_app, is_openai_server=True, openai_port=kwargs['openai_port'], workers=kwargs['openai_workers'], ) if kwargs['function_server']: time.sleep(5) if verbose: print("Starting up Function server") if kwargs['function_server_workers'] == 1: os.environ['H2OGPT_MAIN_KWARGS'] = run_kwargs['main_kwargs'] from openai_server.function_server import app as function_app else: function_app = 'function_server:app' run(**run_kwargs, port=kwargs['function_server_port'], app=function_app, is_openai_server=False, openai_port=kwargs['openai_port'], workers=kwargs['function_server_workers'], ) if kwargs['agent_server']: time.sleep(5) if verbose: print("Starting up Agent proxy server") if kwargs['agent_workers'] == 1: from openai_server.server import app as agent_app else: agent_app = 'server:app' run(**run_kwargs, port=kwargs['agent_port'], app=agent_app, is_openai_server=False, is_agent_server=True, openai_port=kwargs['openai_port'], workers=kwargs['agent_workers'], ) if kwargs['block_gradio_exit']: demo.block_thread() def show_doc(db1s, selection_docs_state1, requests_state1, langchain_mode1, single_document_choice1, view_raw_text_checkbox1, text_context_list1, pdf_height, h2ogpt_key1, dbs1=None, load_db_if_exists1=None, db_type1=None, use_openai_embedding1=None, hf_embedding_model1=None, migrate_embedding_model_or_db1=None, verbose1=False, get_userid_auth1=None, max_raw_chunks=1000000, api=False, n_jobs=-1, enforce_h2ogpt_api_key=True, enforce_h2ogpt_ui_key=True, h2ogpt_api_keys=[], ): valid_key = is_valid_key(enforce_h2ogpt_api_key, enforce_h2ogpt_ui_key, h2ogpt_api_keys, h2ogpt_key1, requests_state1=requests_state1) from_ui = is_from_ui(requests_state1) if not valid_key: raise ValueError(invalid_key_msg) file = single_document_choice1 document_choice1 = [single_document_choice1] content = None db_documents = [] db_metadatas = [] if db_type1 in ['chroma', 'chroma_old']: assert langchain_mode1 is not None langchain_mode_paths = selection_docs_state1['langchain_mode_paths'] langchain_mode_types = selection_docs_state1['langchain_mode_types'] from gpt_langchain import set_userid, get_any_db, get_docs_and_meta set_userid(db1s, requests_state1, get_userid_auth1) top_k_docs = -1 db = get_any_db(db1s, langchain_mode1, langchain_mode_paths, langchain_mode_types, dbs=dbs1, load_db_if_exists=load_db_if_exists1, db_type=db_type1, use_openai_embedding=use_openai_embedding1, hf_embedding_model=hf_embedding_model1, migrate_embedding_model=migrate_embedding_model_or_db1, for_sources_list=True, verbose=verbose1, n_jobs=n_jobs, ) query_action = False # long chunks like would be used for summarize # the below is as or filter, so will show doc or by chunk, unrestricted from langchain_community.vectorstores import Chroma if isinstance(db, Chroma): # chroma >= 0.4 if view_raw_text_checkbox1: one_filter = \ [{"source": {"$eq": x}, "chunk_id": {"$gte": 0}} if query_action else {"source": {"$eq": x}, "chunk_id": { "$gte": -1}} for x in document_choice1][0] else: one_filter = \ [{"source": {"$eq": x}, "chunk_id": {"$gte": 0}} if query_action else {"source": {"$eq": x}, "chunk_id": { "$eq": -1}} for x in document_choice1][0] filter_kwargs = dict(filter={"$and": [dict(source=one_filter['source']), dict(chunk_id=one_filter['chunk_id'])]}) else: # migration for chroma < 0.4 one_filter = \ [{"source": {"$eq": x}, "chunk_id": {"$gte": 0}} if query_action else {"source": {"$eq": x}, "chunk_id": { "$eq": -1}} for x in document_choice1][0] if view_raw_text_checkbox1: # like or, full raw all chunk types filter_kwargs = dict(filter=one_filter) else: filter_kwargs = dict(filter={"$and": [dict(source=one_filter['source']), dict(chunk_id=one_filter['chunk_id'])]}) db_documents, db_metadatas = get_docs_and_meta(db, top_k_docs, filter_kwargs=filter_kwargs, text_context_list=text_context_list1) # order documents from langchain.docstore.document import Document docs_with_score = [(Document(page_content=result[0], metadata=result[1] or {}), 0) for result in zip(db_documents, db_metadatas)] doc_chunk_ids = [x.get('chunk_id', -1) for x in db_metadatas] doc_page_ids = [x.get('page', 0) for x in db_metadatas] doc_hashes = [x.get('doc_hash', 'None') for x in db_metadatas] docs_with_score = [x for hx, px, cx, x in sorted(zip(doc_hashes, doc_page_ids, doc_chunk_ids, docs_with_score), key=lambda x: (x[0], x[1], x[2])) # if cx == -1 ] db_metadatas = [x[0].metadata for x in docs_with_score][:max_raw_chunks] db_documents = [x[0].page_content for x in docs_with_score][:max_raw_chunks] # done reordering if view_raw_text_checkbox1: content = [dict_to_html(x) + '\n' + text_to_html(y) for x, y in zip(db_metadatas, db_documents)] else: content = [text_to_html(y) for x, y in zip(db_metadatas, db_documents)] content = '\n'.join(content) content = f"""