app.py

import streamlit as st
import pandas as pd
import numpy as np
import json

import time
import requests

import os
import glob
import re
import smart_open
import plotly.express as px 
import random
import difflib
import pdb

from sentence_transformers import SentenceTransformer, models, util

enable_summary_button = True
dump_pos_data_for_reporting = True

bucket_name = "paper_n1"

prefix_lst = [
  "pgj_d_4096", 
  "pgj_d_2048", 
  "pgj_d_1024_v2", 
  "pgj_d_1024_layer_14", 
  "pgj_d_1024_layer_7", 
  "pgj_d_1024_layer_2", 
  "pgj_d_1024_layer_1" ]

  # "my_gptj_6b_tpu_size_8", 

model_names = {
  prefix_lst[0]: 'PatentGPT-J-6B',
  prefix_lst[1]: 'PatentGPT-J-1.6B',

  # prefix_lst[2]: 'PatentGPT-J-279M',
  # prefix_lst[3]: 'PatentGPT-J-191M',
  # prefix_lst[4]: 'PatentGPT-J-128M',
  # prefix_lst[5]: 'PatentGPT-J-115M',}

  prefix_lst[2]: 'PatentGPT-J-456M',
  prefix_lst[3]: 'PatentGPT-J-279M',
  prefix_lst[4]: 'PatentGPT-J-191M',
  prefix_lst[5]: 'PatentGPT-J-128M',
  prefix_lst[6]: 'PatentGPT-J-115M',}

  # prefix_lst[7]:'GPT-J-6B'

# experiment 3
# folder = os.path.join('experiments', 'non_patent')
# id_to_scroll = 1  # which of the above to scroll through
# first_claim_only = True 

#experiment 2
# folder = os.path.join('experiments', 'ipg20220104_500')
# #folder = "device_serve_results"
# id_to_scroll = 1  # which of the above to scroll through
# first_claim_only = False

# prefix_lst = ["my_gptj_6b_tpu_size_8", "pgj_d_4096", "pgj_d_2048", "pgj_d_1024_layer_14", "pgj_d_1024_layer_7", "pgj_d_1024_layer_2", "pgj_d_1024_layer_1"]
# #, "pgj_large", "pgj_medium", "pgj_small", ]
# # "pgj_d_1024_layer_14"

# experiment 1
folder = os.path.join('experiments', 'ipg22_500')
# (previous) folder = "eval_ipg22_500"
id_to_scroll = 1  # which of the above to scroll through
first_claim_only = True 

ignore_outscope = True  # ignore pick > 10

def show_diff(a, b):
  #print('{} => {}'.format(a,b))  
  for i, s in enumerate(difflib.ndiff(a, b)):
      if s[0]==' ': continue
      elif s[0]=='-':
          print(u'Delete "{}" from position {}'.format(s[-1],i))
      elif s[0]=='+':
          print(u'Add "{}" to position {}'.format(s[-1],i))    

def handle_char_return(text):
  if text == '(none)':  # unicorn text
    text == ''

  return text
  
  #return ch.replace('\n', '\\n')

  #if ch == '\n':
  #  ch = "'\\n'"
  #return ch

def get_remaining(lst, pos):
  s = ''
  for i in range(pos, len(lst)):
    text = lst[i]['actual_next_token_text']
    if text.startswith(' ') == False:
      s += text
    else:
      break  

  return s

def calc_details(base_fn):
  full_fn = os.path.join(folder, base_fn)
  #gs_fn = "gs://%s/%s/%s" % (bucket_name, folder, base_fn)
  #with smart_open.open(gs_fn) as f:

  if os.path.exists(full_fn) == False:
    return None, -1, -1, None, None, None, None, None

  with open(full_fn) as f:
    result = json.loads(f.read())
    print("Loaded: %s" % full_fn)

  lst = result['output']
  recv = result['recv']
  sum_pick = 0
  sum_prob = 0 
  sum_outscope_count = 0
  sum_outscope_len = 0
  sum_hit_1 = 0
  sum_top_10_len = 0
  full_text = ''

  token_count = 0
  #found_end = False
  
  #pdb.set_trace()

  for i, tk in enumerate(lst[:-1]):
    # if found_end:
    #   break 

    token_text = handle_char_return(tk['actual_next_token_text'])

    # Due to tokenizer difference, the following needs more work in the future.
    # if base_fn.find('gptj') >= 0: 
    #   # using the original gpt-j-6b model
    #   # need to skip special tokens
    #   if i <= 7:
    #     continue  # skip |start of claim|>

    #   remaining_text = get_remaining(lst, i)
    #   if remaining_text.find('<|end_of_claim|>') >= 0:
    #     pos1 = remaining_text.find('<|end_of_claim|>')
    #     token_text = remaining_text[:pos1]
    #     found_end = True
    #     #pdb.set_trace() 
    #     #break

    # The following was for GPT-J-6B. Not needed for PatentGPT-J. 
    #if token_text.find('<|end_of_claim|>') == 0: 
    #  #pdb.set_trace()
    #  break
    
    next_top_seq = int(tk['actual_next_token_top_seq'])
    next_top_prob = float(tk['actual_next_token_top_prob']) 

    full_text += token_text
    if next_top_seq == 0:
      sum_hit_1 += 1   # press "tab" for the top pick

    if ignore_outscope and next_top_seq>=10: 
      sum_outscope_count += 1
      sum_outscope_len += len(token_text)  # use length as keystrokes 
    else:
      sum_pick += min(next_top_seq+1, len(token_text))
      #sum_pick += (next_top_seq+1) # press "down" & "tab"
      sum_prob += next_top_prob
      sum_top_10_len += len(token_text)

    token_count += 1

  if ignore_outscope: 
    if token_count == 0: # unlikely
      avg_pick = 0
      avg_prob = 0
    else:
      avg_pick = float(sum_pick) / token_count
      avg_prob = float(sum_prob) / token_count
  else:
    avg_pick = float(sum_pick) / token_count
    avg_prob = float(sum_prob) / token_count  

  # if len(lst) < 2048:  # for debugging
  #   s = '<|start_of_claim|>' + full_text
  #   if len(s) != len(recv['context']):
  #     print('length mismatch --> full_text: %s, recv: %s' % (len(s), len(recv['context'])))
  #     show_diff(s, recv['context'])
  #     pdb.set_trace() 

  return result, avg_pick, avg_prob, token_count, sum_pick, sum_prob, sum_outscope_count, sum_outscope_len, sum_hit_1, sum_top_10_len, full_text

def show_avg(base_fn, model_name, patent_claim_num, show_pick=False): 

  result, avg_pick, avg_prob, token_count, sum_pick, sum_prob, sum_outscope_count, sum_outscope_len, sum_hit_1, sum_top_10_len, full_text = calc_details(base_fn)  

  if token_count == 0:
    print('debug 2')
    pdb.set_trace()

  if result is None:
    return None

  lst = result['output']
  result = ''
  sum_all = {}
  for i, tk in enumerate(lst):
    token_text = handle_char_return(tk['actual_next_token_text'])

    if token_text == '<|end_of_claim|>': 
      break

    if token_text == '(none)': # for unicorn text
      break      

    # Skip GPT-J, due to different tokenization  
    # if base_fn.find('gptj') >= 0: 
    #   # using the original gpt-j-6b model
    #   # need to skip special tokens
    #   if i <= 7:
    #     continue  # skip |start of claim|>
    #   if token_text == '.<': # assuming .<|end of claim|>
    #     break

    pick = int(tk['actual_next_token_top_seq'])
    prob = float(tk['actual_next_token_top_prob'])

    colors = [
      ['00ff00', '000000', '1'], 
      ['008800', 'ffffff', '2-10'], 
      ['ff0000', 'ffffff', 'out of top 10'], 
    ]
    #colors = [
    #  ['00ff00', '000000', '1'], 
    #  ['008800', 'ffffff', '2-10'], 
    #  ['aa0000', 'ffffff', '11-100'], 
    #  ['ff0000', 'ffffff', '101~']
    #]

    for j, item in enumerate(colors):
      sum_all[item[2]] = 0

    # skip follow-up subword 
    # if token_text.startswith(' ') == False:
    #   bg_color = ''
    #   fg_color = '' 
    # else:     

    if pick == 0:
      bg_color = colors[0][0]
      fg_color = colors[0][1]
      tag = colors[0][2]
      sum_all[tag] += 1
    elif pick >= 1 and pick < 10:
      bg_color = colors[1][0]
      fg_color = colors[1][1]
      tag = colors[1][2]
      sum_all[tag] += 1
    else: # pick >= 10
      #elif pick >= 10 and pick < 100:
      bg_color = colors[2][0]
      fg_color = colors[2][1]
      tag = colors[2][2]
      sum_all[tag] += 1
    #else:   #pick >= 100:
    #  bg_color = colors[3][0]
    #  fg_color = colors[3][1]
    #  tag = colors[3][2]
    #  sum_all[tag] += 1

    if show_pick:
      pick = '[%s]' % pick
    else:
      pick = ''

    result += "<span style=background-color:#%s;color:#%s;border-radius:5px;>%s%s</span>  " % (bg_color, fg_color, token_text, pick) #&nbsp;

  color_msg = ''
  for i, v in enumerate(colors):
    color_msg += "<span style=background-color:#%s;color:#%s;border-radius:5px;>&nbsp;%s&nbsp;</span> " % (v[0], v[1], v[2]) 

  #result, avg_pick, avg_prob, token_count, sum_pick, sum_prob, sum_outscope, sum_hit_1, sum_top_10_len, full_text = calc_details(base_fn)  

  # sum_pick as top 1~10
  keys_with_auto = (sum_pick+sum_outscope_len)
  keys_without_auto = len(full_text)
  saved_ratio = float(keys_without_auto-keys_with_auto)/keys_without_auto * 100
  s = 'model: %s\n' \
    'Autocomplete Effectiveness: %.1f%% (keystrokes saved)\n' \
    'Total keystrokes: %s (with autocomplete), %s (without autocomplete)\n' \
    'Keystroke distribution: top 1~10: %s (top 1: %s), out of top 10: %s' % (model_name, saved_ratio, keys_with_auto, keys_without_auto,  sum_pick, sum_hit_1, sum_outscope_len)
  st.text(s)

  # s = 'file: %s, sum_pick: %s, sum_hit_1: %s, token_count: %s, sum_outscope: %s, avg_pick: %.2f, avg_prob: %.2f, sum_prob: %.2f, hit_1 ratio: %.2f  &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;' % (base_fn, sum_pick, sum_hit_1, token_count, sum_outscope, avg_pick, avg_prob, sum_prob, float(sum_hit_1)/token_count)
  #s += color_msg

  s = color_msg
  st.markdown(s, unsafe_allow_html=True)
  #st.text('file: %s, avg_pick: %5.2f, avg_prob: %.2f, hit count: %s/%s  ' % (base_fn, avg_pick, avg_prob, hit_0_count, len(lst)))
  # show histogram

  st.markdown(result, unsafe_allow_html=True)
  #st.text_area('context with top seq & prob:', result, height=400)  

  sum_lst = [sum_all['1'], sum_all['2-10'], sum_all['out of top 10']]
  #sum_lst = [['1', sum_all['1']], ['2-10', sum_all['2-10']]]
  #sum_lst = [sum_all['1'], sum_all['2-10'], sum_all['11-100'], sum_all['101~']]

  return sum_lst

def show_overall_summary(prefix_lst, select_lst):  
  # accumulate all

  # debug
  # for i, num in enumerate(select_lst): 
  #   pre_full_text = ''
  #   for prefix in prefix_lst:
  #     base_fn = '%s_%s_forward.json' % (prefix, num)
  #     result, avg_pick, avg_prob, token_count, sum_pick, sum_prob, sum_outscope, sum_hit_1, sum_top_10_len, full_text = calc_details(base_fn)  

  #     if pre_full_text == '':
  #       pre_full_text = full_text
  #     else:
  #       if pre_full_text != full_text:
  #         print('debug')
  #         pdb.set_trace() 

  # # 
  # pdb.set_trace()

  for prefix in prefix_lst:
    acc_token_count = 0
    acc_sum_pick = 0
    acc_sum_prob = 0 
    acc_sum_outscope_count = 0 
    acc_sum_outscope_len = 0 
    acc_sum_hit_1 = 0
    acc_sum_top_10_len = 0 
    acc_full_text_len = 0

    pre_full_text = ''
    for i, num in enumerate(select_lst):    
      base_fn = '%s_%s_forward.json' % (prefix, num)
      result, avg_pick, avg_prob, token_count, sum_pick, sum_prob, sum_outscope_count, sum_outscope_len, sum_hit_1, sum_top_10_len, full_text = calc_details(base_fn)  

      acc_token_count += token_count
      acc_sum_pick += sum_pick
      acc_sum_prob += sum_prob
      acc_sum_outscope_count += sum_outscope_count
      acc_sum_outscope_len += sum_outscope_len
      acc_sum_hit_1 += sum_hit_1
      acc_sum_top_10_len += sum_top_10_len
      acc_full_text_len += len(full_text)

    if acc_token_count > 0:
      # acc_sum_pick --> top 1~10
      keys_with_auto = acc_sum_pick + acc_sum_outscope_len
      keys_without_auto = acc_full_text_len
      saved_ratio = float(keys_without_auto-keys_with_auto)/keys_without_auto * 100

      st.text('[ %s ]\n' \
        'Autocomplete Effectiveness: %.1f%% (ratio of saving keystroke)\n' \
        '(sum) keys_with_auto: %s, top_10_keys: %s, out_of_scope: %s, sum_hit_1: %s\n' \
        'keys_without_auto: %s, top_10_len: %s, prob: %.2f' % (
        model_names[prefix], saved_ratio, 
        '{:,}'.format(keys_with_auto),
        '{:,}'.format(acc_sum_pick), 
        '{:,}'.format(acc_sum_outscope_len),
        '{:,}'.format(acc_sum_hit_1), 
        '{:,}'.format(keys_without_auto),
        '{:,}'.format(acc_sum_top_10_len), 
        acc_sum_prob, 
        ))

      st.text('%s & %.1f\\%% & %s & %s & %s & %s & %s \\\\' % (model_names[prefix], saved_ratio, '{:,}'.format(keys_with_auto), '{:,}'.format(acc_sum_pick), '{:,}'.format(acc_sum_outscope_len), '{:,}'.format(acc_sum_hit_1), '{:,}'.format(keys_without_auto)))

      # st.text('* acc_token_count =%s --> (avg) hits: %.2f, keys: %.2f, prob: %.2f, outscope: %.2f' % (
      #     acc_token_count, 
      #     float(acc_sum_hit_1)/acc_token_count,
      #     float(acc_sum_pick)/acc_token_count, 
      #     float(acc_sum_prob)/acc_token_count, 
      #     float(acc_sum_outscope_count)/acc_token_count))

def calc_height(s):
  return int(len(s) / 10 * 3) + 30

def remove_end_of_claim_text(gen_text):  
  tag = '<|end_of_claim|>'
  pos = gen_text.find(tag)
  if pos > 0:
    gen_text = gen_text[:pos+len(tag)]
    return gen_text

  tag = '<|endoftext|>'
  pos = gen_text.find(tag)
  if pos > 0:
    gen_text = gen_text[:pos+len(tag)]

  return gen_text

def dump_pos_data(prefix_lst, select_lst):
  #statistics = [[0]*3]*2048
  statistics = []
  for i in range(2048):
    statistics.append([0,0,0])

  #results.append(['model', 'pos', 'key'])
  #results.append(['model', 'patent_claim', 'pos', 'top-1', 'top-2~10', 'out of top 10'])
  max_len = -1
  for prefix in prefix_lst:
    model_name = model_names[prefix].replace('PatentGPT-J-', '')
    if model_name != '456M':
      continue

    #total = {}
    for i, num in enumerate(select_lst):    
      base_fn = '%s_%s_forward.json' % (prefix, num)
      full_fn = os.path.join(folder, base_fn)
      if os.path.exists(full_fn) == False:
        continue 

      with open(full_fn) as f:
        result = json.loads(f.read())
        print("Loaded: %s" % full_fn)

      lst = result['output']
      for j, tk in enumerate(lst[:-1]):
        max_len = max(j, max_len)        
        next_top_seq = int(tk['actual_next_token_top_seq'])
        #next_top_prob = float(tk['actual_next_token_top_prob']) 

        top_1 = top_2_to_10 = out_of_scope = 0
        if next_top_seq == 0:
          top_1 = 1
          tag = 'top-1'
          statistics[j][0] += 1
        elif next_top_seq > 0 and next_top_seq < 10:
          top_2_to_10 = 1
          tag = 'top-2~10'
          statistics[j][1] += 1
        else:
          out_of_scope = 1
          tag = 'out-of-scope'
          statistics[j][2] += 1

        #total[tag] = total.get(tag, 0) + 1
        #results.append([model_name, str(i+1), tag])
        #results.append([model_name, str(i+1), tag])
        #results.append([model_name, num, str(i+1), tag])
        #results.append([model_name, num, i+1, top_1, top_2_to_10, out_of_scope])
        #pdb.set_trace()
      #pdb.set_trace() 

  dump_file = 'dump4.txt'
  #pdb.set_trace() 
  with open(dump_file, 'w') as f:
    for i in range(max_len+1):
      f.write('%s, top-1, %s\n' % (i+1, statistics[i][0]))
      f.write('%s, top-2~10, %s\n' % (i+1, statistics[i][1]))
      f.write('%s, out_of_scope, %s\n' % (i+1, statistics[i][2]))
      # f.write('%s\n' % ', '.join([str(i+1)] + [ str(v) for v in statistics[i] ] ))
  print('saved: %s' % dump_file)


  # dump_file = 'dump2.txt'
  # with open(dump_file, 'w') as f:
  #   for line in results:
  #     f.write('%s\n' % ', '.join(line))
  # print('saved: %s' % dump_file)


def calc_sentence_similarity(sent_model, sent1, sent2):
  rewards = []
  embedding1 = sent_model.encode(sent1, convert_to_tensor=True)
  embedding2 = sent_model.encode(sent2, convert_to_tensor=True)
  similarity = util.cos_sim(embedding1, embedding2)[0][0]

  #pdb.set_trace()
  
  return similarity

sent_model = 'patent/st-aipd-nlp-g' 
print('loading SentenceTransformer: %s' % sent_model)
sent_aipd = SentenceTransformer(sent_model)

def load_data(demo):
  fn = 'ppo_output/ppo_open_llama_3b_v2.run.12.delta.txt'
  with open(fn, 'r') as f:
    rows = json.load(f)

  if demo == 'demo1':
    new_rows = [ row for row in rows if row['instruction'].find('child') > 0 ]
  elif demo == 'demo2':
    new_rows = [ row for row in rows if row['instruction'].find('parent') > 0 ]
  else:
    new_rows = []    

  return new_rows

container_style = """
  <style>
      .container1 {
          border: 2px solid #3498db;
          border-radius: 8px;
          padding: 10px;
          margin-bottom: 20px;
      }
      .container2 {
          /* Add styles for Container 2 if needed */
      }
  </style>
"""

def main():
  st.set_page_config(  # Alternate names: setup_page, page, layout
    layout="wide",  # Can be "centered" or "wide". In the future also "dashboard", etc.
    initial_sidebar_state="auto",  # Can be "auto", "expanded", "collapsed"
    page_title="Demo 1",  # String or None. Strings get appended with "• Streamlit".
    page_icon=None,  # String, anything supported by st.image, or None.
  )

  opt_1 = 'parent --> child'
  opt_2 = 'child --> parent'
  options = [opt_1, opt_2]
  rows = None
  pos = None
  patent_num = ''
  claim_num1 = ''
  claim_num2 = ''
  instruction= ''
  input_text = ''
  output_text = ''
  response = ''
  query = ''
  score_lst_1 = 0
  score_lst_2 = 0
  rewards = ''  
  with st.container():
    col1, col2, col3 = st.columns([3, 5, 2])
    with col1:
      selected_option = st.selectbox('Select a demo:', options)
    if selected_option == opt_1:
      rows = load_data('demo1')
      msg = 'novelty = sim1-sim2'
      #msg = 'delta of similarities<br>(sim1-sim2)'
      c1_tag = 'pc'
      c2_tag = 'cc1'
      c3_tag = 'cc2'
    elif selected_option == opt_2:  
      rows = load_data('demo2')
      msg = 'similarity of<br>(pc1) and (pc2)'
      c1_tag = 'cc'
      c2_tag = 'pc1'
      c3_tag = 'pc2'
    else:
      st.text('Unknown option')
      return
    #rows = rows[:5000] # for debugging

    with col2:
      pos = st.slider("", 1, len(rows))
      #pos = st.slider("Degree of novelty (Generated v. Actual)", 1, len(rows))
      for i in range(pos):
        #prompt = '%s' % rows[i]
        #pdb.set_trace()

        patent_num = rows[i]['patent_num']
        claim_num1 = rows[i]['claim_num1']
        claim_num2 = rows[i]['claim_num2']
        instruction= rows[i]['instruction']
        input_text = rows[i]['input']
        output_text = rows[i]['output']
        response = rows[i]['response']
        query = rows[i]['query']
        score_lst_1 = rows[i]['score_lst_1']
        score_lst_2 = rows[i]['score_lst_2']
        delta = rows[i]['delta']
        rewards = rows[i]['rewards']
    with col3:
      #v = round(float(score_lst_1)-float(score_lst_2), 4)
      #v = delta #round(delta,10)
      st.markdown("<center><h7>%s<br>%s</h7></center>" % (msg, delta), unsafe_allow_html=True)
      #  style='text-align: center; color: black;'
        

  # selectbox_placeholder = st.empty()
  # selected_option = selectbox_placeholder.selectbox('Select a demo:', options)
  # container1 = st.container()


  # with st.container():
  #     col1, col2 = st.columns(2)
  #     with col1:
  #         st.write('Caption for first chart')
  #     with col2:
  #         st.line_chart((0,1), height=100)
  # with st.container():
  #     col1, col2 = st.columns(2)
  #     with col1:
  #         st.write('Caption for second chart')
  #     with col2:
  #         st.line_chart((1,0), height=100)

  #st.write('patent_num:', patent_num)
  # st.write('claim_num1:', claim_num1)
  # st.write('claim_num2:', claim_num2)
  st.write('(instruction) ', instruction)

  with st.container():
    with st.container(border=True):
      st.write('(%s) [ %s ]\n%s' % (c1_tag, patent_num, input_text))
      #st.write('input:' % patent_num)
      #st.write('input:\n', input_text)

    #container1.markdown("<div class='container1'>", unsafe_allow_html=True)
    col1, col2 = st.columns(2)
    with col1:
      with st.container(border=True):
        st.write('(%s) (actual)' % c2_tag)
        st.write(output_text)
    with col2:
      with st.container(border=True):
        st.write('(%s) (generated)' % c3_tag)
        st.write(response)

    col1, col2 = st.columns(2)
    with col1:
      with st.container(border=True):
        st.write('(sim1) similarity between %s and %s+%s: %s' % (c1_tag, c1_tag, c2_tag, str(score_lst_1)))
    with col2:
      with st.container(border=True):
        st.write('(sim2) similarity between %s and %s+%s: %s' % (c1_tag, c1_tag, c3_tag, str(score_lst_2)))

    #container1.markdown("</div>", unsafe_allow_html=True)

    # st.write("In Container 1")
    # table_name = st.radio("Please Select Table", list_of_tables)

  # st.write('output:')
  # st.write(output_text)
  # st.write('response:')
  # st.write(response)
  #st.write('query:', query)
  # st.write('score_lst_1:', score_lst_1)
  # st.write('score_lst_2:', score_lst_2)
  # st.write('rewards:', rewards)
  # st.text('hello')

  # dict_keys(['patent_num', 'claim_num1', 'claim_num2', 'instruction', 'input', 'output', 'query', 'response', 'score_lst_1', 'score_lst_2', 'rewards'])

  # st.subheader("Inspecting PatentGPT-J Model Evaluation")



  # num_set = set()
  # fn_lst = glob.glob(os.path.join(folder, '*'))
  # for i, fn in enumerate(fn_lst):
  #   for prefix in prefix_lst:    
  #     v = re.search('(.*?)%s\_(\d+\_\d+)\_(.*?)' % prefix, fn)
  #     if v is None:
  #       v = re.search('(.*?)%s\_(\w+\_\d+)\_(.*?)' % prefix, fn)

  #       #pdb.set_trace()
  #       if v is None:
  #         #pdb.set_trace()
  #         continue

  #     v = v.group(2)
  #     if first_claim_only:
  #       if v.endswith('_1'):
  #         num_set.add(v)
  #     else: 
  #       num_set.add(v)

  # num_lst = list(num_set)
  # num_lst.sort()

  # select_lst = []
  # for i, num in enumerate(num_lst):
  #   all_existed = True
  #   for prefix in prefix_lst:
  #     fn = os.path.join(folder, '%s_%s_forward.json' % (prefix, num))
  #     if os.path.exists(fn) == False:
  #       all_existed = False
  #       break
  #   if all_existed: 
  #     select_lst.append(num)
  # select_lst.sort()

  # if len(select_lst) == 0:
  #   st.text('select_lst is empty')
  #   return 

  # if dump_pos_data_for_reporting:
  #   dump_pos_data(prefix_lst, select_lst)
  #   st.text('Dump data: done')
  #   return

  # # debug
  # #base_fn = 'my_gptj_6b_tpu_size_8_11212952_1_forward.json'
  # #base_fn = 'pgj_small_text-1_1_forward.json'
  # #_ = show_avg(base_fn)

  # if enable_summary_button:
  #   if st.button('Show Summary'):
  #     st.text('len(select_lst) = %s' % len(select_lst))
  #     show_overall_summary(prefix_lst, select_lst)

  # # if 'num' not in st.session_state:
  # #   num = random.choice(select_lst)
  # #   st.session_state['num'] = num

  # # set_state('num', num)
  # # def set_state(k, v):
  # #   if k not in st.session_state:
  # #     st.session_state[ k ] = v

  # show_patent_lst = [ s.replace('_', ' (claim ') + ')' for s in select_lst]
  # selected = st.selectbox("Choose a patent claim", show_patent_lst)
  # num = selected.replace(')', '').replace(' (claim ', '_')
  # if st.button('Random pick'):
  #   num = random.choice(select_lst)

  # st.text('Selected: %s' % num)
  # st.session_state['num'] = num

  # avgs = []
  # for prefix in prefix_lst:
  #   base_fn = '%s_%s_forward.json' % (prefix, num)
  #   one_avg = show_avg(base_fn, model_names[prefix], num)
  #   if one_avg is not None:
  #     avgs.append(one_avg) 

  # # debug
  # #pdb.set_trace()
  # #return 
  # #

  # data_lst = []
  # for i in range(len(avgs[0])):
  #   row = []
  #   for j, prefix in enumerate(prefix_lst):
  #     row.append(avgs[j][i])
  #   data_lst.append(row)

  # df = pd.DataFrame(data_lst, index=['1','2-10','out of top 10'])
  # #df = pd.DataFrame(data_lst, index=['1','2-10','11-100','101~'])

  # #  ], index=['(a) 1','(b) 2-10','(c) 11-100','(d) 101~'])
  # #  [avgs[0][0], avgs[1][0], avgs[2][0]],
  # #  [avgs[0][1], avgs[1][1], avgs[2][1]],
  # #  [avgs[0][2], avgs[1][2], avgs[2][2]],
  # #  [avgs[0][3], avgs[1][3], avgs[2][3]],
  
  # #df = pd.DataFrame([[1,2],[3,1]], columns=['a', 'b'])
  # #df = pd.DataFrame([
  # #  [sum1[0], sum1[1], sum1[2], sum1[3]],
  # #  [sum2[0], sum2[1], sum2[2], sum2[3]],
  # #  [sum3[0], sum3[1], sum3[2], sum3[3]],
  # #  ]) #, index=['(a) 1','(b) 2-10','(c) 11-100','(d) 101~'])
  # #df = pd.DataFrame.from_dict(sum_all, orient='index')
  # #st.line_chart(df)

  # #data_canada = px.data.gapminder().query("country == 'Canada'")
  # #fig = px.bar(data_canada, x='year', y='pop')

  # if st.button('Show chart'):
  #   fig = px.bar(df, barmode='group')
  #   st.plotly_chart(fig, use_container_width=True)
  #   #fig.show()
  #   #st.area_chart(df)  
  #   #st.bar_chart(df)    

  # #
  # base_fn = '%s_%s_forward.json' % (prefix_lst[ id_to_scroll ], st.session_state['num'])
  # result, avg_pick, avg_prob, _, _, _, _, _, _, _, _ = calc_details(base_fn)
  # recv = result['recv']
  # lst = result['output']
  # input_tokens = result['input']

  # # (Pdb) print(token_pos_lst[0].keys())
  # #dict_keys(['idx', 'gen_text', 'actual_next_token_text', 'actual_next_token_top_seq', 'actual_next_token_top_prob', 'top_n_lst'])

  # height = calc_height(recv['context'])
  # st.text_area('context:', recv['context'], height=height)

  # pos = st.slider("Token position", 0, len(lst))
  # prompt = ''
  # for i in range(pos+1):
  #   prompt += input_tokens[i]['text']
  # height = calc_height(prompt)
  # st.text_area('prompt:', prompt, height=height)

  # ch = handle_char_return(lst[pos]['actual_next_token_text'])
  # st.text('actual_next_token_text: %s --> pick seq: %s (prob: %.2f)' % (ch, int(lst[pos]['actual_next_token_top_seq'])+1, 
  #   float(lst[pos]['actual_next_token_top_prob'])))

  # st.text('top 10 tokens:')
  # for i, v in enumerate(lst[pos]['top_n_lst']):
  #   ch = handle_char_return(v['top_n_text'])
  #   st.text('[ %s ][ %s ]( %.2f )' % (i+1, ch, float(v['top_n_prob'])))

  # gen_text = lst[pos]['gen_text']
  # gen_text = remove_end_of_claim_text(gen_text)

  # st.text('gen_text: %s' % gen_text)
  # #st.text("done. ok.")
  # #st.text('result:\n%s' % result)

if __name__ == "__main__":
  main()

#def load_data_pre(demo):
#   fn = 'ppo_output/ppo_open_llama_3b_v2.run.12.keep.txt'
#   with open(fn, 'r') as f:
#     rows = json.load(f)

#   new_rows = []
#   for i, row in enumerate(rows):
#     item1 = {}
#     item2 = {}
#     if demo == 'demo1':
#       item1[ 'delta' ] = abs(row['score_lst_1'][0] - row['score_lst_2'][0])
#       item2[ 'delta' ] = abs(row['score_lst_1'][1] - row['score_lst_2'][1])
#     elif demo == 'demo2':
#       #pdb.set_trace()
#       item1[ 'delta' ] = calc_sentence_similarity(sent_aipd, row['output'][0], row['response'][0])
#       item2[ 'delta' ] = calc_sentence_similarity(sent_aipd, row['output'][1], row['response'][1])

#       print('[ %s ] detla = %s' % (i, item1[ 'delta' ]))

#     for k in row.keys():
#       item1[ k ] = row[ k ][0]
#       item2[ k ] = row[ k ][1]

#     if demo == 'demo1':
#       if item1['instruction'].find('child') > 0:
#         new_rows.append(item1)
#       if item2['instruction'].find('child') > 0:
#         new_rows.append(item2)
#     elif demo == 'demo2':
#       if item1['instruction'].find('parent') > 0:
#         new_rows.append(item1)
#       if item2['instruction'].find('parent') > 0:
#         new_rows.append(item2)

#   # Assuming new_rows is your list of dictionaries
#   sorted_rows = sorted(new_rows, key=lambda x: x['delta'])

#   # kv = {}
#   # for i, row in enumerate(new_rows):
#   #   if diff > 0.0001:
#   #     kv[i] = round(diff, 4)

#   # sorted_rows = []
#   # sorted_kv = sorted(kv.items(), key=lambda x:x[1])
#   # for k, v in sorted_kv:
#   #   sorted_rows.append(new_rows[k])

#   #pdb.set_trace() 

#   return sorted_rows