adding these files as a backup of an older project that got mangled by Git LFS's size limit

.devcontainer/devcontainer.json

@@ -0,0 +1,22 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:0-3.11"
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.gitattributes

@@ -0,0 +1 @@
+data/*.json filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+**/.DS_Store
+
+models/*

.python-version

@@ -0,0 +1 @@
+3.9.16

data/train.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:086044dc3464c21b497dffcccd8358731d55454ac2420c6930b7c358502db8ae
+size 58741536

data/val.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:922dccb1d4d0d2a7ba05651a36a3cbf79c991d17e15da9d4d71f2d90d02c20fd
+size 32823037

misc_example/Dockerfile

@@ -0,0 +1,4 @@
+FROM node:alpine
+COPY . /misc_example
+WORKDIR /misc_example
+CMD node example.js

misc_example/example.js

@@ -0,0 +1,2 @@
+console.log("Hello World!")
+console.log("- Ryan Kim (rk2546)")

src/app.py

@@ -0,0 +1,18 @@
+import random
+import numpy as np
+
+random_vals = random.sample(range(10, 30), 5)
+print(random_vals)
+
+arr = np.array( [[ 1, 2, 3],
+                 [ 4, 2, 5]] )
+print(arr)
+
+arr2 = np.array(
+    [
+        [2,5],
+        [2,6],
+        [100,34]
+    ]
+)
+print(arr2)

src/emotion.py

@@ -0,0 +1,87 @@
+import streamlit as st
+from transformers import pipeline
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+import os
+
+# We'll be using Torch this time around
+import torch
+import torch.nn.functional as F
+
+# === VARIABLE DECLARATION ===
+model_names = (
+    "cardiffnlp/twitter-roberta-base-sentiment",
+    "finiteautomata/beto-sentiment-analysis",
+    "bhadresh-savani/distilbert-base-uncased-emotion",
+    "siebert/sentiment-roberta-large-english"
+)
+
+def label_dictionary(model_name):
+    if model_name == "cardiffnlp/twitter-roberta-base-sentiment":
+        def twitter_roberta(label):
+            if label == "LABEL_0":
+                return "Negative"
+            elif label == "LABEL_2":
+                return "Positive"
+            else:
+                return "Neutral"
+        return twitter_roberta
+    return lambda x: x
+
+@st.cache(allow_output_mutation=True)
+def load_model(model_name):
+    model = AutoModelForSequenceClassification.from_pretrained(model_name)
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    classifier = pipeline(task="sentiment-analysis", model=model, tokenizer=tokenizer)
+    parser = label_dictionary(model_name)
+    return model, tokenizer, classifier, parser
+
+# We first initialize a state. The state will include the following:
+# 1) the name of the model (default: cardiffnlp/twitter-roberta-base-sentiment)
+# 2) the model itself, and 
+# 3) the parser for the outputs, in case we actually need to parse the output to something more sensible
+if "model" not in st.session_state:
+    st.session_state.model_name = "cardiffnlp/twitter-roberta-base-sentiment"
+    model, tokenizer, classifier, label_parser = load_model("cardiffnlp/twitter-roberta-base-sentiment")
+    st.session_state.model = model
+    st.session_state.tokenizer = tokenizer
+    st.session_state.classifier = classifier
+    st.session_state.label_parser = label_parser
+
+def model_change():
+    model, tokenizer, classifier, label_parser = load_model(st.session_state.model_name)
+    st.session_state.model = model
+    st.session_state.tokenizer = tokenizer
+    st.session_state.classifier = classifier
+    st.session_state.label_parser = label_parser
+
+model_option = st.selectbox(
+    "What sentiment analysis model do you want to use?",
+    model_names,
+    on_change=model_change,
+    key="model_name"
+)
+placeholder="@AmericanAir just landed - 3hours Late Flight - and now we need to wait TWENTY MORE MINUTES for a gate! I have patience but none for incompetence."
+form = st.form(key='sentiment-analysis-form')
+text_input = form.text_area("Enter some text for sentiment analysis! If you just want to test it out without entering anything, just press the \"Submit\" button and the model will look at the placeholder.", placeholder=placeholder)
+submit = form.form_submit_button('Submit')
+
+if submit:
+    if text_input is None or len(text_input.strip()) == 0:
+        to_eval = placeholder
+    else:
+        to_eval = text_input.strip()
+    st.write("You entered:")
+    st.markdown("> {}".format(to_eval))
+    st.write("Using the NLP model:")
+    st.markdown("> {}".format(st.session_state.model_name))
+    result = st.session_state.classifier(to_eval)  
+    label = result[0]['label']
+    score = result[0]['score']
+
+    label = st.session_state.label_parser(label)
+
+    st.markdown("#### Result:")
+    st.markdown("**{}**: {}".format(label,score))
+    st.write("")
+    st.write("")

src/main.py

@@ -0,0 +1,265 @@
+import os
+import json
+import random
+
+import streamlit as st
+from transformers import TextClassificationPipeline, pipeline
+from transformers import AutoTokenizer, AutoModelForSequenceClassification, DistilBertTokenizerFast, DistilBertForSequenceClassification
+
+emotion_model_names = (
+    "cardiffnlp/twitter-roberta-base-sentiment",
+    "finiteautomata/beto-sentiment-analysis",
+    "bhadresh-savani/distilbert-base-uncased-emotion",
+    "siebert/sentiment-roberta-large-english"
+)
+
+class ModelImplementation(object):
+    def __init__(
+            self, 
+            transformer_model_name, 
+            model_transformer, 
+            tokenizer_model_name,
+            tokenizer_func, 
+            pipeline_func, 
+            parser_func,
+            classifier_args={},
+            placeholders=[""]
+    ):
+        self.transformer_model_name = transformer_model_name
+        self.tokenizer_model_name = tokenizer_model_name
+        self.placeholders = placeholders
+
+        self.model = model_transformer.from_pretrained(self.transformer_model_name)
+        self.tokenizer = tokenizer_func.from_pretrained(self.tokenizer_model_name)
+        self.classifier = pipeline_func(model=self.model, tokenizer=self.tokenizer, padding=True, truncation=True, **classifier_args)
+        self.parser = parser_func
+    
+    def predict(self, val):
+        result = self.classifier(val)
+        return self.parser(self, result)
+
+def ParseEmotionOutput(self, result):
+    label = result[0]['label']
+    score = result[0]['score']
+    output_func = st.info
+    if self.transformer_model_name == "cardiffnlp/twitter-roberta-base-sentiment":
+        if label == "LABEL_0":
+            label = "NEGATIVE"
+            output_func = st.error
+        elif label == "LABEL_2":
+            label = "POSITIVE"
+            output_func = st.success
+        else:
+            label = "NEUTRAL"
+    elif self.transformer_model_name == "finiteautomata/beto-sentiment-analysis":
+        if label == "NEG":
+            label = "NEGATIVE"
+            output_func = st.error
+        elif label == "POS":
+            label = "POSITIVE"
+            output_func = st.success
+        else:
+            label = "NEUTRAL"
+    elif self.transformer_model_name == "bhadresh-savani/distilbert-base-uncased-emotion":
+        if label == "sadness":
+            output_func = st.info
+        elif label == "joy":
+            output_func = st.success
+        elif label == "love":
+            output_func = st.success
+        elif label == "anger":
+            output_func = st.error
+        elif label == "fear":
+            output_func = st.info
+        elif label == "surprise":
+            output_func = st.error
+        label = label.upper()
+    elif self.transformer_model_name == "siebert/sentiment-roberta-large-english":
+        if label == "NEGATIVE":
+            output_func = st.error
+        elif label == "POSITIVE":
+            output_func = st.success
+    return label, score, output_func
+
+def ParsePatentOutput(self, result):
+    return result
+
+def emotion_model_change():
+    st.session_state.emotion_model = ModelImplementation(
+        st.session_state.emotion_model_name,
+        AutoModelForSequenceClassification,
+        st.session_state.emotion_model_name,
+        AutoTokenizer,
+        pipeline,
+        ParseEmotionOutput,
+        classifier_args={ "task" : "sentiment-analysis" },
+        placeholders=["@AmericanAir just landed - 3hours Late Flight - and now we need to wait TWENTY MORE MINUTES for a gate! I have patience but none for incompetence."]
+    )
+    
+if "emotion_model_name" not in st.session_state:
+    st.session_state.emotion_model_name = "cardiffnlp/twitter-roberta-base-sentiment"
+    emotion_model_change()
+
+if "patent_data" not in st.session_state:
+    f = open('./data/val.json')
+    valData = json.load(f)
+    f.close()
+
+    patent_data = {}
+    for num, label, abstract, claim in zip(valData["patent_numbers"],valData["labels"], valData["abstracts"], valData["claims"]):
+        patent_data[num] = {"patent_number":num,"label":label,"abstract":abstract,"claim":claim}
+
+    st.session_state.patent_data = patent_data
+    st.session_state.patent_num = list(patent_data.keys())[0]
+    st.session_state.weight = 0.5
+    st.session_state.patent_abstract_model = ModelImplementation(
+        'rk2546/uspto-patents-abstracts',
+        DistilBertForSequenceClassification,
+        'distilbert-base-uncased',
+        DistilBertTokenizerFast,
+        TextClassificationPipeline,
+        ParsePatentOutput,
+        classifier_args={"return_all_scores":True},
+    )
+    print("Patent abstracts model initialized")
+    st.session_state.patent_claim_model = ModelImplementation(
+        'rk2546/uspto-patents-claims',
+        DistilBertForSequenceClassification,
+        'distilbert-base-uncased',
+        DistilBertTokenizerFast,
+        TextClassificationPipeline,
+        ParsePatentOutput,
+        classifier_args={"return_all_scores":True},
+    )
+    print("Patent claims model initialized")
+
+# Title
+st.title("CSGY-6613 Project")
+# Subtitle
+st.markdown("_**Ryan Kim (rk2546)**_")
+
+sentimentTab, patentTab = st.tabs([
+    "Emotion Analysis [Milestone #2]",
+    "Patent Prediction [Milestone #3]"
+])
+
+with sentimentTab:
+    st.subheader("Sentiment Analysis")
+    if "emotion_model" not in st.session_state:
+        st.write("Loading model...")
+    else:
+        model_option = st.selectbox(
+            "What sentiment analysis model do you want to use? NOTE: Lag may occur when loading a new model!",
+            emotion_model_names,
+            on_change=emotion_model_change,
+            key="emotion_model_name"
+        )
+        form = st.form(key='sentiment-analysis-form')
+        text_input = form.text_area(
+            "Enter some text for sentiment analysis! If you just want to test it out without entering anything, just press the \"Submit\" button and the model will look at the placeholder.", 
+            placeholder=st.session_state.emotion_model.placeholders[0]
+        )
+        submit = form.form_submit_button('Submit')
+        if submit:
+            if text_input is None or len(text_input.strip()) == 0:
+                to_eval = st.session_state.emotion_model.placeholders[0]
+            else:
+                to_eval = text_input.strip()
+            label, score, output_func = st.session_state.emotion_model.predict(to_eval)  
+            output_func("**{}**: {}".format(label,score))
+    
+with patentTab:
+    st.subheader("USPTO Patent Evaluation")
+    st.markdown("Below are two inputs - one for an **ABSTRACT** and another for a list of **CLAIMS**. Enter both and select the \"Submit\" button to evaluate the patenteability of your idea.")
+
+    patent_select_list = list(st.session_state.patent_data.keys())
+    patent_index_option = st.selectbox(
+        "Want to pre-populate with an existing patent? Select the index number of  below.",
+        patent_select_list,
+        key="patent_num",
+    )
+
+    if "patent_abstract_model" not in st.session_state or "patent_claim_model" not in st.session_state:
+        st.write("Loading models...")
+    else:
+        with st.form(key='patent-form'):
+            col1, col2 = st.columns(2)
+            with col1:
+                abstract_input = st.text_area(
+                    "Enter the abstract of the patent below", 
+                    placeholder=st.session_state.patent_data[st.session_state.patent_num]["abstract"],
+                    height=200
+                )
+            with col2:
+                claim_input = st.text_area(
+                    "Enter the claims of the patent below", 
+                    placeholder=st.session_state.patent_data[st.session_state.patent_num]["claim"],
+                    height=200
+                )
+            weight_val = st.slider(
+                "How much do the abstract and claims weight when aggregating a total softmax score?",
+                min_value=-1.0,
+                max_value=1.0,
+                value=0.5,
+            )
+            submit = st.form_submit_button('Submit')
+
+            if submit:
+                
+                is_custom = False
+                if abstract_input is None or len(abstract_input.strip()) == 0:
+                    abstract_to_eval = st.session_state.patent_data[st.session_state.patent_num]["abstract"].strip()
+                else:
+                    abstract_to_eval = abstract_input.strip()
+                    is_custom = True
+
+                if claim_input is None or len(claim_input.strip()) == 0:
+                    claim_to_eval = st.session_state.patent_data[st.session_state.patent_num]["claim"].strip()
+                else:
+                    claim_to_eval = claim_input.strip()
+                    is_custom = True
+
+                abstract_response = st.session_state.patent_abstract_model.predict(abstract_to_eval)
+                claim_response = st.session_state.patent_claim_model.predict(claim_to_eval)
+
+                claim_weight = (1+weight_val)/2
+                abstract_weight = 1-claim_weight
+                aggregate_score = [
+                    {'label':'REJECTED','score':abstract_response[0][0]['score']*abstract_weight + claim_response[0][0]['score']*claim_weight},
+                    {'label':'ACCEPTED','score':abstract_response[0][1]['score']*abstract_weight + claim_response[0][1]['score']*claim_weight}
+                ]
+                aggregate_score_sorted = sorted(aggregate_score, key=lambda d: d['score'], reverse=True) 
+
+                answerCol1, answerCol2, answerCol3 = st.columns(3)
+                with answerCol1:
+                    st.slider(
+                        "Abstract Acceptance Likelihood", 
+                        min_value=0.0, 
+                        max_value=100.0,
+                        value=abstract_response[0][1]["score"]*100.0,
+                        disabled=True
+                    )
+                with answerCol2:
+                    output_func = st.info
+                    if aggregate_score_sorted[0]["label"] == "REJECTED":
+                        output_func = st.error
+                    else:
+                        output_func = st.success
+                    output_func("""
+                        **Final Rating: {}**
+                        {}%
+                    """.format(aggregate_score_sorted[0]["label"],aggregate_score_sorted[0]["score"]*100.0))
+                with answerCol3:
+                    st.slider(
+                        "Claim Acceptance Likelihood", 
+                        min_value=0.0, 
+                        max_value=100.0,
+                        value=claim_response[0][1]["score"]*100.0,
+                        disabled=True
+                    )
+                
+                #if not is_custom:
+                #    st.markdown('**Original Score:**')
+                #    st.markdown(st.session_state.patent_data[st.session_state.patent_num]["label"])
+                
+st.write("")

src/train.py

@@ -0,0 +1,228 @@
+from datasets import load_dataset
+import pandas as pd
+import numpy as np
+import os
+import json
+import torch
+import sys
+
+from torch.utils.data import Dataset, DataLoader
+from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification
+from transformers import Trainer, TrainingArguments, AdamW
+
+torch.backends.cuda.matmul.allow_tf32 = True
+model_name = "distilbert-base-uncased"
+upsto_abstracts_model_path = './models/uspto_abstracts'
+upsto_claims_model_path = './models/uspto_claims'
+
+class USPTODataset(Dataset):
+    def __init__(self, encodings, labels):
+        self.encodings = encodings
+        self.labels = labels
+    def __getitem__(self, idx):
+        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
+        item['labels'] = torch.tensor(self.labels[idx])
+        return item
+    def __len__(self):
+        return len(self.labels)
+
+def LoadDataset():
+    print("=== LOADING THE DATASET ===")
+    # Extracting the dataset, filtering only for Jan. 2016
+    dataset_dict = load_dataset('HUPD/hupd',
+        name='sample',
+        data_files="https://huggingface.co/datasets/HUPD/hupd/blob/main/hupd_metadata_2022-02-22.feather", 
+        icpr_label=None,
+        train_filing_start_date='2016-01-01',
+        train_filing_end_date='2016-01-21',
+        val_filing_start_date='2016-01-22',
+        val_filing_end_date='2016-01-31',
+    )
+
+    print("Separating between training and validation data")
+    df_train = pd.DataFrame(dataset_dict['train'] )
+    df_val = pd.DataFrame(dataset_dict['validation'] )
+
+    
+    print("=== PRE-PROCESSING THE DATASET ===")
+    #We are interested in the following columns:
+    # - Abstract
+    # - Claims
+    # - Decision <- our `y`
+    # Let's preprocess them both out of our training and validation data
+    # Also, consider that the "Decision" column has three types of values: "Accepted", "Rejected", and "Pending". To remove unecessary baggage, we will be only looking for "Accepted" and "Rejected".
+
+    necessary_columns = ["abstract","claims","decision"]
+    output_values = ['ACCEPTED','REJECTED']
+
+    print("Dropping unused columns")
+    trainFeaturesToDrop = [col for col in list(df_train.columns) if col not in necessary_columns]
+    trainDF = df_train.dropna()
+    trainDF.drop(columns=trainFeaturesToDrop, inplace=True)
+    trainDF = trainDF[trainDF['decision'].isin(output_values)]
+    valFeaturesToDrop = [col for col in list(df_val.columns) if col not in necessary_columns]
+    valDF = df_val.dropna()
+    valDF.drop(columns=valFeaturesToDrop, inplace=True)
+    valDF = valDF[valDF['decision'].isin(output_values)]
+
+    # We need to replace the values in the `decision` column to numerical representations. ]
+    # We will set "ACCEPTED" as `1` and "REJECTED" as `0`.
+    print("Replacing values in `decision` column")
+    yKey = {"ACCEPTED":1,"REJECTED":0}
+    trainDF2 = trainDF.replace({"decision": yKey})
+    valDF2 = valDF.replace({"decision": yKey})
+
+    # We re-label the `decision` column to `label`.
+    print("Renaming `decision` to `label`")
+    trainDF3 = trainDF2.rename(columns={'decision': 'label'})
+    valDF3 = valDF2.rename(columns={'decision': 'label'})
+
+    # We can grab the data for each column so that we have a list of values for training labels, 
+    #   training texts, validation labels, and validation texts.
+    print("Extracting label and text data from dataframes")
+    trainData = {
+        "labels":trainDF3["label"].tolist(),
+        "abstracts":trainDF3["abstract"].tolist(),
+        "claims":trainDF3["claims"].tolist(),
+    }
+    valData = {
+        "labels":valDF3["label"].tolist(),
+        "abstracts":valDF3["abstract"].tolist(),
+        "claims":valDF3["claims"].tolist(),
+    }
+    #print(f'TRAINING:\t# labels: {len(trainData["labels"])}\t# texts: {len(trainData["text"])}')
+    #print(f'VALID:\t# labels: {len(valData["labels"])}\t# texts: {len(valData["text"])}')
+
+    if not os.path.exists("./data"):
+        os.makedirs('./data')
+
+    with open("./data/train.json", "w") as outfile:
+        json.dump(trainData, outfile, indent=2)
+    with open("./data/val.json", "w") as outfile:
+        json.dump(valData, outfile, indent=2)
+
+    return trainData, valData
+
+def TrainModel(trainData, valData):
+    print("=== ENCODING DATA ===")
+    #print(len(trainData["labels"]), len(trainData["text"]), len(valData["labels"]), len(valData["text"]))
+    print("\t- initializing tokenizer")
+    tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)
+    print("\t- encoding training data")
+    train_abstracts_encodings = tokenizer(trainData["abstracts"], truncation=True, padding=True)
+    train_claims_encodings = tokenizer(trainData["claims"], truncation=True, padding=True)
+    #print("\t- encoding validation data")
+    #val_abstracts_encodings = tokenizer(valData["abstracts"], truncation=True, padding=True)
+    #val_claims_encodings = tokenizer(valData["claims"], truncation=True, padding=True)
+
+    print(trainData["abstracts"][:10])
+    print(trainData["labels"][:10])
+
+    print("=== CREATING DATASETS ===")
+    print("\t- initializing dataset for training data")
+    train_abstracts_dataset = USPTODataset(train_abstracts_encodings, trainData["labels"])
+    train_claims_dataset = USPTODataset(train_claims_encodings, trainData["labels"])
+    #print("\t- initializing dataset for validation data")
+    #val_abstracts_dataset = USPTODataset(val_abstracts_encodings, valData["labels"])
+    #val_claims_dataset = USPTODataset(val_claims_encodings, valData["labels"])
+
+    print("=== PREPARING MODEL ===")
+    print("\t- setting up device")
+    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
+    print("\t- initializing model")
+    model = DistilBertForSequenceClassification.from_pretrained(model_name)
+    model.to(device)
+    model.train()
+
+    print("== PREPARING TRAINING ===")
+    print("\t- initializing trainers")
+    train_abstracts_loader = DataLoader(train_abstracts_dataset, batch_size=4, shuffle=True)
+    train_claims_loader = DataLoader(train_claims_dataset, batch_size=4, shuffle=True)
+    #train_claims_loader = DataLoader(train_claims_dataset, batch_size=4, shuffle=True)
+    print("\t- initializing optim")
+    optim = AdamW(model.parameters(), lr=5e-5)
+
+    def Train(loader, save_path, num_train_epochs=2):
+        batch_num = len(loader)
+        for epoch in range(num_train_epochs):
+            print(f'\t- Training epoch {epoch+1}/{num_train_epochs}')
+            batch_count = 0
+            for batch in loader:
+                print(f'{batch_count}|{batch_num} - {round((batch_count/batch_num)*100)}%', end="")
+                #print('\t\t- optim zero grad')
+                optim.zero_grad()
+                #print('\t\t- input_ids')
+                input_ids = batch['input_ids'].to(device)
+                #print('\t\t- attention_mask')
+                attention_mask = batch['attention_mask'].to(device)
+                #print('\t\t- labels0')
+                labels = batch['labels'].to(device)
+                #print('\t\t- outputs')
+                outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
+            
+                #print('\t\t- loss')
+                loss = outputs[0]
+                #print('\t\t- backwards')
+                loss.backward()
+                #print('\t\t- step')
+                optim.step()
+
+                batch_count += 1
+                print("\r", end="")
+    
+        model.eval()
+        model.save_pretrained(save_path, from_pt=True) 
+        print(f'Saved model in {save_path}!')
+    
+    print("=== TRAINING ABSTRACTS ===")
+    Train(train_abstracts_loader,upsto_abstracts_model_path)
+    print("=== TRAINING CLAIMS ===")
+    Train(train_claims_loader,upsto_claims_model_path)
+
+def main():
+    trainDataPath = "./data/train.json"
+    valDataPath = "./data/val.json"
+    trainData = None
+    valData = None
+
+    if os.path.exists(trainDataPath) and os.path.exists(valDataPath):
+        print("Loading from existing data files")
+        ftrain = open(trainDataPath)
+        trainData = json.load(ftrain)
+        ftrain.close()
+        fval = open(valDataPath)
+        valData = json.load(fval)
+        fval.close()
+    else:
+        trainData, valData = LoadDataset()
+    
+    #print(len(trainData["labels"]), len(trainData["text"]), len(valData["labels"]), len(valData["text"]))
+    print("Data loaded successfully!")
+
+    TrainModel(trainData, valData)
+
+    """
+    train_args = TrainingArguments(
+        output_dir="./results",
+        num_train_epochs=2,
+        per_device_train_batch_size=16,
+        per_device_eval_batch_size=64,
+        warmup_steps=500,
+        learning_rate=5e-5,
+        weight_decay=0.01,
+        logging_dir="./logs",
+        logging_steps=10
+    )
+
+    model = DistilBertForSequenceClassification.from_pretrained(model_name)
+    trainer = Trainer(
+        model=model,
+        args=train_args,
+        train_dataset=train_dataset,
+        eval_dataset=val_dataset
+    )
+    trainer.train()
+    """
+
+if __name__ == "__main__":
+    main()

src/val.ipynb

@@ -0,0 +1 @@
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyPbIO5QK/V8keB7h6h+8Ju2"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":22,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ePuwhQ7QyzUW","executionInfo":{"status":"ok","timestamp":1682571700367,"user_tz":240,"elapsed":29378,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"9c939d4a-7622-4c48-ba58-b83162400692"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (2.11.0)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.1)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (6.0)\n","Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.65.0)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.5.3)\n","Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets) (3.2.0)\n","Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.18.0)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.22.4)\n","Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.14.1)\n","Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets) (0.70.14)\n","Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (0.3.6)\n","Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n","Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.4.0)\n","Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.27.1)\n","Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets) (3.8.4)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (23.1.0)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.3)\n","Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (2.0.12)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (6.0.4)\n","Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (4.0.2)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.3.1)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (1.9.2)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.12.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (3.4)\n","Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: streamlit in /usr/local/lib/python3.9/dist-packages (1.21.0)\n","Requirement already satisfied: packaging>=14.1 in /usr/local/lib/python3.9/dist-packages (from streamlit) (23.1)\n","Requirement already satisfied: toml in /usr/local/lib/python3.9/dist-packages (from streamlit) (0.10.2)\n","Requirement already satisfied: tzlocal>=1.1 in /usr/local/lib/python3.9/dist-packages (from streamlit) (4.3)\n","Requirement already satisfied: protobuf<4,>=3.12 in /usr/local/lib/python3.9/dist-packages (from streamlit) (3.20.3)\n","Requirement already satisfied: importlib-metadata>=1.4 in /usr/local/lib/python3.9/dist-packages (from streamlit) (6.6.0)\n","Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (13.3.4)\n","Requirement already satisfied: python-dateutil in /usr/local/lib/python3.9/dist-packages (from streamlit) (2.8.2)\n","Requirement already satisfied: pympler>=0.9 in /usr/local/lib/python3.9/dist-packages (from streamlit) (1.0.1)\n","Requirement already satisfied: pandas<2,>=0.25 in /usr/local/lib/python3.9/dist-packages (from streamlit) (1.5.3)\n","Requirement already satisfied: typing-extensions>=3.10.0.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (4.5.0)\n","Requirement already satisfied: validators>=0.2 in /usr/local/lib/python3.9/dist-packages (from streamlit) (0.20.0)\n","Requirement already satisfied: blinker>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (1.6.2)\n","Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (8.4.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from streamlit) (1.22.4)\n","Requirement already satisfied: tornado>=6.0.3 in /usr/local/lib/python3.9/dist-packages (from streamlit) (6.2)\n","Requirement already satisfied: watchdog in /usr/local/lib/python3.9/dist-packages (from streamlit) (3.0.0)\n","Requirement already satisfied: gitpython!=3.1.19 in /usr/local/lib/python3.9/dist-packages (from streamlit) (3.1.31)\n","Requirement already satisfied: pydeck>=0.1.dev5 in /usr/local/lib/python3.9/dist-packages (from streamlit) (0.8.1b0)\n","Requirement already satisfied: requests>=2.4 in /usr/local/lib/python3.9/dist-packages (from streamlit) (2.27.1)\n","Requirement already satisfied: cachetools>=4.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (5.3.0)\n","Requirement already satisfied: altair<5,>=3.2.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (4.2.2)\n","Requirement already satisfied: pyarrow>=4.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (9.0.0)\n","Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.9/dist-packages (from streamlit) (8.1.3)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from altair<5,>=3.2.0->streamlit) (3.1.2)\n","Requirement already satisfied: toolz in /usr/local/lib/python3.9/dist-packages (from altair<5,>=3.2.0->streamlit) (0.12.0)\n","Requirement already satisfied: entrypoints in /usr/local/lib/python3.9/dist-packages (from altair<5,>=3.2.0->streamlit) (0.4)\n","Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.9/dist-packages (from altair<5,>=3.2.0->streamlit) (4.3.3)\n","Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.9/dist-packages (from gitpython!=3.1.19->streamlit) (4.0.10)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/dist-packages (from importlib-metadata>=1.4->streamlit) (3.15.0)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas<2,>=0.25->streamlit) (2022.7.1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil->streamlit) (1.16.0)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests>=2.4->streamlit) (2.0.12)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.4->streamlit) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.4->streamlit) (1.26.15)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.4->streamlit) (3.4)\n","Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.9/dist-packages (from rich>=10.11.0->streamlit) (2.14.0)\n","Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /usr/local/lib/python3.9/dist-packages (from rich>=10.11.0->streamlit) (2.2.0)\n","Requirement already satisfied: pytz-deprecation-shim in /usr/local/lib/python3.9/dist-packages (from tzlocal>=1.1->streamlit) (0.1.0.post0)\n","Requirement already satisfied: decorator>=3.4.0 in /usr/local/lib/python3.9/dist-packages (from validators>=0.2->streamlit) (4.4.2)\n","Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.9/dist-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.19->streamlit) (5.0.0)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->altair<5,>=3.2.0->streamlit) (2.1.2)\n","Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.9/dist-packages (from jsonschema>=3.0->altair<5,>=3.2.0->streamlit) (23.1.0)\n","Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.9/dist-packages (from jsonschema>=3.0->altair<5,>=3.2.0->streamlit) (0.19.3)\n","Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.9/dist-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=10.11.0->streamlit) (0.1.2)\n","Requirement already satisfied: tzdata in /usr/local/lib/python3.9/dist-packages (from pytz-deprecation-shim->tzlocal>=1.1->streamlit) (2023.3)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.28.1)\n","Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.3)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.12.0)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.14.1)\n","Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n","Requirement already satisfied: fsspec in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (2023.4.0)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (4.65.0)\n"]}],"source":["!pip install datasets\n","!pip install streamlit\n","!pip install transformers\n","!pip install tqdm"]},{"cell_type":"code","source":["from datasets import load_dataset\n","import pandas as pd\n","import numpy as np\n","import os\n","import json\n","import torch\n","import sys\n","from tqdm import tqdm\n","\n","import streamlit as st\n","from transformers import TextClassificationPipeline, pipeline\n","from transformers import AutoTokenizer, AutoModelForSequenceClassification, DistilBertTokenizerFast, DistilBertForSequenceClassification"],"metadata":{"id":"xqhKMsNVzBtY","executionInfo":{"status":"ok","timestamp":1682571793784,"user_tz":240,"elapsed":3,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}}},"execution_count":27,"outputs":[]},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/gdrive')"],"metadata":{"id":"4E_xZUUwzGJm","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1682570070672,"user_tz":240,"elapsed":23530,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"a6fbb01a-caeb-4dc5-bef1-837c5dce202f"},"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/gdrive\n"]}]},{"cell_type":"code","source":["abstract_model = TextClassificationPipeline(\n","  model = DistilBertForSequenceClassification.from_pretrained('rk2546/uspto-patents-abstracts'),\n","  tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased'),\n","  padding = True, \n","  truncation = True,\n","  return_all_scores = True\n",")\n","\n","claim_model = TextClassificationPipeline(\n","  model = DistilBertForSequenceClassification.from_pretrained('rk2546/uspto-patents-claims'),\n","  tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased'),\n","  padding = True, \n","  truncation = True,\n","  return_all_scores = True\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Mj3hQGRU90bA","executionInfo":{"status":"ok","timestamp":1682573368942,"user_tz":240,"elapsed":7417,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"05cc8f93-1c72-4880-ae76-8d132d500c5f"},"execution_count":39,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.9/dist-packages/transformers/pipelines/text_classification.py:104: UserWarning: `return_all_scores` is now deprecated,  if want a similar funcionality use `top_k=None` instead of `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`.\n","  warnings.warn(\n"]}]},{"cell_type":"code","source":["path_to_valData = \"./gdrive/MyDrive/AI [Spring 2023]/cs-gy-6613-project-rk2546/val.json\"\n","f = open(path_to_valData)\n","valData = json.load(f)\n","f.close()"],"metadata":{"id":"0oimA5tO9c1G","executionInfo":{"status":"ok","timestamp":1682570188049,"user_tz":240,"elapsed":1507,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}}},"execution_count":7,"outputs":[]},{"cell_type":"code","source":["# We track the successes of abstracts, claims, and both combined\n","abstract_successes = 0\n","claim_successes = 0\n","aggregate_successes = 0\n","total_num = len(valData['labels'])\n","\n","# By default, we weigh the claims more highly than abstracts\n","claim_weight = 0.5\n","abstract_weight = 0.5\n","\n","# To randomize the data, we generate random indices \n","index_perms = np.random.permutation(total_num)\n","labels = []\n","abstracts = []\n","claims = []\n","# We generate up to 500 samples to validate against\n","new_total_num = min(1000,len(index_perms))\n","for i in range(new_total_num):\n","  labels.append(valData['labels'][index_perms[i]])\n","  abstracts.append(valData['abstracts'][index_perms[i]])\n","  claims.append(valData['claims'][index_perms[i]])\n","\n","# Now we validate\n","for i in tqdm(range(new_total_num)):\n","  label = labels[i]\n","  abstract = abstracts[i]\n","  claim = claims[i]\n","\n","  abstract_response = abstract_model(abstract)[0]\n","  claim_response = claim_model(claim)[0]\n","  aggregate_response = [\n","    {'label':'REJECTED','score':abstract_response[0]['score']*abstract_weight + claim_response[0]['score']*claim_weight},\n","    {'label':'ACCEPTED','score':abstract_response[1]['score']*abstract_weight + claim_response[1]['score']*claim_weight}\n","  ]\n","\n","  abstract_sorted = sorted(abstract_response, key=lambda d: d['score'], reverse=True) \n","  claim_sorted = sorted(claim_response, key=lambda d: d['score'], reverse=True)\n","  aggregate_sorted = sorted(aggregate_response, key=lambda d: d['score'], reverse=True) \n","\n","  if abstract_sorted[0]['label'] == 'LABEL_1' and label == 1:\n","    abstract_successes += 1\n","  elif abstract_sorted[0]['label'] == 'LABEL_0' and label == 0:\n","    abstract_successes += 1\n","  \n","  if claim_sorted[0]['label'] == 'LABEL_1' and label == 1:\n","    claim_successes += 1\n","  elif claim_sorted[0]['label'] == 'LABEL_0' and label == 0:\n","    claim_successes += 1\n","  \n","  if aggregate_sorted[0]['label'] == 'ACCEPTED' and label == 1:\n","    aggregate_successes += 1\n","  elif aggregate_sorted[0]['label'] == 'REJECTED' and label == 0:\n","    aggregate_successes += 1\n","\n","  # At 10% intervals, we print the current results\n","  if i > 0 and i % (new_total_num * 0.1) == 0:\n","    print(f\"\\nAbs: {abstract_successes}/{i} | Cl: {claim_successes}/{i} | Agg: {aggregate_successes}/{i}\")\n","\n","# Calculate final accuracy\n","abstract_accuracy = abstract_successes / new_total_num\n","claim_accuracy = claim_successes / new_total_num\n","aggregate_accuracy = aggregate_successes / new_total_num\n","\n","# Display accuracy\n","print(\"\\n\")\n","print(f\"Abstract Model Accuracy: {abstract_accuracy * 100}%\")\n","print(f\"Claim Model Accuracy: {claim_accuracy * 100}%\")\n","print(f\"Aggregated Model Accuracy: {aggregate_accuracy * 100}%\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FLE-9qlw9qW7","executionInfo":{"status":"ok","timestamp":1682577092672,"user_tz":240,"elapsed":1356393,"user":{"displayName":"Ryan Kim","userId":"18356277368138721144"}},"outputId":"fe0fb5ed-b075-4e4d-c616-6dbac5148a75"},"execution_count":48,"outputs":[{"output_type":"stream","name":"stderr","text":[" 10%|█         | 101/1000 [02:25<22:03,  1.47s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 70/100 | Cl: 73/100 | Agg: 73/100\n"]},{"output_type":"stream","name":"stderr","text":[" 20%|██        | 201/1000 [04:38<21:25,  1.61s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 148/200 | Cl: 155/200 | Agg: 155/200\n"]},{"output_type":"stream","name":"stderr","text":[" 30%|███       | 301/1000 [06:53<13:59,  1.20s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 220/300 | Cl: 224/300 | Agg: 234/300\n"]},{"output_type":"stream","name":"stderr","text":[" 40%|████      | 401/1000 [09:08<11:16,  1.13s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 295/400 | Cl: 293/400 | Agg: 308/400\n"]},{"output_type":"stream","name":"stderr","text":[" 50%|█████     | 501/1000 [11:24<10:34,  1.27s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 362/500 | Cl: 365/500 | Agg: 383/500\n"]},{"output_type":"stream","name":"stderr","text":[" 60%|██████    | 601/1000 [13:37<10:44,  1.61s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 443/600 | Cl: 440/600 | Agg: 462/600\n"]},{"output_type":"stream","name":"stderr","text":[" 70%|███████   | 701/1000 [15:54<06:52,  1.38s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 523/700 | Cl: 517/700 | Agg: 546/700\n"]},{"output_type":"stream","name":"stderr","text":[" 80%|████████  | 801/1000 [18:07<03:42,  1.12s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 601/800 | Cl: 591/800 | Agg: 626/800\n"]},{"output_type":"stream","name":"stderr","text":[" 90%|█████████ | 901/1000 [20:24<01:56,  1.18s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","Abs: 670/900 | Cl: 666/900 | Agg: 703/900\n"]},{"output_type":"stream","name":"stderr","text":["100%|██████████| 1000/1000 [22:36<00:00,  1.36s/it]"]},{"output_type":"stream","name":"stdout","text":["\n","\n","Abstract Model Accuracy: 72.89999999999999%\n","Claim Model Accuracy: 72.8%\n","Aggregated Model Accuracy: 76.2%\n"]},{"output_type":"stream","name":"stderr","text":["\n"]}]},{"cell_type":"code","source":[],"metadata":{"id":"Enwp7rw___5t"},"execution_count":null,"outputs":[]}]}