Update app.py

app.py

@@ -1,65 +1,27 @@
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from datasets import load_dataset
-from transformers import TrainingArguments, Trainer
-
-# Load LLAMA3 8B model
-tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
-model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B")
-
-# Load datasets
-python_codes_dataset = load_dataset('flytech/python-codes-25k', split='train')
-streamlit_issues_dataset = load_dataset("andfanilo/streamlit-issues")
-streamlit_docs_dataset = load_dataset("sai-lohith/streamlit_docs")
-
-# Combine datasets
-combined_dataset = python_codes_dataset['text'] + streamlit_issues_dataset['text'] + streamlit_docs_dataset['text']
-
-# Define training arguments
-training_args = TrainingArguments(
-    per_device_train_batch_size=2,
-    num_train_epochs=3,
-    logging_dir='./logs',
-    output_dir='./output',
-    overwrite_output_dir=True,
-    report_to="none"  # Disable logging to avoid cluttering output
-)
-
-# Define training function
-def tokenize_function(examples):
-    return tokenizer(examples["text"])
-
-def group_texts(examples):
-    # Concatenate all texts.
-    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
-    total_length = len(concatenated_examples[list(examples.keys())[0]])
-    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can customize this part to your needs.
-    total_length = (total_length // tokenizer.max_len) * tokenizer.max_len
-    # Split by chunks of max_len.
-    result = {
-        k: [t[i : i + tokenizer.max_len] for i in range(0, total_length, tokenizer.max_len)]
-        for k, t in concatenated_examples.items()
-    }
-    return result
-
-# Tokenize dataset
-tokenized_datasets = combined_dataset.map(tokenize_function, batched=True, num_proc=4)
-
-# Group texts into chunks of max_len
-tokenized_datasets = tokenized_datasets.map(
-    group_texts,
-    batched=True,
-    num_proc=4,
-)
-
-# Train the model
-trainer = Trainer(
-    model=model,
-    args=training_args,
-    train_dataset=tokenized_datasets,
-    tokenizer=tokenizer,
-)
-
-trainer.train()
-
-# Save the trained model
-trainer.save_model("PyStreamlitGPT")
+import streamlit as st
+from datasets import load_dataset, concatenate_datasets
+
+def load_and_combine_datasets():
+    # Load the datasets
+    python_codes_dataset = load_dataset('flytech/python-codes-25k', split='train')
+    streamlit_issues_dataset = load_dataset("andfanilo/streamlit-issues")
+    streamlit_docs_dataset = load_dataset("sai-lohith/streamlit_docs")
+
+    # Combine the datasets
+    combined_dataset = concatenate_datasets([python_codes_dataset, streamlit_issues_dataset, streamlit_docs_dataset])
+    return combined_dataset
+
+def main():
+    st.title("Combined Dataset Viewer")
+    
+    # Load and combine datasets
+    combined_dataset = load_and_combine_datasets()
+    
+    # Display random sample from the combined dataset
+    random_sample = combined_dataset.shuffle(seed=42).select(range(10))
+    
+    st.header("Random Sample from Combined Dataset")
+    st.write(random_sample)
+
+if __name__ == "__main__":
+    main()