Update app.py

app.py

@@ -1,16 +1,15 @@
-import gradio as gr
-import json
+from sklearn.feature_extraction.text import CountVectorizer
 import numpy as np
-import pandas as pd
-from sklearn.feature_extraction.text import TfidfVectorizer
-
 from datasets import load_dataset
 
 text_dataset = load_dataset("HuggingFaceFW/fineweb", name="sample-10BT", split="train", streaming=True, columns=['text'])
 
-def greet(name):
-    print(text_dataset)
-    return "Hello " + name + "!!"
-
-app = gr.Interface(fn=greet, inputs="text", outputs="text")
-app.launch()
+bigram_vectorizer = CountVectorizer(ngram_range=(2, 2), analyzer="word")
+co_occurrences = bigram_vectorizer.fit_transform(doc['text'] for doc in text_dataset)
+print('Printing sparse matrix:')
+print(co_occurrences)
+print('Printing dense matrix')
+print(co_occurrences.todense())
+sum_occ = np.sum(co_occurrences.todense(), axis=0)
+print('Sum of word-word occurrences:')
+print(sum_occ)