Using data_loader to load vector files instead of json.

.gitattributes

@@ -33,4 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 data/semi_embedding_v6.zip filter=lfs diff=lfs merge=lfs -text
 data/half_embedding_v6.zip filter=lfs diff=lfs merge=lfs -text
-data/wiki-news-300d-1M.vec filter=lfs diff=lfs merge=lfs -text
+data/wiki-news-300d-1M.vec filter=lfs diff=lfs merge=lfs -text
+data/fasttext-sbwc.100k.vec filter=lfs diff=lfs merge=lfs -text

app.py

@@ -13,11 +13,14 @@ from interfaces.interface_BiasWordExplorer import interface as biasWordExplorer_
 # --- Tool config ---
 AVAILABLE_LOGS      = True                          # [True     | False]
 LANGUAGE            = "spanish"                     # [spanish  | english]
-EMBEDDING_SUBSET    = "fasttext"                    # [fasttext | mini]
+EMBEDDINGS_PATH     = "data/fasttext-sbwc.100k.vec"
 
 # --- Init classes ---
 embedding = Embedding(
-    subset_name=EMBEDDING_SUBSET
+    path=EMBEDDINGS_PATH,
+    binary=EMBEDDINGS_PATH.endswith('.bin'),
+    limit=None,
+    randomizedPCA=False
 )
 labels = pd.read_json(f"language/{LANGUAGE}.json")["app"]
 

data/data_loader.py

@@ -13,16 +13,12 @@ def load_embeddings(path, binary = False, randomPCA = False, limit = None):
     else:
         pca = PCA(n_components=2)
     
+    print("--------> PATH:", path)
     model = KeyedVectors.load_word2vec_format(path, binary=binary, limit=limit)
 
     # Cased Vocab
-    cased_words = model.vocab.keys()
-
-    #Normalized vectors
-    model.init_sims(replace=True)
-    cased_emb = [model[word] for word in cased_words]
-
-    # PCA reduction
+    cased_words = model.index_to_key
+    cased_emb = model.get_normed_vectors()
     cased_pca = pca.fit_transform(cased_emb)
 
     df_cased = pd.DataFrame(
@@ -36,6 +32,6 @@ def load_embeddings(path, binary = False, randomPCA = False, limit = None):
 
     df_cased['word'] = df_cased.word.apply(lambda w: w.lower())
     df_uncased = df_cased.drop_duplicates(subset='word')
-    df_uncased.to_json(path[:-3] + 'json')
+    return df_uncased
 
-load_embeddings('./wiki-news-300d-1M.vec', limit=10000)
+#load_embeddings('data/fasttext-sbwc.100k.vec', limit=1000)

data/fasttext-sbwc.100k.vec

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:062e67c948fc14de8c9327868293c76a6d6f536f62d6df3d8afbf6d62c4b9cc1
+size 262330934

modules/model_embbeding.py

@@ -1,3 +1,4 @@
+import os
 import operator
 import numpy as np
 import pandas as pd
@@ -6,14 +7,14 @@ from gensim import matutils
 from modules.module_ann import Ann
 from memory_profiler import profile
 from sklearn.neighbors import NearestNeighbors
+from data.data_loader import load_embeddings
 
 
 class Embedding:
     @profile
-    def __init__(self, subset_name):
+    def __init__(self, path, binary, limit = None, randomizedPCA = False):
         # Dataset info
-        self.ds_subset = subset_name
-        self.ds_path = f"data/{subset_name}_embedding_v6.zip"
+        self.path = path
         
         # Pandas dataset
         self.ds = None
@@ -25,16 +26,16 @@ class Embedding:
         self.ann = None
 
         # Load embedding and pca dataset
-        self.__load()
+        self.__load(binary, limit, randomizedPCA)
 
     def __contains__(self, word):
         return word in self.ds['word'].to_list()
 
-    def __load(self):
-        print(f"Preparing {self.ds_subset} embedding...")
+    def __load(self, binary, limit, randomizedPCA):
+        print(f"Preparing {os.path.basename(self.path)} embeddings...")
 
-        # --- Download dataset ---
-        self.ds = pd.read_json(self.ds_path)
+        # --- Prepare dataset ---
+        self.ds = load_embeddings(self.path, binary, randomizedPCA, limit)
 
         # --- Get embedding from string
         self.embedding = self.ds['embedding'].to_list()