|
|
|
|
|
|
|
|
|
|
|
import argparse |
|
import glob |
|
|
|
import numpy as np |
|
|
|
|
|
DIM = 1024 |
|
|
|
|
|
def compute_dist(source_embs, target_embs, k=5, return_sim_mat=False): |
|
target_ids = [tid for tid in target_embs] |
|
source_mat = np.stack(source_embs.values(), axis=0) |
|
normalized_source_mat = source_mat / np.linalg.norm( |
|
source_mat, axis=1, keepdims=True |
|
) |
|
target_mat = np.stack(target_embs.values(), axis=0) |
|
normalized_target_mat = target_mat / np.linalg.norm( |
|
target_mat, axis=1, keepdims=True |
|
) |
|
sim_mat = normalized_source_mat.dot(normalized_target_mat.T) |
|
if return_sim_mat: |
|
return sim_mat |
|
neighbors_map = {} |
|
for i, sentence_id in enumerate(source_embs): |
|
idx = np.argsort(sim_mat[i, :])[::-1][:k] |
|
neighbors_map[sentence_id] = [target_ids[tid] for tid in idx] |
|
return neighbors_map |
|
|
|
|
|
def load_embeddings(directory, LANGS): |
|
sentence_embeddings = {} |
|
sentence_texts = {} |
|
for lang in LANGS: |
|
sentence_embeddings[lang] = {} |
|
sentence_texts[lang] = {} |
|
lang_dir = f"{directory}/{lang}" |
|
embedding_files = glob.glob(f"{lang_dir}/all_avg_pool.{lang}.*") |
|
for embed_file in embedding_files: |
|
shard_id = embed_file.split(".")[-1] |
|
embeddings = np.fromfile(embed_file, dtype=np.float32) |
|
num_rows = embeddings.shape[0] // DIM |
|
embeddings = embeddings.reshape((num_rows, DIM)) |
|
|
|
with open(f"{lang_dir}/sentences.{lang}.{shard_id}") as sentence_file: |
|
for idx, line in enumerate(sentence_file): |
|
sentence_id, sentence = line.strip().split("\t") |
|
sentence_texts[lang][sentence_id] = sentence |
|
sentence_embeddings[lang][sentence_id] = embeddings[idx, :] |
|
|
|
return sentence_embeddings, sentence_texts |
|
|
|
|
|
def compute_accuracy(directory, LANGS): |
|
sentence_embeddings, sentence_texts = load_embeddings(directory, LANGS) |
|
|
|
top_1_accuracy = {} |
|
|
|
top1_str = " ".join(LANGS) + "\n" |
|
for source_lang in LANGS: |
|
top_1_accuracy[source_lang] = {} |
|
top1_str += f"{source_lang} " |
|
for target_lang in LANGS: |
|
top1 = 0 |
|
top5 = 0 |
|
neighbors_map = compute_dist( |
|
sentence_embeddings[source_lang], sentence_embeddings[target_lang] |
|
) |
|
for sentence_id, neighbors in neighbors_map.items(): |
|
if sentence_id == neighbors[0]: |
|
top1 += 1 |
|
if sentence_id in neighbors[:5]: |
|
top5 += 1 |
|
n = len(sentence_embeddings[target_lang]) |
|
top1_str += f"{top1/n} " |
|
top1_str += "\n" |
|
|
|
print(top1_str) |
|
print(top1_str, file=open(f"{directory}/accuracy", "w")) |
|
|
|
|
|
if __name__ == "__main__": |
|
parser = argparse.ArgumentParser(description="Analyze encoder outputs") |
|
parser.add_argument("directory", help="Source language corpus") |
|
parser.add_argument("--langs", help="List of langs") |
|
args = parser.parse_args() |
|
langs = args.langs.split(",") |
|
compute_accuracy(args.directory, langs) |
|
|
