added tutorials

Tutorials/0_Embeddings.ipynb

@@ -0,0 +1,257 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "274e6135-2d97-4244-9183-65bcb1d24c80",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Use the trained astroBERT model to generate embedings of text\n",
+    "# to be used for downstream tasks"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2cc88ed3-6f52-49a2-99c0-344387758ab5",
+   "metadata": {},
+   "source": [
+    "# Tutorial 0: Loading astroBERT to produce text embeddings\n",
+    "This tutorial will show you how to load astroBERT and produce text embeddings that can be used on downstream tasks."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9e65c041-9d66-4fb1-96b9-4937000da02e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 1 - load models and tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "67d99e96-c532-49ef-8542-a48eef818956",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2022-10-17 12:10:19.355203: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, AutoModel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "00e1d48e-9898-44ef-b00e-43e3ab7fed7d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the model path can either be the name of the Huggingface repository\n",
+    "remote_model_path = 'adsabs/astroBERT'\n",
+    "# or the local path to the directory containing model weight and tokenizer vocab\n",
+    "local_model_path = '../'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "9bcc6009-6009-463f-a7da-f010c5fae27e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# make sure you load the tokenier with do_lower_case=False\n",
+    "astroBERT_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=remote_model_path,\n",
+    "                                                    use_auth_token=True,\n",
+    "                                                    add_special_tokens=True,\n",
+    "                                                    do_lower_case=False,\n",
+    "                                                   )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dbd144f0-6038-4917-94b0-aea9da72cac5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "PreTrainedTokenizerFast(name_or_path='adsabs/astroBERT', vocab_size=30000, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "astroBERT_tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "dd9a9257-cbe4-4908-a9f4-8e1431dc375a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n",
+      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    }
+   ],
+   "source": [
+    "# automodels: defaults to BertModel\n",
+    "# it's normal to get warnings as a BertModel will not load the weights used for PreTraining\n",
+    "astroBERT_automodel = AutoModel.from_pretrained(remote_model_path, \n",
+    "                                                use_auth_token=True,\n",
+    "                                               )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "572ddd38-a0dc-4583-a5a6-c4f3b2cb2553",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 2 - make some inference, the outputs are the embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "32fc0b97-4a2d-42ab-aa83-f5d8b39672b1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "torch.Size([3, 54])\n"
+     ]
+    }
+   ],
+   "source": [
+    "# list of strings for which we want embeddings\n",
+    "strings = ['The Chandra X-ray Observatory (CXO), previously known as the Advanced X-ray Astrophysics Facility (AXAF), is a Flagship-class space telescope launched aboard the Space Shuttle Columbia during STS-93 by NASA on July 23, 1999.',\n",
+    "           'Independent lines of evidence from Type Ia supernovae and the CMB imply that the universe today is dominated by a mysterious form of energy known as dark energy, which appears to homogeneously permeate all of space.',\n",
+    "           'This work has been developed in the framework of the ‘Darklight’ programme, supported by the European Research Council through an Advanced Research Grant to LG (Project # 291521).'\n",
+    "          ]\n",
+    "\n",
+    "# tokenizer the strings, with padding (needed to process multiple strings efficiently)\n",
+    "inputs = astroBERT_tokenizer(strings, \n",
+    "                             padding=True, \n",
+    "                             return_tensors='pt'\n",
+    "                            )\n",
+    "\n",
+    "# check the shape of the inputs\n",
+    "print(inputs['input_ids'].shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "8b7c9456-573a-48e7-9bc2-839fcc25631d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pass the inputs through astroBERT\n",
+    "import torch\n",
+    "# no need for gradients, since we are only doing inference\n",
+    "with torch.no_grad():\n",
+    "    output = astroBERT_automodel(**inputs, \n",
+    "                                  output_hidden_states=False\n",
+    "                                 )  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "116de57a-bb31-48d7-9556-64e01a16d56f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "torch.Size([3, 54, 768])\n"
+     ]
+    }
+   ],
+   "source": [
+    "# BertModel outputs two tensors: last_hidden_state (our embeddings) and pooler_output (to be discarded as it's not meaningful)\n",
+    "# see https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel.forward\n",
+    "# embeddings will have shape = (# of strings, size of tokenized strings(padded), 768 (BERT embedding size))\n",
+    "embeddings = output[0]\n",
+    "print(embeddings.shape)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "38e45291-6fd7-48cf-83df-e1cc5c8a699f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tensor([[ 0.5546,  0.9121,  0.6550,  ..., -0.1925,  0.7077, -0.2405],\n",
+      "        [ 0.6252,  0.3175,  1.0899,  ...,  0.0576,  0.0529,  0.0603],\n",
+      "        [ 0.1803, -0.4567,  1.2688,  ...,  0.6026, -0.5718, -0.2060],\n",
+      "        ...,\n",
+      "        [-0.4397, -0.5334,  1.1682,  ...,  0.9541,  0.4046, -0.4756],\n",
+      "        [-0.3911,  0.7793,  0.2432,  ...,  0.2268, -1.0489, -1.4864],\n",
+      "        [-0.4529, -0.7346,  0.0675,  ..., -0.3246, -0.2333, -0.6154]])\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(embeddings[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "26acf89f-b7fc-4872-ac81-0ee65030b465",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

Tutorials/1_Fill-Mask.ipynb

@@ -0,0 +1,427 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "33df4373-a37b-4fd0-bc67-c297812871e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Use the trained astroBERT model with the fill-mask pipeline"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "164ee9bd-27f9-40a4-8461-3ce12fc928b0",
+   "metadata": {},
+   "source": [
+    "# Tutorial 1: using astroBERT with the fill-mask pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "59429414-f07e-45e5-8825-6fc6a8d26653",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 1 - load models and tokenizer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "db8ee724-6a2a-4ea5-820e-5e2aa0a0f622",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2022-10-14 15:27:35.809315: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, BertForMaskedLM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "9a98fb63-0793-4684-a202-931cad17c7ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the model path can either be the name of the Huggingface repository\n",
+    "remote_model_path = 'adsabs/astroBERT'\n",
+    "# or the local path to the directory containing model weight and tokenizer vocab\n",
+    "local_model_path = '../'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "25fedd16-283b-4817-9b19-2a5ff1c5ba88",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# make sure you load the tokenier with do_lower_case=False\n",
+    "astroBERT_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=remote_model_path,\n",
+    "                                                    use_auth_token=True,\n",
+    "                                                    add_special_tokens=False,\n",
+    "                                                    do_lower_case=False,\n",
+    "                                                   )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "fb10db03-a5f0-44f7-8d41-0285f898a90d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
+      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    }
+   ],
+   "source": [
+    "astroBERT_automodel_for_mlm = BertForMaskedLM.from_pretrained(pretrained_model_name_or_path=remote_model_path, \n",
+    "                                                use_auth_token=True,\n",
+    "                                               )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e8b9b073-3876-4d0b-b8b2-e46fa25c76f0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# for pipeline to work you have to ensure that the model returns a dict\n",
+    "astroBERT_automodel_for_mlm.config.return_dict=True"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "94338f6f-3467-4696-bf7d-f41a12eb889d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import FillMaskPipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "7b980d9f-4d86-4b54-9324-d57dd9b4b64f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "astroBERT_pipeline = FillMaskPipeline(model=astroBERT_automodel_for_mlm,\n",
+    "                                      tokenizer=astroBERT_tokenizer,\n",
+    "                                      task='fill-mask',\n",
+    "                                     )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "5cb4d27b-ee3c-4ac7-ace2-4cc57ea9ce7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "clean_sentences = ['M67 is one of the most studied open clusters.',\n",
+    "'A solar twin is a star with atmospheric parameters and chemical composition very similar to our Sun.',\n",
+    "'The dynamical evolution of planets close to their star is affected by tidal effects',\n",
+    "'The Kepler satellite collected high-precision long-term and continuous light curves for more than 100,000 solar-type stars',\n",
+    "'The Local Group is composed of the Milky Way, the Andromeda Galaxy, and numerous smaller satellite galaxies.',\n",
+    "'Cepheid variables are used to determine the distances to galaxies in the local universe.',\n",
+    "'Jets are created and sustained by accretion of matter onto a compact massive object.',\n",
+    "'A single star of one solar mass will evolve into a white dwarf.',\n",
+    "'The Very Large Array observes the sky at radio wavelengths.',\n",
+    "'Elements heavier than iron are generated in supernovae explosions.',\n",
+    "'Spitzer was the first spacecraft to fly in an Earth-trailing orbit.',\n",
+    "'Galaxy mergers can occur when two (or more) galaxies collide',\n",
+    "'Dark matter is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.',\n",
+    "'The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the universe.',\n",
+    "'The Local Group of galaxies is pulled toward The Great Attractor.',\n",
+    "'The Moon is the only satellite of the Earth.',\n",
+    "'Galaxies are categorized according to their visual morphology as elliptical, spiral, or irregular.',\n",
+    "'Stars are made mostly of hydrogen.',\n",
+    "'Comet tails are created as comets approach the Sun.',\n",
+    "'Pluto is a dwarf planet in the Kuiper Belt.',\n",
+    "'The Large and Small Magellanic Clouds are irregular dwarf galaxies and are two satellite galaxies of the Milky Way.',\n",
+    "'The Milky Way has a supermassive black hole, Sagittarius A*, at its center.',\n",
+    "'Andromeda is the nearest large galaxy to the Milky Way and is roughly its equal in mass.',\n",
+    "'The interstellar medium is the gas and dust between stars.']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "9f3a6fdc-182f-4edb-8ef4-7e4253c2d4db",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "masked_sentences =  ['M67 is one of the most studied [MASK] clusters.',\n",
+    "'A solar twin is a star with [MASK] parameters and chemical composition very similar to our Sun.',\n",
+    "'The dynamical evolution of planets close to their star is affected by [MASK] effects',\n",
+    "'The Kepler satellite collected high-precision long-term and continuous light [MASK] for more than 100,000 solar-type stars',\n",
+    "'The Local Group is composed of the Milky Way, the [MASK] Galaxy, and numerous smaller satellite galaxies.',\n",
+    "'Cepheid variables are used to determine the [MASK] to galaxies in the local universe.',\n",
+    "'Jets are created and sustained by [MASK] of matter onto a compact massive object.',\n",
+    "'A single star of one solar mass will evolve into a [MASK] dwarf.',\n",
+    "'The Very Large Array observes the sky at [MASK] wavelengths.',\n",
+    "'Elements heavier than [MASK] are generated in supernovae explosions.',\n",
+    "'Spitzer was the first [MASK] to fly in an Earth-trailing orbit.',\n",
+    "'Galaxy [MASK] can occur when two (or more) galaxies collide',\n",
+    "'Dark [MASK] is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.',\n",
+    "'The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the [MASK].',\n",
+    "'The Local Group of galaxies is pulled toward The Great [MASK].',\n",
+    "'The Moon is the only [MASK] of the Earth.',\n",
+    "'Galaxies are categorized according to their visual morphology as [MASK], spiral, or irregular.',\n",
+    "'Stars are made mostly of [MASK].',\n",
+    "'Comet tails are created as comets approach the [MASK].',\n",
+    "'Pluto is a dwarf [MASK] in the Kuiper Belt.',\n",
+    "'The Large and Small Magellanic Clouds are irregular [MASK] galaxies and are two satellite galaxies of the Milky Way.',\n",
+    "'The Milky Way has a [MASK] black hole, Sagittarius A*, at its center.',\n",
+    "'Andromeda is the nearest large [MASK] to the Milky Way and is roughly its equal in mass.',\n",
+    "'The [MASK] medium is the gas and dust between stars.']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "d4c729ad-89f4-4e70-b433-a65b6035c10b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "masked_words = [x for s1,s2 in zip(clean_sentences, masked_sentences) \n",
+    "              for x,y in zip(s1.split(), s2.split()) if y=='[MASK]']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "2958cfea-f6cf-4d45-9e15-84a5fae7f3cb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['open',\n",
+       " 'atmospheric',\n",
+       " 'tidal',\n",
+       " 'curves',\n",
+       " 'Andromeda',\n",
+       " 'distances',\n",
+       " 'accretion',\n",
+       " 'white',\n",
+       " 'radio',\n",
+       " 'iron',\n",
+       " 'spacecraft',\n",
+       " 'mergers',\n",
+       " 'matter',\n",
+       " 'satellite',\n",
+       " 'planet',\n",
+       " 'dwarf',\n",
+       " 'supermassive',\n",
+       " 'galaxy',\n",
+       " 'interstellar']"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "masked_words"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "2a07a641-61a7-42dd-b70e-62eb97ad4e4b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results = astroBERT_pipeline(inputs=masked_sentences, \n",
+    "                             top_k=3\n",
+    "                            )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "ec2880d9-a8ad-4919-ab5b-732f3bcc21ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "M67 is one of the most studied [MASK] clusters.\n",
+      "original: open\n",
+      "\t  open 0.87\n",
+      "\t  globular 0.07\n",
+      "\t  star 0.03\n",
+      "\n",
+      "A solar twin is a star with [MASK] parameters and chemical composition very similar to our Sun.\n",
+      "original: atmospheric\n",
+      "\t  fundamental 0.56\n",
+      "\t  physical 0.25\n",
+      "\t  stellar 0.05\n",
+      "\n",
+      "The dynamical evolution of planets close to their star is affected by [MASK] effects\n",
+      "original: tidal\n",
+      "\t  tidal 0.07\n",
+      "\t  electromagnetic 0.05\n",
+      "\t  electrostatic 0.04\n",
+      "\n",
+      "The Kepler satellite collected high-precision long-term and continuous light [MASK] for more than 100,000 solar-type stars\n",
+      "original: curves\n",
+      "\t  curves 0.43\n",
+      "\t  ##s 0.04\n",
+      "\t  conditions 0.04\n",
+      "\n",
+      "The Local Group is composed of the Milky Way, the [MASK] Galaxy, and numerous smaller satellite galaxies.\n",
+      "original: Andromeda\n",
+      "\t  Andromeda 0.99\n",
+      "\t  M31 0.00\n",
+      "\t  Sagittarius 0.00\n",
+      "\n",
+      "Cepheid variables are used to determine the [MASK] to galaxies in the local universe.\n",
+      "original: distances\n",
+      "\t  distances 0.79\n",
+      "\t  distance 0.21\n",
+      "\t  redshifts 0.00\n",
+      "\n",
+      "Jets are created and sustained by [MASK] of matter onto a compact massive object.\n",
+      "original: accretion\n",
+      "\t  accretion 0.79\n",
+      "\t  infall 0.13\n",
+      "\t  fall 0.02\n",
+      "\n",
+      "A single star of one solar mass will evolve into a [MASK] dwarf.\n",
+      "original: white\n",
+      "\t  white 0.77\n",
+      "\t  brown 0.19\n",
+      "\t  red 0.02\n",
+      "\n",
+      "The Very Large Array observes the sky at [MASK] wavelengths.\n",
+      "original: radio\n",
+      "\t  radio 0.29\n",
+      "\t  centimeter 0.10\n",
+      "\t  all 0.09\n",
+      "\n",
+      "Elements heavier than [MASK] are generated in supernovae explosions.\n",
+      "original: iron\n",
+      "\t  iron 0.34\n",
+      "\t  helium 0.16\n",
+      "\t  oxygen 0.07\n",
+      "\n",
+      "Spitzer was the first [MASK] to fly in an Earth-trailing orbit.\n",
+      "original: spacecraft\n",
+      "\t  satellite 0.42\n",
+      "\t  spacecraft 0.20\n",
+      "\t  observatory 0.16\n",
+      "\n",
+      "Galaxy [MASK] can occur when two (or more) galaxies collide\n",
+      "original: mergers\n",
+      "\t  . 0.26\n",
+      "\t  A 0.05\n",
+      "\t  1 0.04\n",
+      "\n",
+      "Dark [MASK] is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.\n",
+      "original: matter\n",
+      "\t  energy 0.64\n",
+      "\t  Energy 0.24\n",
+      "\t  matter 0.10\n",
+      "\n",
+      "The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the [MASK].\n",
+      "original: satellite\n",
+      "\t  universe 0.45\n",
+      "\t  Universe 0.26\n",
+      "\t  expansion 0.09\n",
+      "\n",
+      "The Local Group of galaxies is pulled toward The Great [MASK].\n",
+      "original: planet\n",
+      "\t  Wall 0.96\n",
+      "\t  East 0.01\n",
+      "\t  Planet 0.00\n",
+      "\n",
+      "The Moon is the only [MASK] of the Earth.\n",
+      "original: dwarf\n",
+      "\t  satellite 0.38\n",
+      "\t  moon 0.31\n",
+      "\t  constituent 0.07\n",
+      "\n",
+      "Galaxies are categorized according to their visual morphology as [MASK], spiral, or irregular.\n",
+      "original: supermassive\n",
+      "\t  elliptical 0.92\n",
+      "\t  spheroidal 0.02\n",
+      "\t  irregular 0.01\n",
+      "\n",
+      "Stars are made mostly of [MASK].\n",
+      "original: galaxy\n",
+      "\t  hydrogen 0.20\n",
+      "\t  helium 0.14\n",
+      "\t  carbon 0.12\n",
+      "\n",
+      "Comet tails are created as comets approach the [MASK].\n",
+      "original: interstellar\n",
+      "\t  Sun 0.45\n",
+      "\t  sun 0.23\n",
+      "\t  Earth 0.19\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "for w, s, rs in zip(masked_words, masked_sentences,results):\n",
+    "    print(s)\n",
+    "    print('original: {}'.format(w))\n",
+    "    for r in rs:\n",
+    "        print('\\t  {} {:0.2f}'.format(r['token_str'], r['score']))\n",
+    "    print()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e03b19d6-66cd-4eba-8ddb-6924011037e7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}