{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python", "version": "3.6.6", "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py" }, "colab": { "provenance": [], "toc_visible": true }, "widgets": { "application/vnd.jupyter.widget-state+json": { "d626e63ef6684ccc8e2f1ae4ef3b01d4": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_4ee97de6ff0c433fb84a86e2bd8d2d26", "IPY_MODEL_8ff8668bccc4444dbc13931150cfae66", "IPY_MODEL_69115a73d27a4ec9837f563c70c8f06e" ], "layout": "IPY_MODEL_ea5893b299564b7b8337e00c438b29f3" } }, "4ee97de6ff0c433fb84a86e2bd8d2d26": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_113eb6f67bed4d4b8c381e571795f20f", "placeholder": "​", "style": "IPY_MODEL_1553ccaa3a5a44e1a4978d4a3f3e996d", "value": "tokenizer_config.json: 100%" } }, "8ff8668bccc4444dbc13931150cfae66": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1f14797df3c7482cae15feb855ef0aa8", "max": 25, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_de1f48e2591443bdaa34106df15da6ea", "value": 25 } }, "69115a73d27a4ec9837f563c70c8f06e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3b4284e2028a433e80c5d34605369c65", "placeholder": "​", "style": "IPY_MODEL_91a8864cd74d46508d0cd9a0450d7127", "value": " 25.0/25.0 [00:00<00:00, 1.87kB/s]" } }, "ea5893b299564b7b8337e00c438b29f3": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "113eb6f67bed4d4b8c381e571795f20f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1553ccaa3a5a44e1a4978d4a3f3e996d": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "1f14797df3c7482cae15feb855ef0aa8": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "de1f48e2591443bdaa34106df15da6ea": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "3b4284e2028a433e80c5d34605369c65": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "91a8864cd74d46508d0cd9a0450d7127": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "78023e4833db43d998b358f38864b29c": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_dc1e959212144b4381012daeaf7bdaa1", "IPY_MODEL_9fad48a3cc9b4416b4db3f605fc4dd53", "IPY_MODEL_837dff859e7c4669bd397a825fc00357" ], "layout": "IPY_MODEL_a2bd96c0ba434973a16425f47a0edf95" } }, "dc1e959212144b4381012daeaf7bdaa1": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6707f55bf558400393bf58d70d298108", "placeholder": "​", "style": "IPY_MODEL_e55044f60d7e4861b58bb3949288b027", "value": "vocab.json: 100%" } }, "9fad48a3cc9b4416b4db3f605fc4dd53": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1b08fe8b5391416fbf0886455dc54366", "max": 898823, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_c32ba7036a9d41c49b0f09a1098b61ec", "value": 898823 } }, "837dff859e7c4669bd397a825fc00357": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_da65770f08ce43c2b60406cfa4ff1d70", "placeholder": "​", "style": "IPY_MODEL_befa339f82dc4001801fd4a979de5af2", "value": " 899k/899k [00:00<00:00, 4.27MB/s]" } }, "a2bd96c0ba434973a16425f47a0edf95": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6707f55bf558400393bf58d70d298108": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e55044f60d7e4861b58bb3949288b027": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "1b08fe8b5391416fbf0886455dc54366": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c32ba7036a9d41c49b0f09a1098b61ec": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "da65770f08ce43c2b60406cfa4ff1d70": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "befa339f82dc4001801fd4a979de5af2": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "12a0c6734eb84b9bab635de43e41bcf0": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_57ac1a83bc7345ae98ee1217d7b516bf", "IPY_MODEL_c19ce9d99d5a439a8e70e670be6a0f1c", "IPY_MODEL_27143a182ed3418cbb88e73e42e56ea7" ], "layout": "IPY_MODEL_973703acf61d4212ace942bf98fdbdf3" } }, "57ac1a83bc7345ae98ee1217d7b516bf": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e7714c0b206041e680d174ef48713edf", "placeholder": "​", "style": "IPY_MODEL_553e63b430a24c16b9809a6b7314562c", "value": "merges.txt: 100%" } }, "c19ce9d99d5a439a8e70e670be6a0f1c": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_15483492741f4909a79d6b944e6ba2ea", "max": 456318, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_09569a31a2d543c982ef148a80a83672", "value": 456318 } }, "27143a182ed3418cbb88e73e42e56ea7": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_dfdd5965e06943449c86be11c3b12514", "placeholder": "​", "style": "IPY_MODEL_6d7fbb019d3c438fa97159a0131386c9", "value": " 456k/456k [00:00<00:00, 723kB/s]" } }, "973703acf61d4212ace942bf98fdbdf3": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e7714c0b206041e680d174ef48713edf": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "553e63b430a24c16b9809a6b7314562c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "15483492741f4909a79d6b944e6ba2ea": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "09569a31a2d543c982ef148a80a83672": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "dfdd5965e06943449c86be11c3b12514": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6d7fbb019d3c438fa97159a0131386c9": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d7bc1e86ee5144c3bb32089b71df8b8d": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_8edcde0ff08246c6b6a1331bbe46b8e1", "IPY_MODEL_4d6d7bd81a10471cb4855e25a329a707", "IPY_MODEL_6c9f1bfbe10b4286bedeb58f03d7c724" ], "layout": "IPY_MODEL_1b82fde751f043c1ab9e1357a3003254" } }, "8edcde0ff08246c6b6a1331bbe46b8e1": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3a6aded39a8b4c22b5391473043aeb66", "placeholder": "​", "style": "IPY_MODEL_56f1537981404e83920e632d032ba169", "value": "tokenizer.json: 100%" } }, "4d6d7bd81a10471cb4855e25a329a707": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_beb4c4bc569f405e96526a1570d76f83", "max": 1355863, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_353370bb1bd84109bbddfe1083616698", "value": 1355863 } }, "6c9f1bfbe10b4286bedeb58f03d7c724": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ddf4f24edcd044299a7f6d95a9ebcbdc", "placeholder": "​", "style": "IPY_MODEL_2df915813ef04d91a77eab28922b9d28", "value": " 1.36M/1.36M [00:00<00:00, 3.21MB/s]" } }, "1b82fde751f043c1ab9e1357a3003254": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3a6aded39a8b4c22b5391473043aeb66": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "56f1537981404e83920e632d032ba169": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "beb4c4bc569f405e96526a1570d76f83": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "353370bb1bd84109bbddfe1083616698": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "ddf4f24edcd044299a7f6d95a9ebcbdc": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2df915813ef04d91a77eab28922b9d28": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "98426a6c84014366b7e384921bb01262": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e2811b2d327340988ccea1f94a62470f", "IPY_MODEL_a14e79b40cc8414f8949cb4d4ef39f03", "IPY_MODEL_112783b768c04333894b940e701f47dd" ], "layout": "IPY_MODEL_fdd39845acc84daaa3753003f6863fb9" } }, "e2811b2d327340988ccea1f94a62470f": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6e371fab9947480e80dc31857077921e", "placeholder": "​", "style": "IPY_MODEL_0301840910834291a4739863e86bea85", "value": "config.json: 100%" } }, "a14e79b40cc8414f8949cb4d4ef39f03": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_9e145826d0494e4c80a12c8e4dc282ff", "max": 481, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_7aa7f74dfb504150b088ff8c1f2cc41c", "value": 481 } }, "112783b768c04333894b940e701f47dd": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_9f70e844a4b94113be53da60ce746f5a", "placeholder": "​", "style": "IPY_MODEL_ce242bbe689c470bb963042573c58e51", "value": " 481/481 [00:00<00:00, 44.1kB/s]" } }, "fdd39845acc84daaa3753003f6863fb9": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6e371fab9947480e80dc31857077921e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0301840910834291a4739863e86bea85": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "9e145826d0494e4c80a12c8e4dc282ff": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7aa7f74dfb504150b088ff8c1f2cc41c": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "9f70e844a4b94113be53da60ce746f5a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ce242bbe689c470bb963042573c58e51": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "93c011a128264a62ba9e7a115ca466e6": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_c6fb19c7ba4440c6b334137f2e241260", "IPY_MODEL_916bb3e712b34c5fbc3854f9770735fd", "IPY_MODEL_6b90817e349c4f8e812221ac2f3572a0" ], "layout": "IPY_MODEL_3e72ccee212641c1855778bbba45b68a" } }, "c6fb19c7ba4440c6b334137f2e241260": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a041f8f7d6844f85bf4d6c5884b1f6a0", "placeholder": "​", "style": "IPY_MODEL_626b5393d8ea440eafc878c1aee28e43", "value": "model.safetensors: 100%" } }, "916bb3e712b34c5fbc3854f9770735fd": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_918bd31660e84b8a9d4882355a35b469", "max": 498818054, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_f178ce94deec416eb1c3d71ee031562c", "value": 498818054 } }, "6b90817e349c4f8e812221ac2f3572a0": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_de204df91d5d461eaec99e7c27479bdd", "placeholder": "​", "style": "IPY_MODEL_e680c81236a34841992c400ace608e99", "value": " 499M/499M [00:02<00:00, 242MB/s]" } }, "3e72ccee212641c1855778bbba45b68a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a041f8f7d6844f85bf4d6c5884b1f6a0": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "626b5393d8ea440eafc878c1aee28e43": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "918bd31660e84b8a9d4882355a35b469": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f178ce94deec416eb1c3d71ee031562c": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "de204df91d5d461eaec99e7c27479bdd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e680c81236a34841992c400ace608e99": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "8OhO4xlwqExT" }, "source": [ "# Fine Tuning Roberta for Sentiment Analysis\n", "\n", "\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "WTdfPjhFqExX" }, "source": [ "Note: Most of this was reused from [this Collab notebook](https://colab.research.google.com/github/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb\n", "), which demostrates fine-tuning roberta on sentiment analysis (from huggingface resources docs on roberta)\n", "\n", "### Introduction\n", "\n", "We're going to use a roberta model for **spam vs ham classification**. \"spam\" labels define spam (self-explanatory), and \"ham\" labels define a message sent by a homie (proba\n", "\n", "```\n", "# This is formatted as code\n", "```\n", "\n", "bly).\n", "\n", "### Table of Contents\n", "1. [Imports](#section01)\n", "2. [Data Pre-processing](#section02)\n", "3. [Data Prep](#section03)\n", "4. [Creating the Neural Net](#section04)\n", "5. [Fine Tuning the Model](#section05)\n", "6. [Testing the Model](#section06)\n", "7. [Saving the Model](#section07)\n", "\n", "\n", "#### Technical Details\n", "\n", "\n", "- **Data**: I used 3 datasets: two for fine-tuning, one for classification.\n", "\t- [Enron](https://huggingface.co/datasets/SetFit/enron_spam/blob/main/enron_spam_data.csv) (email data, but using only the messages) - this is part of the training data.\n", "\t- [Telegram](https://huggingface.co/datasets/thehamkercat/telegram-spam-ham/blob/main/dataset.csv) - this is part of the training data.\n", "\t- [Spam SMS Classification Using NLP](https://www.kaggle.com/code/shadymohammed205/nlp-meets-sms-distilbert-for-spam-detection) dataset from the Kaggle Competition - this is our test dataset.\n", "\n", "\n", "- **Language Model**: I used RoBERTa, as seen in [this research paper](https://arxiv.org/pdf/1907.11692). Its architecture was based on Google’s encoder-only BERT model released back in 2018 (which at the time was significantly undertrained, according to the authors of the paper), which is now a staple among NLP-ists.\n", "\t- [Blog-Post](https://ai.facebook.com/blog/roberta-an-optimized-method-for-pretraining-self-supervised-nlp-systems/)\n", "\t- [Documentation for python](https://huggingface.co/transformers/model_doc/roberta.html)\n", "\n", "- **Hardware Requirements**: I ran this on Google Collab using a A100 GPU (when in Rome...).\n", "\t- Strongly recommend this to be a GPU enabled setup, it's a 5x fine-tuning difference between a T4 and an A100, and I can imagine the CPU to be even slower.\n", "\t- Since this is Google Collab, we're using Python 3.10+, but I think most of these require 3.6+" ] }, { "cell_type": "code", "source": [ "!python --version" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "XNmD4Qkv4Kjw", "outputId": "23acfabd-8714-4417-a188-063c6eabacaf" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Python 3.10.12\n" ] } ] }, { "cell_type": "markdown", "metadata": { "id": "97CEi-bdqExb" }, "source": [ "\n", "### Imports" ] }, { "cell_type": "code", "metadata": { "id": "a-GlywkSFegL", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "761be786-0b10-47f5-8d7b-acb0c947ce37" }, "source": [ "# Need to install transformers in Google Collab\n", "!pip install transformers" ], "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.24.7)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n", "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n", "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.5)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (2024.6.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.2->transformers) (4.12.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n" ] } ] }, { "cell_type": "code", "metadata": { "trusted": true, "_uuid": "e7b5f5ab6f8f300c8900321a91b9340376c986f2", "id": "979OUro5Eac3" }, "source": [ "# Importing the necessaries\n", "import pandas as pd\n", "import numpy as np\n", "\n", "import json\n", "\n", "import seaborn as sns\n", "\n", "from sklearn.model_selection import train_test_split\n", "\n", "import torch\n", "import transformers\n", "from tqdm import tqdm\n", "from torch.utils.data import Dataset, DataLoader\n", "from transformers import RobertaModel, RobertaTokenizer\n", "\n", "import logging\n", "logging.basicConfig(level=logging.ERROR)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "sb1Q5N6LGK7z" }, "source": [ "# If you got that sweet CUD...\n", "# As cheese? You gouda.\n", "# On CPU you must sort?\n", "# You smell like Rockfort.\n", "# Just kidding, this is all part of me losing my sweet mind :)\n", "\n", "from torch import cuda\n", "device = 'cuda' if cuda.is_available() else 'cpu'" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "\n", "### Data Pre-processing\n", "\n", "We use Enron + Twitter as our training data, and our test data is the original kaggle competition" ], "metadata": { "id": "tc7Hy5ES6ADL" } }, { "cell_type": "code", "source": [ "# Enron - remember, since these collab instances are ephemeral, you need to upload\n", "enron_df = pd.read_csv('data/enron_spam_data.csv')\n", "enron_df = enron_df[['Spam/Ham', 'Message']].dropna().drop_duplicates('Message')\n", "enron_df.columns = ['label', 'message']\n", "\n", "# Twitter\n", "telegram_df = pd.read_csv('data/telegram_spam_ham_data.csv').dropna().drop_duplicates('text')\n", "telegram_df.columns = ['label', 'message']" ], "metadata": { "id": "2ZAwDOFQ_L17" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "train_df = pd.concat((enron_df, telegram_df), ignore_index=True)" ], "metadata": { "id": "eGRSXO96_gPK" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "test_df = pd.read_csv('data/Spam_SMS.csv')\n", "test_df.columns = test_df.columns.map(str.lower)\n", "test_df.columns = ['label', 'message']" ], "metadata": { "id": "-byXtnaw_ifL" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "print(train_df.message.apply(len).max())\n", "print(test_df.message.apply(len).max())" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "TUzp9a34_7Dn", "outputId": "c6970349-381d-43e8-af84-62c373ecb036" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "228353\n", "910\n" ] } ] }, { "cell_type": "code", "source": [ "train_df['label'] = train_df['label'].apply(lambda x: 1 if x=='spam' else 0)\n", "test_df['label'] = test_df['label'].apply(lambda x: 1 if x=='spam' else 0)\n" ], "metadata": { "id": "opBFgJH4BJQq" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "c3Q9NDdmqEyo" }, "source": [ "\n", "### Data Prep\n", "Most of this is kept the same from the original author's work.\n", "Some differences are renaming of the Dataset class, and changing the number of classes from 5 (sentiments) to 2 (spam/!spam). This is done in the Dataset class, the \"pre-processor\" to the NN.\n", "\n", "We use the [Roberta tokenizer](https://huggingface.co/transformers/model_doc/roberta.html#robertatokenizer), with encode_plus method to generate `ids` and `attention_mask` as outputs; `target` is the spam / not spam label.\n", "\n", "Originally, the author had used a training / testing split, but in our case we are using two explicitly differently distributed and not overlapping predefined datasets. We still use a validation function though.\n", "\n", "The Dataloader is used to for creating training and validation dataloader that loads data to the neural network in a defined manner. The original author of this collab had to split it since VRAM was limited to (I think) 12GB on the T4, but using the A100, it's increased to 40, which helps. Regardless, most of these are kept the same.Original author links further reading for Dataset and Dataloader [here](https://pytorch.org/docs/stable/data.html).\n", "\n", "#### **SpamOrHamData** Dataset Class\n", "\n", "- The tokenizer uses the `encode_plus` method to perform tokenization and generate the necessary outputs,method to perform tokenization and generate the necessary outputs, namely: `ids`, `attention_mask`\n", "- To read further into the tokenizer, [refer to this document](https://huggingface.co/transformers/model_doc/roberta.html#robertatokenizer)\n", "- `target` is the encoded category on the news headline.\n", "- The *SentimentData* class is used to create 2 datasets, for training (enron + telegram) and for testing (telegram)\n", "\n", "#### Dataloader\n" ] }, { "cell_type": "code", "metadata": { "id": "nvXxpfNCGER2", "outputId": "d5f0edb8-540c-4782-9c58-4cb6e42f01e2", "colab": { "base_uri": "https://localhost:8080/", "height": 344, "referenced_widgets": [ "d626e63ef6684ccc8e2f1ae4ef3b01d4", "4ee97de6ff0c433fb84a86e2bd8d2d26", "8ff8668bccc4444dbc13931150cfae66", "69115a73d27a4ec9837f563c70c8f06e", "ea5893b299564b7b8337e00c438b29f3", "113eb6f67bed4d4b8c381e571795f20f", "1553ccaa3a5a44e1a4978d4a3f3e996d", "1f14797df3c7482cae15feb855ef0aa8", "de1f48e2591443bdaa34106df15da6ea", "3b4284e2028a433e80c5d34605369c65", "91a8864cd74d46508d0cd9a0450d7127", "78023e4833db43d998b358f38864b29c", "dc1e959212144b4381012daeaf7bdaa1", "9fad48a3cc9b4416b4db3f605fc4dd53", "837dff859e7c4669bd397a825fc00357", "a2bd96c0ba434973a16425f47a0edf95", "6707f55bf558400393bf58d70d298108", "e55044f60d7e4861b58bb3949288b027", "1b08fe8b5391416fbf0886455dc54366", "c32ba7036a9d41c49b0f09a1098b61ec", "da65770f08ce43c2b60406cfa4ff1d70", "befa339f82dc4001801fd4a979de5af2", "12a0c6734eb84b9bab635de43e41bcf0", "57ac1a83bc7345ae98ee1217d7b516bf", "c19ce9d99d5a439a8e70e670be6a0f1c", "27143a182ed3418cbb88e73e42e56ea7", "973703acf61d4212ace942bf98fdbdf3", "e7714c0b206041e680d174ef48713edf", "553e63b430a24c16b9809a6b7314562c", "15483492741f4909a79d6b944e6ba2ea", "09569a31a2d543c982ef148a80a83672", "dfdd5965e06943449c86be11c3b12514", "6d7fbb019d3c438fa97159a0131386c9", "d7bc1e86ee5144c3bb32089b71df8b8d", "8edcde0ff08246c6b6a1331bbe46b8e1", "4d6d7bd81a10471cb4855e25a329a707", "6c9f1bfbe10b4286bedeb58f03d7c724", "1b82fde751f043c1ab9e1357a3003254", "3a6aded39a8b4c22b5391473043aeb66", "56f1537981404e83920e632d032ba169", "beb4c4bc569f405e96526a1570d76f83", "353370bb1bd84109bbddfe1083616698", "ddf4f24edcd044299a7f6d95a9ebcbdc", "2df915813ef04d91a77eab28922b9d28", "98426a6c84014366b7e384921bb01262", "e2811b2d327340988ccea1f94a62470f", "a14e79b40cc8414f8949cb4d4ef39f03", "112783b768c04333894b940e701f47dd", "fdd39845acc84daaa3753003f6863fb9", "6e371fab9947480e80dc31857077921e", "0301840910834291a4739863e86bea85", "9e145826d0494e4c80a12c8e4dc282ff", "7aa7f74dfb504150b088ff8c1f2cc41c", "9f70e844a4b94113be53da60ce746f5a", "ce242bbe689c470bb963042573c58e51" ] } }, "source": [ "# Defining some key variables that will be used later on in the training\n", "MAX_LEN = 512\n", "TRAIN_BATCH_SIZE = 8\n", "VALID_BATCH_SIZE = 4\n", "LEARNING_RATE = 1e-05\n", "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', truncation=True, do_lower_case=True)" ], "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", "You will be able to reuse this secret in all of your notebooks.\n", "Please note that authentication is recommended but still optional to access public models or datasets.\n", " warnings.warn(\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "tokenizer_config.json: 0%| | 0.00/25.0 [00:00\n", "### Creating the Neural Net\n", "(From Original Author, this part is largely unchanged)\n", "#### Neural Network\n", " - We will be creating a neural network with the `RobertaClass`.\n", " - This network will have the Roberta Language model followed by a `dropout` and finally a `Linear` layer to obtain the final outputs.\n", " - The data will be fed to the Roberta Language model as defined in the dataset.\n", " - Final layer outputs is what will be compared to the `Sentiment category` to determine the accuracy of models prediction.\n", " - We will initiate an instance of the network called `model`. This instance will be used for training and then to save the final trained model for future inference.\n", "\n", "#### Loss Function and Optimizer\n", " - `Loss Function` and `Optimizer` and defined in the next cell.\n", " - The `Loss Function` is used the calculate the difference in the output created by the model and the actual output.\n", " - `Optimizer` is used to update the weights of the neural network to improve its performance." ] }, { "cell_type": "code", "metadata": { "trusted": true, "_uuid": "cb8f194ee79d76356be0002b0e18f947e1412d66", "id": "HMqQTafXEaei" }, "source": [ "class RobertaClass(torch.nn.Module):\n", " def __init__(self):\n", " super(RobertaClass, self).__init__()\n", " self.l1 = RobertaModel.from_pretrained(\"roberta-base\")\n", " self.pre_classifier = torch.nn.Linear(768, 768)\n", " self.dropout = torch.nn.Dropout(0.3)\n", " self.classifier = torch.nn.Linear(768, 2)\n", "\n", " def forward(self, input_ids, attention_mask, token_type_ids):\n", " output_1 = self.l1(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n", " hidden_state = output_1[0]\n", " pooler = hidden_state[:, 0]\n", " pooler = self.pre_classifier(pooler)\n", " pooler = torch.nn.ReLU()(pooler)\n", " pooler = self.dropout(pooler)\n", " output = self.classifier(pooler)\n", " return output" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "sZ55mIPZIkp_", "outputId": "70d2ae48-cc48-43d2-f5bc-ad655dab88ef", "colab": { "base_uri": "https://localhost:8080/", "height": 951, "referenced_widgets": [ "93c011a128264a62ba9e7a115ca466e6", "c6fb19c7ba4440c6b334137f2e241260", "916bb3e712b34c5fbc3854f9770735fd", "6b90817e349c4f8e812221ac2f3572a0", "3e72ccee212641c1855778bbba45b68a", "a041f8f7d6844f85bf4d6c5884b1f6a0", "626b5393d8ea440eafc878c1aee28e43", "918bd31660e84b8a9d4882355a35b469", "f178ce94deec416eb1c3d71ee031562c", "de204df91d5d461eaec99e7c27479bdd", "e680c81236a34841992c400ace608e99" ] } }, "source": [ "model = RobertaClass()\n", "model.to(device)" ], "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "model.safetensors: 0%| | 0.00/499M [00:00\n", "### Fine Tuning the Model" ] }, { "cell_type": "code", "metadata": { "id": "XYZ7YuJ5InOS" }, "source": [ "# Creating the loss function and optimizer\n", "loss_function = torch.nn.CrossEntropyLoss()\n", "optimizer = torch.optim.Adam(params = model.parameters(), lr=LEARNING_RATE)" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "yPhA2V3iIpzN" }, "source": [ "def calcuate_accuracy(preds, targets):\n", " n_correct = (preds==targets).sum().item()\n", " return n_correct" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "mhqvtY2SIup7" }, "source": [ "# Defining the training function on the 80% of the dataset for tuning the distilbert model\n", "\n", "def train(epoch):\n", " tr_loss = 0\n", " n_correct = 0\n", " nb_tr_steps = 0\n", " nb_tr_examples = 0\n", " model.train()\n", " for _,data in tqdm(enumerate(training_loader, 0)):\n", " ids = data['ids'].to(device, dtype = torch.long)\n", " mask = data['mask'].to(device, dtype = torch.long)\n", " token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n", " targets = data['targets'].to(device, dtype = torch.long)\n", "\n", " outputs = model(ids, mask, token_type_ids)\n", " loss = loss_function(outputs, targets)\n", " tr_loss += loss.item()\n", " big_val, big_idx = torch.max(outputs.data, dim=1)\n", " n_correct += calcuate_accuracy(big_idx, targets)\n", "\n", " nb_tr_steps += 1\n", " nb_tr_examples+=targets.size(0)\n", "\n", " if _%5000==0:\n", " loss_step = tr_loss/nb_tr_steps\n", " accu_step = (n_correct*100)/nb_tr_examples\n", " print(f\"Training Loss per 5000 steps: {loss_step}\")\n", " print(f\"Training Accuracy per 5000 steps: {accu_step}\")\n", "\n", " optimizer.zero_grad()\n", " loss.backward()\n", " # # When using GPU\n", " optimizer.step()\n", "\n", " print(f'The Total Accuracy for Epoch {epoch}: {(n_correct*100)/nb_tr_examples}')\n", " epoch_loss = tr_loss/nb_tr_steps\n", " epoch_accu = (n_correct*100)/nb_tr_examples\n", " print(f\"Training Loss Epoch: {epoch_loss}\")\n", " print(f\"Training Accuracy Epoch: {epoch_accu}\")\n", "\n", " return" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "Afn7xaunJHnI", "outputId": "09ab98d7-fe7e-4844-a184-e70c359f9695", "colab": { "base_uri": "https://localhost:8080/", "height": 590 } }, "source": [ "EPOCHS = 3\n", "for epoch in range(EPOCHS):\n", " train(epoch)" ], "execution_count": null, "outputs": [ { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "\r0it [00:00, ?it/s]Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n", "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2870: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n", " warnings.warn(\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "Training Loss per 5000 steps: 0.6652650237083435\n", "Training Accuracy per 5000 steps: 37.5\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "5001it [15:35, 5.35it/s]" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "Training Loss per 5000 steps: 0.09503122924742213\n", "Training Accuracy per 5000 steps: 96.53819236152769\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "6265it [19:31, 5.35it/s]\n" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "The Total Accuracy for Epoch 0: 96.82916608464869\n", "Training Loss Epoch: 0.08699450095377213\n", "Training Accuracy Epoch: 96.82916608464869\n" ] }, { "metadata": { "tags": null }, "name": "stderr", "output_type": "stream", "text": [ "\r0it [00:00, ?it/s]" ] }, { "metadata": { "tags": null }, "name": "stdout", "output_type": "stream", "text": [ "Training Loss per 5000 steps: 0.0015362361446022987\n", "Training Accuracy per 5000 steps: 100.0\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "1152it [03:35, 5.36it/s]\n" ] }, { "output_type": "error", "ename": "KeyboardInterrupt", "evalue": "", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mEPOCHS\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEPOCHS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(epoch)\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_type_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0mtr_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 18\u001b[0m \u001b[0mbig_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbig_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mn_correct\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcalcuate_accuracy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbig_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ] }, { "cell_type": "markdown", "metadata": { "id": "vOcgTsovqE1A" }, "source": [ "\n", "### Testing the Model\n", "\n", "Our data split is about 90%-10%, with different distributions. I think by mixing the three, we can definitely increase performance from ~95% upwards." ] }, { "cell_type": "code", "metadata": { "id": "bFiNcy16JLwt" }, "source": [ "def valid(model, testing_loader):\n", " model.eval()\n", " n_correct = 0; n_wrong = 0; total = 0; tr_loss=0; nb_tr_steps=0; nb_tr_examples=0\n", " with torch.no_grad():\n", " for _, data in tqdm(enumerate(testing_loader, 0)):\n", " ids = data['ids'].to(device, dtype = torch.long)\n", " mask = data['mask'].to(device, dtype = torch.long)\n", " token_type_ids = data['token_type_ids'].to(device, dtype=torch.long)\n", " targets = data['targets'].to(device, dtype = torch.long)\n", " outputs = model(ids, mask, token_type_ids).squeeze()\n", " loss = loss_function(outputs, targets)\n", " tr_loss += loss.item()\n", " big_val, big_idx = torch.max(outputs.data, dim=1)\n", " n_correct += calcuate_accuracy(big_idx, targets)\n", "\n", " nb_tr_steps += 1\n", " nb_tr_examples+=targets.size(0)\n", "\n", " if _%5000==0:\n", " loss_step = tr_loss/nb_tr_steps\n", " accu_step = (n_correct*100)/nb_tr_examples\n", " print(f\"Validation Loss per 100 steps: {loss_step}\")\n", " print(f\"Validation Accuracy per 100 steps: {accu_step}\")\n", " epoch_loss = tr_loss/nb_tr_steps\n", " epoch_accu = (n_correct*100)/nb_tr_examples\n", " print(f\"Validation Loss Epoch: {epoch_loss}\")\n", " print(f\"Validation Accuracy Epoch: {epoch_accu}\")\n", "\n", " return epoch_accu\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "UcUylInzKdV-", "outputId": "5341d361-05c5-4996-bd30-01a421478782", "colab": { "base_uri": "https://localhost:8080/" } }, "source": [ "acc = valid(model, testing_loader)\n", "print(\"Accuracy on test data = %0.2f%%\" % acc)" ], "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "6it [00:00, 26.19it/s]" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Validation Loss per 100 steps: 0.0016986319096758962\n", "Validation Accuracy per 100 steps: 100.0\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "1394it [00:48, 28.87it/s]" ] }, { "output_type": "stream", "name": "stdout", "text": [ "Validation Loss Epoch: 0.12748833583852476\n", "Validation Accuracy Epoch: 95.03049874416936\n", "Accuracy on test data = 95.03%\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "\n" ] } ] }, { "cell_type": "markdown", "metadata": { "id": "tZgO6C1BqE1a" }, "source": [ "\n", "### Saving The Model\n", "\n" ] }, { "cell_type": "code", "metadata": { "id": "8eKt004BKjyT", "outputId": "956832e8-09eb-4ced-d8c2-b3a755a1712d", "colab": { "base_uri": "https://localhost:8080/" } }, "source": [ "output_model_file = 'pytorch_roberta_spam.bin'\n", "output_vocab_file = './'\n", "\n", "model_to_save = model\n", "torch.save(model_to_save, output_model_file)\n", "tokenizer.save_vocabulary(output_vocab_file)\n", "\n", "print('All files saved')\n", "print('This tutorial is completed')" ], "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "All files saved\n", "This tutorial is completed\n" ] } ] }, { "cell_type": "code", "metadata": { "id": "IetKrn_SY-OT" }, "source": [], "execution_count": null, "outputs": [] } ] }