fastai_hf_self_distillation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fastai_hf_self_distillation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNaoA588dQqzO8NrKxikv5y",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f7497a88ef0340bb9d9f72755e4a92fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6fd1c8bfb6cc493cad4089159c730903",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4123b9e0b86e4fbea7fbd321c69f7ac2",
              "IPY_MODEL_234736284208438383ec1246f0fff4c0"
            ]
          }
        },
        "6fd1c8bfb6cc493cad4089159c730903": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4123b9e0b86e4fbea7fbd321c69f7ac2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5b0a5cdab2924f9c9cb334cce79e79a1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 480,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 480,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_02a17cadcf234def9c2dde3ac34b571d"
          }
        },
        "234736284208438383ec1246f0fff4c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_56685af582e942dcae0def756315f78b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 480/480 [00:09&lt;00:00, 49.0B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_47ed65f464c14e62aad79dfa0a98d674"
          }
        },
        "5b0a5cdab2924f9c9cb334cce79e79a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "02a17cadcf234def9c2dde3ac34b571d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56685af582e942dcae0def756315f78b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "47ed65f464c14e62aad79dfa0a98d674": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6b9012cbc764a73b8c736d92a8710a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8d8dd30f9a36499997016b7946c056af",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_870be6cf53514cf1995501277efaad2f",
              "IPY_MODEL_d7e3c4b336ac42c69a3b37ca98462388"
            ]
          }
        },
        "8d8dd30f9a36499997016b7946c056af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "870be6cf53514cf1995501277efaad2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1611eb56880747bb959701ab2dd4cbb7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 898823,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 898823,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9664da2db9746648d7b2837f70dc3fe"
          }
        },
        "d7e3c4b336ac42c69a3b37ca98462388": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f288262e758d44979033d3ba0dc0ff6a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 899k/899k [00:00&lt;00:00, 7.36MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4b9bc5d93eb44e63b65768066096a826"
          }
        },
        "1611eb56880747bb959701ab2dd4cbb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9664da2db9746648d7b2837f70dc3fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f288262e758d44979033d3ba0dc0ff6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4b9bc5d93eb44e63b65768066096a826": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8ad5cb6603f64a6cbac84f91fd0ff7ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_10055610fe99426b98fe47e7a8dd266a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_280a84059af643909a45f3de1985d3c2",
              "IPY_MODEL_52ec7483ed3e47b380807644ba94655d"
            ]
          }
        },
        "10055610fe99426b98fe47e7a8dd266a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "280a84059af643909a45f3de1985d3c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_35c7be8158db4ff49c6425161e9ae778",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8270604815144a1cbd731b929717a4ac"
          }
        },
        "52ec7483ed3e47b380807644ba94655d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ff537965e1e24930ad5bec57ff8cad9a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:09&lt;00:00, 47.7kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_67964b45451f4e4a9865b9e7253e6c1a"
          }
        },
        "35c7be8158db4ff49c6425161e9ae778": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8270604815144a1cbd731b929717a4ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ff537965e1e24930ad5bec57ff8cad9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "67964b45451f4e4a9865b9e7253e6c1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a29764f8440e41cd89de0f41f901eb3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c6e1bfb37bbe4686ba1ed68bf7212f7b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4de30ea2b8764ec188766e1221d875e0",
              "IPY_MODEL_3fda686524ee4397856c4a21b30b0ac7"
            ]
          }
        },
        "c6e1bfb37bbe4686ba1ed68bf7212f7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4de30ea2b8764ec188766e1221d875e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_733df8d7046b43afb8a037b2041c9c6c",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1355863,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1355863,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b003df6a476c48268691858b7b8d1701"
          }
        },
        "3fda686524ee4397856c4a21b30b0ac7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ccf6e20e9ced4793b907da5bb480fa58",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 2.87MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e005687623cd4457b1804fb0ddee0652"
          }
        },
        "733df8d7046b43afb8a037b2041c9c6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b003df6a476c48268691858b7b8d1701": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ccf6e20e9ced4793b907da5bb480fa58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e005687623cd4457b1804fb0ddee0652": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e0148d26d84547159d38ca490deb3986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_96e07678a0eb41faa52d009a0d33438a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_842712db1a9f4009b8254b1c0e31c8cc",
              "IPY_MODEL_427d9924799346808a1226d772e19709"
            ]
          }
        },
        "96e07678a0eb41faa52d009a0d33438a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "842712db1a9f4009b8254b1c0e31c8cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_49e97256506041e08b5afbecc2bdf1e4",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 331070498,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 331070498,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b5eddaa0668940a18a3d8c525059a772"
          }
        },
        "427d9924799346808a1226d772e19709": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4ffd34182ae04fbd9b852f90d12321aa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 331M/331M [00:08&lt;00:00, 36.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f78260e436c4a3ea20cc1563e0eb6c6"
          }
        },
        "49e97256506041e08b5afbecc2bdf1e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b5eddaa0668940a18a3d8c525059a772": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4ffd34182ae04fbd9b852f90d12321aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f78260e436c4a3ea20cc1563e0eb6c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/maxmatical/fast.ai/blob/master/fastai_hf_self_distillation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NQSjSP3bZT-f"
      },
      "source": [
        "%%capture\n",
        "!pip install fastai==2.2.7 # as of may 1 2020 fastai 2.3.0 breaks blurr\n",
        "!pip install transformers\n",
        "!pip install ohmeow-blurr==0.0.22"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ac8rAJJ-Zc-_",
        "outputId": "ca3dec9f-4118-40af-96cc-2efd7b944d53"
      },
      "source": [
        "from fastai.test_utils import show_install\n",
        "show_install()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "```text\n",
            "=== Software === \n",
            "python        : 3.7.10\n",
            "fastai        : 2.2.7\n",
            "fastcore      : 1.3.19\n",
            "fastprogress  : 0.2.7\n",
            "torch         : 1.7.1\n",
            "nvidia driver : 460.32\n",
            "torch cuda    : 10.2 / is available\n",
            "torch cudnn   : 7605 / is enabled\n",
            "\n",
            "=== Hardware === \n",
            "nvidia gpus   : 1\n",
            "torch devices : 1\n",
            "  - gpu0      : Tesla T4\n",
            "\n",
            "=== Environment === \n",
            "platform      : Linux-4.19.112+-x86_64-with-Ubuntu-18.04-bionic\n",
            "distro        : #1 SMP Thu Jul 23 08:00:38 PDT 2020\n",
            "conda env     : Unknown\n",
            "python        : /usr/bin/python3\n",
            "sys.path      : \n",
            "/content\n",
            "/env/python\n",
            "/usr/lib/python37.zip\n",
            "/usr/lib/python3.7\n",
            "/usr/lib/python3.7/lib-dynload\n",
            "/usr/local/lib/python3.7/dist-packages\n",
            "/usr/lib/python3/dist-packages\n",
            "/usr/local/lib/python3.7/dist-packages/IPython/extensions\n",
            "/root/.ipython\n",
            "```\n",
            "\n",
            "Please make sure to include opening/closing ``` when you paste into forums/github to make the reports appear formatted as code sections.\n",
            "\n",
            "Optional package(s) to enhance the diagnostics can be installed with:\n",
            "pip install distro\n",
            "Once installed, re-run this utility to get the additional information\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oh_QNPDhZeHy",
        "outputId": "44020167-64dd-41a3-e9bb-a3b500b86b88"
      },
      "source": [
        "from transformers import *\n",
        "from fastai.text.all import *\n",
        "from blurr.data.all import *\n",
        "from blurr.modeling.all import *\n",
        "import torch.nn as nn\n",
        "\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "4p6_sKNfZks_",
        "outputId": "efa0b624-c124-4e1b-b4c5-2d066459ef2f"
      },
      "source": [
        "path = untar_data(URLs.IMDB_SAMPLE)\n",
        "\n",
        "model_path = Path('models')\n",
        "imdb_df = pd.read_csv(path/'texts.csv')\n",
        "\n",
        "imdb_df.shape"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "3VdxfkknaNbs",
        "outputId": "90be6e20-e8af-49fa-8bd0-11637bf16c85"
      },
      "source": [
        "imdb_df.head(5)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>is_valid</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>negative</td>\n",
              "      <td>Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>positive</td>\n",
              "      <td>This is a extremely well-made film. The acting, script and camera-work are all first-rate. The music is good, too, though it is mostly early in the film, when things are still relatively cheery. There are no really superstars in the cast, though several faces will be familiar. The entire cast does an excellent job with the script.&lt;br /&gt;&lt;br /&gt;But it is hard to watch, because there is no good end to a situation like the one presented. It is now fashionable to blame the British for setting Hindus and Muslims against each other, and then cruelly separating them into two countries. There is som...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negative</td>\n",
              "      <td>Every once in a long while a movie will come along that will be so awful that I feel compelled to warn people. If I labor all my days and I can save but one soul from watching this movie, how great will be my joy.&lt;br /&gt;&lt;br /&gt;Where to begin my discussion of pain. For starters, there was a musical montage every five minutes. There was no character development. Every character was a stereotype. We had swearing guy, fat guy who eats donuts, goofy foreign guy, etc. The script felt as if it were being written as the movie was being shot. The production value was so incredibly low that it felt li...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>Name just says it all. I watched this movie with my dad when it came out and having served in Korea he had great admiration for the man. The disappointing thing about this film is that it only concentrate on a short period of the man's life - interestingly enough the man's entire life would have made such an epic bio-pic that it is staggering to imagine the cost for production.&lt;br /&gt;&lt;br /&gt;Some posters elude to the flawed characteristics about the man, which are cheap shots. The theme of the movie \"Duty, Honor, Country\" are not just mere words blathered from the lips of a high-brassed offic...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>negative</td>\n",
              "      <td>This movie succeeds at being one of the most unique movies you've seen. However this comes from the fact that you can't make heads or tails of this mess. It almost seems as a series of challenges set up to determine whether or not you are willing to walk out of the movie and give up the money you just paid. If you don't want to feel slighted you'll sit through this horrible film and develop a real sense of pity for the actors involved, they've all seen better days, but then you realize they actually got paid quite a bit of money to do this and you'll lose pity for them just like you've alr...</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      label  ... is_valid\n",
              "0  negative  ...    False\n",
              "1  positive  ...    False\n",
              "2  negative  ...    False\n",
              "3  positive  ...    False\n",
              "4  negative  ...    False\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "me3H5pnwZnUW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269,
          "referenced_widgets": [
            "f7497a88ef0340bb9d9f72755e4a92fa",
            "6fd1c8bfb6cc493cad4089159c730903",
            "4123b9e0b86e4fbea7fbd321c69f7ac2",
            "234736284208438383ec1246f0fff4c0",
            "5b0a5cdab2924f9c9cb334cce79e79a1",
            "02a17cadcf234def9c2dde3ac34b571d",
            "56685af582e942dcae0def756315f78b",
            "47ed65f464c14e62aad79dfa0a98d674",
            "d6b9012cbc764a73b8c736d92a8710a6",
            "8d8dd30f9a36499997016b7946c056af",
            "870be6cf53514cf1995501277efaad2f",
            "d7e3c4b336ac42c69a3b37ca98462388",
            "1611eb56880747bb959701ab2dd4cbb7",
            "d9664da2db9746648d7b2837f70dc3fe",
            "f288262e758d44979033d3ba0dc0ff6a",
            "4b9bc5d93eb44e63b65768066096a826",
            "8ad5cb6603f64a6cbac84f91fd0ff7ba",
            "10055610fe99426b98fe47e7a8dd266a",
            "280a84059af643909a45f3de1985d3c2",
            "52ec7483ed3e47b380807644ba94655d",
            "35c7be8158db4ff49c6425161e9ae778",
            "8270604815144a1cbd731b929717a4ac",
            "ff537965e1e24930ad5bec57ff8cad9a",
            "67964b45451f4e4a9865b9e7253e6c1a",
            "a29764f8440e41cd89de0f41f901eb3c",
            "c6e1bfb37bbe4686ba1ed68bf7212f7b",
            "4de30ea2b8764ec188766e1221d875e0",
            "3fda686524ee4397856c4a21b30b0ac7",
            "733df8d7046b43afb8a037b2041c9c6c",
            "b003df6a476c48268691858b7b8d1701",
            "ccf6e20e9ced4793b907da5bb480fa58",
            "e005687623cd4457b1804fb0ddee0652",
            "e0148d26d84547159d38ca490deb3986",
            "96e07678a0eb41faa52d009a0d33438a",
            "842712db1a9f4009b8254b1c0e31c8cc",
            "427d9924799346808a1226d772e19709",
            "49e97256506041e08b5afbecc2bdf1e4",
            "b5eddaa0668940a18a3d8c525059a772",
            "4ffd34182ae04fbd9b852f90d12321aa",
            "6f78260e436c4a3ea20cc1563e0eb6c6"
          ]
        },
        "outputId": "f1851b15-5fef-4821-c35e-59b55b5f7899"
      },
      "source": [
        "pretrained_model_name = 'distilroberta-base'\n",
        "hf_model_cls = RobertaForSequenceClassification\n",
        "hf_arch, hf_config, hf_tokenizer, hf_model = BLURR_MODEL_HELPER.get_hf_objects(pretrained_model_name,\n",
        "                                                                               model_cls=hf_model_cls)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f7497a88ef0340bb9d9f72755e4a92fa",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=480.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d6b9012cbc764a73b8c736d92a8710a6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8ad5cb6603f64a6cbac84f91fd0ff7ba",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a29764f8440e41cd89de0f41f901eb3c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1355863.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e0148d26d84547159d38ca490deb3986",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=331070498.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPQRhs2WcUMg"
      },
      "source": [
        "blocks = (HF_TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model), CategoryBlock)\n",
        "dblock = DataBlock(blocks=blocks,  get_x=ColReader('text'), get_y=ColReader('label'), splitter=ColSplitter())\n",
        "\n",
        "dls = dblock.dataloaders(imdb_df, bs=4)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        },
        "id": "Mjp9DntkyHN7",
        "outputId": "c09cabb3-d974-4444-ea56-9a98d225494a"
      },
      "source": [
        "dls.show_batch(dataloaders=dls, max_n=2)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Raising Victor Vargas: A Review&lt;br /&gt;&lt;br /&gt;You know, Raising Victor Vargas is like sticking your hands into a big, steaming bowl of oatmeal. It's warm and gooey, but you're not sure if it feels right. Try as I might, no matter how warm and gooey Raising Victor Vargas became I was always aware that something didn't quite feel right. Victor Vargas suffers from a certain overconfidence on the director's part. Apparently, the director thought that the ethnic backdrop of a Latino family on the lower east side, and an idyllic storyline would make the film critic proof. He was right, but it didn't fool me. Raising Victor Vargas is the story about a seventeen-year old boy called, you guessed it, Victor Vargas (Victor Rasuk) who lives his teenage years chasing more skirt than the Rolling Stones could do in all the years they've toured. The movie starts off in `Ugly Fat' Donna's bedroom where Victor is sure to seduce her, but a cry from outside disrupts his plans when his best-friend Harold (Kevin Rivera) comes-a-looking for him. Caught in the attempt by Harold and his sister, Victor Vargas runs off for damage control. Yet even with the embarrassing implication that he's been boffing the homeliest girl in the neighborhood, nothing dissuades young Victor from going off on the hunt for more fresh meat. On a hot, New York City day they make way to the local public swimming pool where Victor's eyes catch a glimpse of the lovely young nymph Judy (Judy Marte), who's not just pretty, but a strong and independent too. The relationship that develops between Victor and Judy becomes the focus of the film. The story also focuses on Victor's family that is comprised of his grandmother or abuelita (Altagracia Guzman), his brother Nino (also played by real life brother to Victor, Silvestre Rasuk) and his sister Vicky (Krystal Rodriguez). The action follows Victor between scenes with Judy and scenes with his family. Victor tries to cope with being an oversexed pimp-daddy, his feelings for Judy and his grandmother's conservative Catholic upbringing.&lt;br /&gt;&lt;br /&gt;The problems that arise from Raising Victor Vargas are a few, but glaring errors. Throughout the film you get to know certain characters like Vicky, Nino, Grandma,</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Now that Che(2008) has finished its relatively short Australian cinema run (extremely limited release:1 screen in Sydney, after 6wks), I can guiltlessly join both hosts of \"At The Movies\" in taking Steven Soderbergh to task.&lt;br /&gt;&lt;br /&gt;It's usually satisfying to watch a film director change his style/subject, but Soderbergh's most recent stinker, The Girlfriend Experience(2009), was also missing a story, so narrative (and editing?) seem to suddenly be Soderbergh's main challenge. Strange, after 20-odd years in the business. He was probably never much good at narrative, just hid it well inside \"edgy\" projects.&lt;br /&gt;&lt;br /&gt;None of this excuses him this present, almost diabolical failure. As David Stratton warns, \"two parts of Che don't (even) make a whole\". &lt;br /&gt;&lt;br /&gt;Epic biopic in name only, Che(2008) barely qualifies as a feature film! It certainly has no legs, inasmuch as except for its uncharacteristic ultimate resolution forced upon it by history, Soderbergh's 4.5hrs-long dirge just goes nowhere.&lt;br /&gt;&lt;br /&gt;Even Margaret Pomeranz, the more forgiving of Australia's At The Movies duo, noted about Soderbergh's repetitious waste of (HD digital storage): \"you're in the woods...you're in the woods...you're in the woods...\". I too am surprised Soderbergh didn't give us another 2.5hrs of THAT somewhere between his existing two Parts, because he still left out massive chunks of Che's \"revolutionary\" life! &lt;br /&gt;&lt;br /&gt;For a biopic of an important but infamous historical figure, Soderbergh unaccountably alienates, if not deliberately insults, his audiences by&lt;br /&gt;&lt;br /&gt;1. never providing most of Che's story; &lt;br /&gt;&lt;br /&gt;2. imposing unreasonable film lengths with mere dullard repetition; &lt;br /&gt;&lt;br /&gt;3. ignoring both true hindsight and a narrative of events; &lt;br /&gt;&lt;br /&gt;4. barely developing an idea, or a character; &lt;br /&gt;&lt;br /&gt;5. remaining claustrophobically episodic; &lt;br /&gt;&lt;br /&gt;6. ignoring proper context for scenes---whatever we do get is mired in disruptive timeshifts; &lt;br /&gt;&lt;br /&gt;7. linguistically</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nLLqFQ6QyMTC"
      },
      "source": [
        "### Training 1st model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "84usX3gHyJMi"
      },
      "source": [
        "model = HF_BaseModelWrapper(hf_model)\n",
        "\n",
        "learn = Learner(dls, \n",
        "                model,\n",
        "                loss_func=LabelSmoothingCrossEntropyFlat(),\n",
        "                metrics=[accuracy],\n",
        "                cbs=[HF_BaseModelCallback],\n",
        "                splitter=hf_splitter).to_fp16()\n",
        "\n",
        "learn.unfreeze()\n",
        "\n",
        "cbs = [\n",
        "    GradientAccumulation(n_acc=32),\n",
        "    ReduceLROnPlateau(monitor='accuracy', factor=5, patience=5),\n",
        "    EarlyStoppingCallback(monitor='accuracy', min_delta=0.01, patience=7),\n",
        "    SaveModelCallback(monitor=\"accuracy\", fname=\"model_1\")\n",
        "]\n",
        "\n"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        },
        "id": "QRP-NoQjyUeH",
        "outputId": "07a03a9d-93c1-4659-8391-76900ec873d4"
      },
      "source": [
        "learn.fit_one_cycle(\n",
        "    3,\n",
        "    lr_max=3e-5,\n",
        "    cbs = cbs\n",
        ")\n",
        "\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.688996</td>\n",
              "      <td>0.672182</td>\n",
              "      <td>0.780000</td>\n",
              "      <td>00:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.477257</td>\n",
              "      <td>0.407832</td>\n",
              "      <td>0.875000</td>\n",
              "      <td>00:14</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.347807</td>\n",
              "      <td>0.374730</td>\n",
              "      <td>0.915000</td>\n",
              "      <td>00:15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Better model found at epoch 0 with accuracy value: 0.7799999713897705.\n",
            "Better model found at epoch 1 with accuracy value: 0.875.\n",
            "Better model found at epoch 2 with accuracy value: 0.9150000214576721.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U5OBv-GXyY6v"
      },
      "source": [
        "### generate psuedo-labels for both train\n",
        "\n",
        "use 1 hot encoded for validation data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UENZ4kY-5W70"
      },
      "source": [
        "df_train = imdb_df[imdb_df[\"is_valid\"]==False]"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3UYKMODFyYIQ"
      },
      "source": [
        "test_dl = learn.dls.test_dl(list(df_train[\"text\"]))"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7MUvRVMVytZA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "f38418a2-9b6d-4e14-bca7-321eb7658ff8"
      },
      "source": [
        "pred_proba, _ = learn.get_preds(dl=test_dl, with_decoded=False)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3THJcUH0qKf2"
      },
      "source": [
        "pred_proba = pred_proba.cpu().numpy()"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vy4eihhPo3us",
        "outputId": "b60d7cf7-9274-4618-99ad-12f375b2ecc3"
      },
      "source": [
        "pred_proba.shape"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(800, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CwlqoUP0o4yt",
        "outputId": "8585cdc7-3f99-4c79-dbf3-0d644cfab562"
      },
      "source": [
        "pred_proba[1]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.17050801, 0.82949203], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8tCioTorrSyM"
      },
      "source": [
        "`learn.get_preds` defaults to activation associated with the loss function, so applying softmax to the logits"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmtmjwFfr5-5",
        "outputId": "3128999c-0586-44b9-8fe5-0f77d3113378"
      },
      "source": [
        "# get the mapping from class labels to index of predicted probabilities\n",
        "learn.dls.vocab"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['negative', 'positive']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bpZw1VBDqafg"
      },
      "source": [
        "# old code, leave commented for now because it might be useful in the future\n",
        "\n",
        "# pred_proba_df = pd.DataFrame(pred_proba, columns=[\"negative\", \"positive\"])\n",
        "# df_pseudolabels = pd.concat([imdb_df, pred_proba_df], axis=1)\n",
        "# df_pseudolabels.head(5)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gZUFjxn956po"
      },
      "source": [
        "add pseudo labels for training data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "MAB-c2RU5nKo",
        "outputId": "305e9adf-7569-4d03-f92b-cde8edb67518"
      },
      "source": [
        "pred_proba_df_train = pd.DataFrame(pred_proba, columns=[\"negative\", \"positive\"])\n",
        "df_pseudolabels_train = pd.concat([df_train, pred_proba_df_train], axis=1)\n",
        "df_pseudolabels_train.head(5)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>is_valid</th>\n",
              "      <th>negative</th>\n",
              "      <th>positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>negative</td>\n",
              "      <td>Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!</td>\n",
              "      <td>False</td>\n",
              "      <td>0.902180</td>\n",
              "      <td>0.097820</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>positive</td>\n",
              "      <td>This is a extremely well-made film. The acting, script and camera-work are all first-rate. The music is good, too, though it is mostly early in the film, when things are still relatively cheery. There are no really superstars in the cast, though several faces will be familiar. The entire cast does an excellent job with the script.&lt;br /&gt;&lt;br /&gt;But it is hard to watch, because there is no good end to a situation like the one presented. It is now fashionable to blame the British for setting Hindus and Muslims against each other, and then cruelly separating them into two countries. There is som...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.170508</td>\n",
              "      <td>0.829492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negative</td>\n",
              "      <td>Every once in a long while a movie will come along that will be so awful that I feel compelled to warn people. If I labor all my days and I can save but one soul from watching this movie, how great will be my joy.&lt;br /&gt;&lt;br /&gt;Where to begin my discussion of pain. For starters, there was a musical montage every five minutes. There was no character development. Every character was a stereotype. We had swearing guy, fat guy who eats donuts, goofy foreign guy, etc. The script felt as if it were being written as the movie was being shot. The production value was so incredibly low that it felt li...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.933249</td>\n",
              "      <td>0.066751</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>Name just says it all. I watched this movie with my dad when it came out and having served in Korea he had great admiration for the man. The disappointing thing about this film is that it only concentrate on a short period of the man's life - interestingly enough the man's entire life would have made such an epic bio-pic that it is staggering to imagine the cost for production.&lt;br /&gt;&lt;br /&gt;Some posters elude to the flawed characteristics about the man, which are cheap shots. The theme of the movie \"Duty, Honor, Country\" are not just mere words blathered from the lips of a high-brassed offic...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.506622</td>\n",
              "      <td>0.493378</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>negative</td>\n",
              "      <td>This movie succeeds at being one of the most unique movies you've seen. However this comes from the fact that you can't make heads or tails of this mess. It almost seems as a series of challenges set up to determine whether or not you are willing to walk out of the movie and give up the money you just paid. If you don't want to feel slighted you'll sit through this horrible film and develop a real sense of pity for the actors involved, they've all seen better days, but then you realize they actually got paid quite a bit of money to do this and you'll lose pity for them just like you've alr...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.842215</td>\n",
              "      <td>0.157785</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      label  ...  positive\n",
              "0  negative  ...  0.097820\n",
              "1  positive  ...  0.829492\n",
              "2  negative  ...  0.066751\n",
              "3  positive  ...  0.493378\n",
              "4  negative  ...  0.157785\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YN1m4z2759gI"
      },
      "source": [
        "one-hot encode val df"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "9gngaTDz5_xL",
        "outputId": "1a106498-de2b-4bff-c49e-06f21177dd5f"
      },
      "source": [
        "df_val = imdb_df[imdb_df[\"is_valid\"]==True]\n",
        "pred_proba_df_val = pd.get_dummies(df_val[\"label\"])\n",
        "df_pseudolabels_val = pd.concat([df_val, pred_proba_df_val], axis=1)\n",
        "df_pseudolabels_val.head(5)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>is_valid</th>\n",
              "      <th>negative</th>\n",
              "      <th>positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>800</th>\n",
              "      <td>positive</td>\n",
              "      <td>This very funny British comedy shows what might happen if a section of London, in this case Pimlico, were to declare itself independent from the rest of the UK and its laws, taxes &amp; post-war restrictions. Merry mayhem is what would happen.&lt;br /&gt;&lt;br /&gt;The explosion of a wartime bomb leads to the discovery of ancient documents which show that Pimlico was ceded to the Duchy of Burgundy centuries ago, a small historical footnote long since forgotten. To the new Burgundians, however, this is an unexpected opportunity to live as they please, free from any interference from Whitehall.&lt;br /&gt;&lt;br /&gt;...</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>801</th>\n",
              "      <td>positive</td>\n",
              "      <td>I saw this movie once as a kid on the late-late show and fell in love with it.&lt;br /&gt;&lt;br /&gt;It took 30+ years, but I recently did find it on DVD - it wasn't cheap, either - in a catalog that specialized in war movies. We watched it last night for the first time. The audio was good, however it was grainy and had the trailers between reels. Even so, it was better than I remembered it. I was also impressed at how true it was to the play.&lt;br /&gt;&lt;br /&gt;The catalog is around here someplace. If you're sincere in finding it, fire me a missive and I'll see if I can get you the info. cartwrightbride@yah...</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>802</th>\n",
              "      <td>positive</td>\n",
              "      <td>This is, in my opinion, a very good film, especially for Michael Jackson lovers. It contains a message on drugs, stunning special effects, and an awesome music video.&lt;br /&gt;&lt;br /&gt;The main film is centered around the song and music video 'Smooth Criminal.' Unlike the four-minute music video, it is normal speed and, in my opinion, much easier to watch.&lt;br /&gt;&lt;br /&gt;The plot is rather weird, however. Michael Jackson plays a magical 'gangster' that, when he sees a shooting star, he transforms into a piece of machinery. Throughout the film, he transforms into a race car, a giant robot, and a space...</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>803</th>\n",
              "      <td>positive</td>\n",
              "      <td>In Iran, women are not permitted to attend men's sporting events, apparently to \"protect\" them from all the cursing and foul language they might hear emanating from the male fans (so since men can't restrain or behave themselves, women are forced to suffer. Go figure.). \"Offside\" tells the tale of a half dozen or so young women who, dressed like men, attempt to sneak into the high-stakes match between Iran and Bahrain that, in 2005, qualified Iran to go to the World Cup (the movie was actually filmed in large part during that game).&lt;br /&gt;&lt;br /&gt;\"Offside\" is a slice-of-life comedy that will ...</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>804</th>\n",
              "      <td>positive</td>\n",
              "      <td>\"In April 1946, the University of Chicago agreed to operate Argonne National Laboratory, with an association of Midwestern universities offering to sponsor the research. Argonne thereby became the first \"national\" laboratory. It did not, however, remain at its original location in the Argonne forest. In 1947, it moved farther west from the \"Windy City\" to a new site on Illinois farmland. When Alvin Weinberg visited Argonne's director, Walter Zinn, in 1947, he asked him what kind of reactor was to be built at the new site. When Zinn described a heavy-water reactor operating at one-tenth the...</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        label  ... positive\n",
              "800  positive  ...        1\n",
              "801  positive  ...        1\n",
              "802  positive  ...        1\n",
              "803  positive  ...        1\n",
              "804  positive  ...        1\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "ikHzxOh57nyB",
        "outputId": "c0c14468-9412-4e68-dd8f-d11991bffeec"
      },
      "source": [
        "# combine the 2 dfs\n",
        "df_pseudolabels = pd.concat([df_pseudolabels_train, df_pseudolabels_val], axis=0, ignore_index=True)\n",
        "df_pseudolabels.head(5)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>is_valid</th>\n",
              "      <th>negative</th>\n",
              "      <th>positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>negative</td>\n",
              "      <td>Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!</td>\n",
              "      <td>False</td>\n",
              "      <td>0.902180</td>\n",
              "      <td>0.097820</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>positive</td>\n",
              "      <td>This is a extremely well-made film. The acting, script and camera-work are all first-rate. The music is good, too, though it is mostly early in the film, when things are still relatively cheery. There are no really superstars in the cast, though several faces will be familiar. The entire cast does an excellent job with the script.&lt;br /&gt;&lt;br /&gt;But it is hard to watch, because there is no good end to a situation like the one presented. It is now fashionable to blame the British for setting Hindus and Muslims against each other, and then cruelly separating them into two countries. There is som...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.170508</td>\n",
              "      <td>0.829492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negative</td>\n",
              "      <td>Every once in a long while a movie will come along that will be so awful that I feel compelled to warn people. If I labor all my days and I can save but one soul from watching this movie, how great will be my joy.&lt;br /&gt;&lt;br /&gt;Where to begin my discussion of pain. For starters, there was a musical montage every five minutes. There was no character development. Every character was a stereotype. We had swearing guy, fat guy who eats donuts, goofy foreign guy, etc. The script felt as if it were being written as the movie was being shot. The production value was so incredibly low that it felt li...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.933249</td>\n",
              "      <td>0.066751</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>Name just says it all. I watched this movie with my dad when it came out and having served in Korea he had great admiration for the man. The disappointing thing about this film is that it only concentrate on a short period of the man's life - interestingly enough the man's entire life would have made such an epic bio-pic that it is staggering to imagine the cost for production.&lt;br /&gt;&lt;br /&gt;Some posters elude to the flawed characteristics about the man, which are cheap shots. The theme of the movie \"Duty, Honor, Country\" are not just mere words blathered from the lips of a high-brassed offic...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.506622</td>\n",
              "      <td>0.493378</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>negative</td>\n",
              "      <td>This movie succeeds at being one of the most unique movies you've seen. However this comes from the fact that you can't make heads or tails of this mess. It almost seems as a series of challenges set up to determine whether or not you are willing to walk out of the movie and give up the money you just paid. If you don't want to feel slighted you'll sit through this horrible film and develop a real sense of pity for the actors involved, they've all seen better days, but then you realize they actually got paid quite a bit of money to do this and you'll lose pity for them just like you've alr...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.842215</td>\n",
              "      <td>0.157785</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      label  ...  positive\n",
              "0  negative  ...  0.097820\n",
              "1  positive  ...  0.829492\n",
              "2  negative  ...  0.066751\n",
              "3  positive  ...  0.493378\n",
              "4  negative  ...  0.157785\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "zGlEJinv7-qH",
        "outputId": "f783ea4a-61b9-4a80-8675-436347d648f5"
      },
      "source": [
        "df_pseudolabels.tail(5)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>is_valid</th>\n",
              "      <th>negative</th>\n",
              "      <th>positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>negative</td>\n",
              "      <td>There are many different versions of this one floating around, so make sure you can locate one of the unrated copies, otherwise some gore and one scene of nudity might be missing. Some versions also omit most of the opening sequence and other bits here and there. The cut I saw has the on-screen title WITCHCRAFT: EVIL ENCOUNTERS and was released by Shriek Show, who maintain the original US release title WITCHERY for the DVD release. It's a nice-looking print and seems to have all of the footage, but has some cropping/aspect ratio issues. In Italy, it was released as LA CASA 4 (WITCHCRAFT). ...</td>\n",
              "      <td>True</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>positive</td>\n",
              "      <td>Once upon a time Hollywood produced live-action, G-rated movies without foul language, immorality, and gore-splattered violence. These movies neither insulted your intelligence no manipulated your emotions. The heroes differed little from the crowd. They shared the same feelings and bore the same burdens. Since the 1970s, the film industry has pretty much written off G-rated movies for adults. Basically, modern mature audiences demand large doses of embellished realism for their cinematic diet, laced heavily with vile profanity, mattress-thumping sex, and knuckle-bruising fisticuffs. These...</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>negative</td>\n",
              "      <td>Wenders was great with Million $ Hotel.I don't know how he came up with this film! The idea of giving the situation after spt11 and the view of American Society is hopeful,that makes it 2 out of ten.But this is not a movie.Is that the best someone can do with a great idea(the west-east clash).There are important things going on in middle east and it is just issued on the screen of a MAC* with the fingers of an Amerian girl who is actually at the level of stupidity(because she is just ignorant about the facts).The characters are not well shaped.And the most important thing is the idea that ...</td>\n",
              "      <td>True</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>negative</td>\n",
              "      <td>Although a film with Bruce Willis is always worth watching, you better skip this one. I watched this one on television, so I didn't have to plunk down cash for it. Lucky me.&lt;br /&gt;&lt;br /&gt;The plot develops slowly, very slowly. Although the first 30 minutes or so are quite believable, it gets more and more unbelievable towards the end. It is highly questionable, if a seasoned soldier like Lt. Waters would disobey direct orders. And even if he would, if the rest of his platoon would. They know he puts them in direct danger, and they know they will certainly die if they follow him, but what the ...</td>\n",
              "      <td>True</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>positive</td>\n",
              "      <td>A compelling, honest, daring, and unforgettable psychological horror film that touches on the painful experiences of pain caused by rape - \"Descent\" is a film that went under-the-radar due to its lack of distribution because, frankly, the film is so brutal in its depictions, that if it had been released theatrically, it may have met itself to some strong biased hate.&lt;br /&gt;&lt;br /&gt;The film deserves to be discovered for, not only its dark themes, and not only for its amazing direction and authentic style - but most of all for its performances. Chad Faust is absolutely stunning, bringing enough...</td>\n",
              "      <td>True</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        label  ... positive\n",
              "995  negative  ...      0.0\n",
              "996  positive  ...      1.0\n",
              "997  negative  ...      0.0\n",
              "998  negative  ...      0.0\n",
              "999  positive  ...      1.0\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ES3zN2Tgsfx_"
      },
      "source": [
        "## Train on pseudolabels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "akzXWIIPsW0x"
      },
      "source": [
        "blocks = (HF_TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model), RegressionBlock) # use regression block for regression tasks\n",
        "dblock = DataBlock(blocks=blocks,  get_x=ColReader('text'), get_y=ColReader(['negative', 'positive']), splitter=ColSplitter()) # multiple columns for probabilities\n",
        "\n",
        "dls = dblock.dataloaders(df_pseudolabels, bs=4)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        },
        "id": "P7TiTNYDs65e",
        "outputId": "a5302d45-b416-439a-f251-9379310ba13e"
      },
      "source": [
        "dls.show_batch(dataloaders=dls, max_n=2)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>text_</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Raising Victor Vargas: A Review&lt;br /&gt;&lt;br /&gt;You know, Raising Victor Vargas is like sticking your hands into a big, steaming bowl of oatmeal. It's warm and gooey, but you're not sure if it feels right. Try as I might, no matter how warm and gooey Raising Victor Vargas became I was always aware that something didn't quite feel right. Victor Vargas suffers from a certain overconfidence on the director's part. Apparently, the director thought that the ethnic backdrop of a Latino family on the lower east side, and an idyllic storyline would make the film critic proof. He was right, but it didn't fool me. Raising Victor Vargas is the story about a seventeen-year old boy called, you guessed it, Victor Vargas (Victor Rasuk) who lives his teenage years chasing more skirt than the Rolling Stones could do in all the years they've toured. The movie starts off in `Ugly Fat' Donna's bedroom where Victor is sure to seduce her, but a cry from outside disrupts his plans when his best-friend Harold (Kevin Rivera) comes-a-looking for him. Caught in the attempt by Harold and his sister, Victor Vargas runs off for damage control. Yet even with the embarrassing implication that he's been boffing the homeliest girl in the neighborhood, nothing dissuades young Victor from going off on the hunt for more fresh meat. On a hot, New York City day they make way to the local public swimming pool where Victor's eyes catch a glimpse of the lovely young nymph Judy (Judy Marte), who's not just pretty, but a strong and independent too. The relationship that develops between Victor and Judy becomes the focus of the film. The story also focuses on Victor's family that is comprised of his grandmother or abuelita (Altagracia Guzman), his brother Nino (also played by real life brother to Victor, Silvestre Rasuk) and his sister Vicky (Krystal Rodriguez). The action follows Victor between scenes with Judy and scenes with his family. Victor tries to cope with being an oversexed pimp-daddy, his feelings for Judy and his grandmother's conservative Catholic upbringing.&lt;br /&gt;&lt;br /&gt;The problems that arise from Raising Victor Vargas are a few, but glaring errors. Throughout the film you get to know certain characters like Vicky, Nino, Grandma,</td>\n",
              "      <td>(0.43151307106018066, 0.5684869289398193)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>This film sat on my Tivo for weeks before I watched it. I dreaded a self-indulgent yuppie flick about relationships gone bad. I was wrong; this was an engrossing excursion into the screwed-up libidos of New Yorkers.&lt;br /&gt;&lt;br /&gt;The format is the same as Max Ophuls' \"La Ronde,\" based on a play by Arthur Schnitzler, who is given an \"inspired by\" credit. It starts from one person, a prostitute, standing on a street corner in Brooklyn. She is picked up by a home contractor, who has sex with her on the hood of a car, but can't come. He refuses to pay her. When he's off peeing, she answers his cell phone and takes a message. She runs away with his keys.&lt;br /&gt;&lt;br /&gt;Then the story switches to the contractor, who pays a professional call on a rich, bored New York woman, who plays with him until he is aroused, then she pulls away. She tells him how desperate and unhappy she is; he tells her how beautiful she is, and lucky. As he is leaving, she asks if he would have sex with her. She sits on top of him, bounces up and down. This time he comes, the he leaves.&lt;br /&gt;&lt;br /&gt;The woman and her husband throw a dinner party for their trendy friends. Hubby (Robert) is talking business, wife (Ellen) is bored, and switches the subject to sex, and how often men and women think about it. Husband switches conversation to desert. Later, after the guests leave, Ellen tries to entice Robert into sex. Robert wants none of it, and puts on a jazz record. Ellen turns on the radio; Robert turns up the music; Ellen turns on the TV; Robert turns on another TV. Cacophony ensues. Ellen goes up on the roof, Robert joins her. Ellen confesses that she needs to experience more men, men other than Robert. Robert says that he too needs to experience men.&lt;br /&gt;&lt;br /&gt;We next follow Robert as he visits an artist, Martin, played by Steve Buscemi. I wish Buscemi could have more roles like this, where he is a sexy, smart, totally desirable guy. Robert praises Martin's work, much more than it deserves, promises to get it into a show. Martin is excited, until it turns out that Robert is speaking out of his groin, it</td>\n",
              "      <td>(0.1911766231060028, 0.8088233470916748)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5N5L_6ZhuhcW"
      },
      "source": [
        "# accuracy for pseudolabels\n",
        "def pseudolabel_accuracy(pred, target):\n",
        "    return (pred.argmax(dim=1) == target.argmax(dim=1)).float().mean()\n"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpI9G4nxs75D"
      },
      "source": [
        "model = HF_BaseModelWrapper(hf_model)\n",
        "\n",
        "learn = Learner(dls, \n",
        "                model,\n",
        "                loss_func=MSELossFlat(),\n",
        "                metrics=[pseudolabel_accuracy],\n",
        "                cbs=[HF_BaseModelCallback],\n",
        "                splitter=hf_splitter).to_fp16()\n",
        "\n",
        "learn.unfreeze()\n",
        "\n",
        "# maybe this should be (training?) loss?\n",
        "cbs = [\n",
        "    GradientAccumulation(n_acc=32),\n",
        "    ReduceLROnPlateau(monitor='pseudolabel_accuracy', factor=5, patience=5),\n",
        "    EarlyStoppingCallback(monitor='pseudolabel_accuracy', min_delta=0.01, patience=7),\n",
        "    SaveModelCallback(monitor=\"pseudolabel_accuracy\", fname=\"model_1\")\n",
        "]\n",
        "\n"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        },
        "id": "TXM3sKJiuepG",
        "outputId": "1fadf567-c39b-43ad-9627-57c9c2bcd652"
      },
      "source": [
        "learn.fit_one_cycle(\n",
        "    3,\n",
        "    lr_max=3e-5,\n",
        "    cbs = cbs\n",
        ")\n"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>pseudolabel_accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.084418</td>\n",
              "      <td>0.110880</td>\n",
              "      <td>0.895000</td>\n",
              "      <td>00:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.021435</td>\n",
              "      <td>0.086709</td>\n",
              "      <td>0.895000</td>\n",
              "      <td>00:15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.017692</td>\n",
              "      <td>0.085655</td>\n",
              "      <td>0.900000</td>\n",
              "      <td>00:15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Better model found at epoch 0 with pseudolabel_accuracy value: 0.8949999809265137.\n",
            "Better model found at epoch 2 with pseudolabel_accuracy value: 0.8999999761581421.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-XYvyq7fu-ad"
      },
      "source": [
        "check results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "y06qLGm0vPen",
        "outputId": "fab2d2b5-c76c-4d38-f9e5-032be54d8e9e"
      },
      "source": [
        "df_pseudolabels.head(5)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>is_valid</th>\n",
              "      <th>negative</th>\n",
              "      <th>positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>negative</td>\n",
              "      <td>Un-bleeping-believable! Meg Ryan doesn't even look her usual pert lovable self in this, which normally makes me forgive her shallow ticky acting schtick. Hard to believe she was the producer on this dog. Plus Kevin Kline: what kind of suicide trip has his career been on? Whoosh... Banzai!!! Finally this was directed by the guy who did Big Chill? Must be a replay of Jonestown - hollywood style. Wooofff!</td>\n",
              "      <td>False</td>\n",
              "      <td>0.902180</td>\n",
              "      <td>0.097820</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>positive</td>\n",
              "      <td>This is a extremely well-made film. The acting, script and camera-work are all first-rate. The music is good, too, though it is mostly early in the film, when things are still relatively cheery. There are no really superstars in the cast, though several faces will be familiar. The entire cast does an excellent job with the script.&lt;br /&gt;&lt;br /&gt;But it is hard to watch, because there is no good end to a situation like the one presented. It is now fashionable to blame the British for setting Hindus and Muslims against each other, and then cruelly separating them into two countries. There is som...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.170508</td>\n",
              "      <td>0.829492</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negative</td>\n",
              "      <td>Every once in a long while a movie will come along that will be so awful that I feel compelled to warn people. If I labor all my days and I can save but one soul from watching this movie, how great will be my joy.&lt;br /&gt;&lt;br /&gt;Where to begin my discussion of pain. For starters, there was a musical montage every five minutes. There was no character development. Every character was a stereotype. We had swearing guy, fat guy who eats donuts, goofy foreign guy, etc. The script felt as if it were being written as the movie was being shot. The production value was so incredibly low that it felt li...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.933249</td>\n",
              "      <td>0.066751</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>Name just says it all. I watched this movie with my dad when it came out and having served in Korea he had great admiration for the man. The disappointing thing about this film is that it only concentrate on a short period of the man's life - interestingly enough the man's entire life would have made such an epic bio-pic that it is staggering to imagine the cost for production.&lt;br /&gt;&lt;br /&gt;Some posters elude to the flawed characteristics about the man, which are cheap shots. The theme of the movie \"Duty, Honor, Country\" are not just mere words blathered from the lips of a high-brassed offic...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.506622</td>\n",
              "      <td>0.493378</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>negative</td>\n",
              "      <td>This movie succeeds at being one of the most unique movies you've seen. However this comes from the fact that you can't make heads or tails of this mess. It almost seems as a series of challenges set up to determine whether or not you are willing to walk out of the movie and give up the money you just paid. If you don't want to feel slighted you'll sit through this horrible film and develop a real sense of pity for the actors involved, they've all seen better days, but then you realize they actually got paid quite a bit of money to do this and you'll lose pity for them just like you've alr...</td>\n",
              "      <td>False</td>\n",
              "      <td>0.842215</td>\n",
              "      <td>0.157785</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      label  ...  positive\n",
              "0  negative  ...  0.097820\n",
              "1  positive  ...  0.829492\n",
              "2  negative  ...  0.066751\n",
              "3  positive  ...  0.493378\n",
              "4  negative  ...  0.157785\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "39K_en-Qur0Q",
        "outputId": "f6b811e1-84fa-4821-80a4-e8317d04e72d"
      },
      "source": [
        "test_dl = learn.dls.test_dl(list(imdb_df[\"text\"]))\n",
        "pred_proba, _ = learn.get_preds(dl=test_dl, with_decoded=False)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8tmMk0Tlu_mc",
        "outputId": "f75a8356-ef94-4741-cbb1-06b8a0b9bf0b"
      },
      "source": [
        "pred_proba = pred_proba.cpu().numpy()\n",
        "pred_proba.shape"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1000, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kVCGVxLkva2f"
      },
      "source": [
        "test accuracy with actual labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sRQuHfXYvG6O",
        "outputId": "64acf16c-4097-4683-83b3-504918cff6fc"
      },
      "source": [
        "preds = pred_proba.argmax(axis=1)\n",
        "pred_labels = [\"negative\" if p == 0 else \"positive\" for p in preds ]\n",
        "print(len(pred_labels))\n",
        "pred_labels[:5]\n"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['negative', 'positive', 'negative', 'positive', 'negative']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "id": "k-EwPv26vonw",
        "outputId": "c173e9cc-eea6-4f62-8e36-427b20ba1aa0"
      },
      "source": [
        "df_distilled_preds = pd.DataFrame(data=pred_labels, columns=[\"self_distilled_pred_label\"])\n",
        "df_performance = pd.concat([imdb_df, df_distilled_preds], axis=1)\n",
        "df_performance_val = df_performance[df_performance[\"is_valid\"]==True]\n",
        "df_performance_val.head(5)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>is_valid</th>\n",
              "      <th>self_distilled_pred_label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>800</th>\n",
              "      <td>positive</td>\n",
              "      <td>This very funny British comedy shows what might happen if a section of London, in this case Pimlico, were to declare itself independent from the rest of the UK and its laws, taxes &amp; post-war restrictions. Merry mayhem is what would happen.&lt;br /&gt;&lt;br /&gt;The explosion of a wartime bomb leads to the discovery of ancient documents which show that Pimlico was ceded to the Duchy of Burgundy centuries ago, a small historical footnote long since forgotten. To the new Burgundians, however, this is an unexpected opportunity to live as they please, free from any interference from Whitehall.&lt;br /&gt;&lt;br /&gt;...</td>\n",
              "      <td>True</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>801</th>\n",
              "      <td>positive</td>\n",
              "      <td>I saw this movie once as a kid on the late-late show and fell in love with it.&lt;br /&gt;&lt;br /&gt;It took 30+ years, but I recently did find it on DVD - it wasn't cheap, either - in a catalog that specialized in war movies. We watched it last night for the first time. The audio was good, however it was grainy and had the trailers between reels. Even so, it was better than I remembered it. I was also impressed at how true it was to the play.&lt;br /&gt;&lt;br /&gt;The catalog is around here someplace. If you're sincere in finding it, fire me a missive and I'll see if I can get you the info. cartwrightbride@yah...</td>\n",
              "      <td>True</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>802</th>\n",
              "      <td>positive</td>\n",
              "      <td>This is, in my opinion, a very good film, especially for Michael Jackson lovers. It contains a message on drugs, stunning special effects, and an awesome music video.&lt;br /&gt;&lt;br /&gt;The main film is centered around the song and music video 'Smooth Criminal.' Unlike the four-minute music video, it is normal speed and, in my opinion, much easier to watch.&lt;br /&gt;&lt;br /&gt;The plot is rather weird, however. Michael Jackson plays a magical 'gangster' that, when he sees a shooting star, he transforms into a piece of machinery. Throughout the film, he transforms into a race car, a giant robot, and a space...</td>\n",
              "      <td>True</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>803</th>\n",
              "      <td>positive</td>\n",
              "      <td>In Iran, women are not permitted to attend men's sporting events, apparently to \"protect\" them from all the cursing and foul language they might hear emanating from the male fans (so since men can't restrain or behave themselves, women are forced to suffer. Go figure.). \"Offside\" tells the tale of a half dozen or so young women who, dressed like men, attempt to sneak into the high-stakes match between Iran and Bahrain that, in 2005, qualified Iran to go to the World Cup (the movie was actually filmed in large part during that game).&lt;br /&gt;&lt;br /&gt;\"Offside\" is a slice-of-life comedy that will ...</td>\n",
              "      <td>True</td>\n",
              "      <td>positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>804</th>\n",
              "      <td>positive</td>\n",
              "      <td>\"In April 1946, the University of Chicago agreed to operate Argonne National Laboratory, with an association of Midwestern universities offering to sponsor the research. Argonne thereby became the first \"national\" laboratory. It did not, however, remain at its original location in the Argonne forest. In 1947, it moved farther west from the \"Windy City\" to a new site on Illinois farmland. When Alvin Weinberg visited Argonne's director, Walter Zinn, in 1947, he asked him what kind of reactor was to be built at the new site. When Zinn described a heavy-water reactor operating at one-tenth the...</td>\n",
              "      <td>True</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        label  ... self_distilled_pred_label\n",
              "800  positive  ...                  positive\n",
              "801  positive  ...                  positive\n",
              "802  positive  ...                  positive\n",
              "803  positive  ...                  positive\n",
              "804  positive  ...                  negative\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7BLpjHNhwa0F"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, confusion_matrix"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FaN4TfqJwqt1",
        "outputId": "575077ec-8e07-42a8-8bde-83c9ab4c2630"
      },
      "source": [
        "accuracy_score(df_performance_val[\"label\"], df_performance_val[\"self_distilled_pred_label\"])"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vw3Tp4EQxC0U",
        "outputId": "d5c6f763-6ffe-4056-89b6-211bff4f4c77"
      },
      "source": [
        "confusion_matrix(df_performance_val[\"label\"], df_performance_val[\"self_distilled_pred_label\"])"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[101,   6],\n",
              "       [ 14,  79]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XziDG_3P4Ds3"
      },
      "source": [
        "# Notes\n",
        "\n",
        "1. should psuedolabels be generated for only training data?\n",
        "Thought:\n",
        "- only get pseudo-label for train data\n",
        "- use 1 hot encoding for validation data\n",
        "- optimize for train MSE/MAE (looking at validation MSE isn't important since it's one hot encoded)\n",
        "- callback monitors, `train_loss` or `pseudolabel_accuracy`?\n",
        "\n",
        "2. if pseudolabels should be generated for both train and val data\n",
        "- optimize for MSE then\n",
        "- this seems like the wrong way to go about it?\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2G237ytxqgR"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}